[BUGFIX] Fix several typos in php comments
[Packages/TYPO3.CMS.git] / typo3 / sysext / impexp / Classes / Import.php
1 <?php
2 namespace TYPO3\CMS\Impexp;
3
4 /*
5 * This file is part of the TYPO3 CMS project.
6 *
7 * It is free software; you can redistribute it and/or modify it under
8 * the terms of the GNU General Public License, either version 2
9 * of the License, or any later version.
10 *
11 * For the full copyright and license information, please read the
12 * LICENSE.txt file that was distributed with this source code.
13 *
14 * The TYPO3 project - inspiring people to share!
15 */
16
17 use TYPO3\CMS\Backend\Utility\BackendUtility;
18 use TYPO3\CMS\Core\Configuration\FlexForm\FlexFormTools;
19 use TYPO3\CMS\Core\Core\Environment;
20 use TYPO3\CMS\Core\Database\ConnectionPool;
21 use TYPO3\CMS\Core\DataHandling\DataHandler;
22 use TYPO3\CMS\Core\Exception;
23 use TYPO3\CMS\Core\Resource\Exception\InsufficientFolderAccessPermissionsException;
24 use TYPO3\CMS\Core\Resource\File;
25 use TYPO3\CMS\Core\Resource\FileInterface;
26 use TYPO3\CMS\Core\Resource\ResourceFactory;
27 use TYPO3\CMS\Core\Resource\ResourceStorage;
28 use TYPO3\CMS\Core\Resource\StorageRepository;
29 use TYPO3\CMS\Core\Utility\ExtensionManagementUtility;
30 use TYPO3\CMS\Core\Utility\GeneralUtility;
31 use TYPO3\CMS\Core\Utility\MathUtility;
32 use TYPO3\CMS\Core\Utility\PathUtility;
33 use TYPO3\CMS\Core\Utility\StringUtility;
34
35 /**
36 * T3D file Import library (TYPO3 Record Document)
37 * @internal this is not part of TYPO3's Core API.
38 */
39 class Import extends ImportExport
40 {
41 /**
42 * Used to register the forged UID values for imported records that we want
43 * to create with the same UIDs as in the import file. Admin-only feature.
44 *
45 * @var array
46 */
47 public $suggestedInsertUids = [];
48
49 /**
50 * Disable logging when importing
51 *
52 * @var bool
53 */
54 public $enableLogging = false;
55
56 /**
57 * Keys are [tablename]:[new NEWxxx ids (or when updating it is uids)]
58 * while values are arrays with table/uid of the original record it is based on.
59 * With the array keys the new ids can be looked up inside DataHandler
60 *
61 * @var array
62 */
63 public $import_newId = [];
64
65 /**
66 * Page id map for page tree (import)
67 *
68 * @var array
69 */
70 public $import_newId_pids = [];
71
72 /**
73 * Internal data accumulation for writing records during import
74 *
75 * @var array
76 */
77 public $import_data = [];
78
79 /**
80 * Array of current registered storage objects
81 *
82 * @var ResourceStorage[]
83 */
84 protected $storageObjects = [];
85
86 /**
87 * @var string|null
88 */
89 protected $filesPathForImport;
90
91 /**
92 * @var array
93 */
94 protected $unlinkFiles = [];
95
96 /**
97 * @var array
98 */
99 protected $filePathMap = [];
100
101 /**************************
102 * Initialize
103 *************************/
104
105 /**
106 * Init the object
107 */
108 public function init()
109 {
110 parent::init();
111 $this->mode = 'import';
112 }
113
114 /***********************
115 * Import
116 ***********************/
117
118 /**
119 * Initialize all settings for the import
120 */
121 protected function initializeImport()
122 {
123 // Set this flag to indicate that an import is being/has been done.
124 $this->doesImport = 1;
125 // Initialize:
126 // These vars MUST last for the whole section not being cleared. They are used by the method setRelations() which are called at the end of the import session.
127 $this->import_mapId = [];
128 $this->import_newId = [];
129 $this->import_newId_pids = [];
130 // Temporary files stack initialized:
131 $this->unlinkFiles = [];
132
133 $this->initializeStorageObjects();
134 }
135
136 /**
137 * Initialize the all present storage objects
138 */
139 protected function initializeStorageObjects()
140 {
141 /** @var StorageRepository $storageRepository */
142 $storageRepository = GeneralUtility::makeInstance(StorageRepository::class);
143 $this->storageObjects = $storageRepository->findAll();
144 }
145
146 /**
147 * Imports the internal data array to $pid.
148 *
149 * @param int $pid Page ID in which to import the content
150 */
151 public function importData($pid)
152 {
153 $this->initializeImport();
154
155 // Write sys_file_storages first
156 $this->writeSysFileStorageRecords();
157 // Write sys_file records and write the binary file data
158 $this->writeSysFileRecords();
159 // Write records, first pages, then the rest
160 // Fields with "hard" relations to database, files and flexform fields are kept empty during this run
161 $this->writeRecords_pages($pid);
162 $this->writeRecords_records($pid);
163 // Finally all the file and DB record references must be fixed. This is done after all records have supposedly been written to database:
164 // $this->import_mapId will indicate two things: 1) that a record WAS written to db and 2) that it has got a new id-number.
165 $this->setRelations();
166 // And when all DB relations are in place, we can fix file and DB relations in flexform fields (since data structures often depends on relations to a DS record):
167 $this->setFlexFormRelations();
168 // Unlink temporary files:
169 $this->unlinkTempFiles();
170 // Finally, traverse all records and process softreferences with substitution attributes.
171 $this->processSoftReferences();
172 }
173
174 /**
175 * Imports the sys_file_storage records from internal data array.
176 */
177 protected function writeSysFileStorageRecords()
178 {
179 if (!isset($this->dat['header']['records']['sys_file_storage'])) {
180 return;
181 }
182 $sysFileStorageUidsToBeResetToDefaultStorage = [];
183 foreach ($this->dat['header']['records']['sys_file_storage'] as $sysFileStorageUid => $_) {
184 $storageRecord = $this->dat['records']['sys_file_storage:' . $sysFileStorageUid]['data'];
185 // continue with Local, writable and online storage only
186 if ($storageRecord['driver'] === 'Local' && $storageRecord['is_writable'] && $storageRecord['is_online']) {
187 foreach ($this->storageObjects as $localStorage) {
188 if ($this->isEquivalentObjectStorage($localStorage, $storageRecord)) {
189 $this->import_mapId['sys_file_storage'][$sysFileStorageUid] = $localStorage->getUid();
190 break;
191 }
192 }
193
194 if (!isset($this->import_mapId['sys_file_storage'][$sysFileStorageUid])) {
195 // Local, writable and online storage. Is allowed to be used to later write files in.
196 // Does currently not exist so add the record.
197 $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);
198 }
199 } else {
200 // Storage with non Local drivers could be imported but must not be used to saves files in, because you
201 // could not be sure, that this is supported. The default storage will be used in this case.
202 // It could happen that non writable and non online storage will be created as dupes because you could not
203 // check the detailed configuration options at this point
204 $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);
205 $sysFileStorageUidsToBeResetToDefaultStorage[] = $sysFileStorageUid;
206 }
207 }
208
209 // Importing the added ones
210 $tce = $this->getNewTCE();
211 // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.
212 $tce->reverseOrder = 1;
213 $tce->isImporting = true;
214 $tce->start($this->import_data, []);
215 $tce->process_datamap();
216 $this->addToMapId($tce->substNEWwithIDs);
217
218 $defaultStorageUid = null;
219 // get default storage
220 $defaultStorage = ResourceFactory::getInstance()->getDefaultStorage();
221 if ($defaultStorage !== null) {
222 $defaultStorageUid = $defaultStorage->getUid();
223 }
224 foreach ($sysFileStorageUidsToBeResetToDefaultStorage as $sysFileStorageUidToBeResetToDefaultStorage) {
225 $this->import_mapId['sys_file_storage'][$sysFileStorageUidToBeResetToDefaultStorage] = $defaultStorageUid;
226 }
227
228 // unset the sys_file_storage records to prevent an import in writeRecords_records
229 unset($this->dat['header']['records']['sys_file_storage']);
230 }
231
232 /**
233 * Determines whether the passed storage object and record (sys_file_storage) can be
234 * seen as equivalent during import.
235 *
236 * @param ResourceStorage $storageObject The storage object which should get compared
237 * @param array $storageRecord The storage record which should get compared
238 * @return bool Returns TRUE when both object storages can be seen as equivalent
239 */
240 protected function isEquivalentObjectStorage(ResourceStorage $storageObject, array $storageRecord)
241 {
242 // compare the properties: driver, writable and online
243 if ($storageObject->getDriverType() === $storageRecord['driver']
244 && (bool)$storageObject->isWritable() === (bool)$storageRecord['is_writable']
245 && (bool)$storageObject->isOnline() === (bool)$storageRecord['is_online']
246 ) {
247 $storageRecordConfiguration = ResourceFactory::getInstance()->convertFlexFormDataToConfigurationArray($storageRecord['configuration']);
248 $storageObjectConfiguration = $storageObject->getConfiguration();
249 // compare the properties: pathType and basePath
250 if ($storageRecordConfiguration['pathType'] === $storageObjectConfiguration['pathType']
251 && $storageRecordConfiguration['basePath'] === $storageObjectConfiguration['basePath']
252 ) {
253 return true;
254 }
255 }
256 return false;
257 }
258
259 /**
260 * Checks any prerequisites necessary to get fulfilled before import
261 *
262 * @return array Messages explaining issues which need to get resolved before import
263 */
264 public function checkImportPrerequisites()
265 {
266 $messages = [];
267
268 // Check #1: Extension dependencies
269 $extKeysToInstall = [];
270 foreach ($this->dat['header']['extensionDependencies'] as $extKey) {
271 if (!empty($extKey) && !ExtensionManagementUtility::isLoaded($extKey)) {
272 $extKeysToInstall[] = $extKey;
273 }
274 }
275 if (!empty($extKeysToInstall)) {
276 $messages['missingExtensions'] = 'Before you can install this T3D file you need to install the extensions "'
277 . implode('", "', $extKeysToInstall) . '".';
278 }
279
280 // Check #2: If the path for every local storage object exists.
281 // Else files can't get moved into a newly imported storage.
282 if (!empty($this->dat['header']['records']['sys_file_storage'])) {
283 foreach ($this->dat['header']['records']['sys_file_storage'] as $sysFileStorageUid => $_) {
284 $storageRecord = $this->dat['records']['sys_file_storage:' . $sysFileStorageUid]['data'];
285 // continue with Local, writable and online storage only
286 if ($storageRecord['driver'] === 'Local'
287 && $storageRecord['is_writable']
288 && $storageRecord['is_online']
289 ) {
290 foreach ($this->storageObjects as $localStorage) {
291 if ($this->isEquivalentObjectStorage($localStorage, $storageRecord)) {
292 // There is already an existing storage
293 break;
294 }
295
296 // The storage from the import does not have an equivalent storage
297 // in the current instance (same driver, same path, etc.). Before
298 // the storage record can get inserted later on take care the path
299 // it points to really exists and is accessible.
300 $storageRecordUid = $storageRecord['uid'];
301 // Unset the storage record UID when trying to create the storage object
302 // as the record does not already exist in DB. The constructor of the
303 // storage object will check whether the target folder exists and set the
304 // isOnline flag depending on the outcome.
305 $storageRecord['uid'] = 0;
306 $resourceStorage = ResourceFactory::getInstance()->createStorageObject($storageRecord);
307 if (!$resourceStorage->isOnline()) {
308 $configuration = $resourceStorage->getConfiguration();
309 $messages['resourceStorageFolderMissing_' . $storageRecordUid] =
310 'The resource storage "'
311 . $resourceStorage->getName()
312 . $configuration['basePath']
313 . '" does not exist. Please create the directory prior to starting the import!';
314 }
315 }
316 }
317 }
318 }
319
320 return $messages;
321 }
322
323 /**
324 * Imports the sys_file records and the binary files data from internal data array.
325 */
326 protected function writeSysFileRecords()
327 {
328 if (!isset($this->dat['header']['records']['sys_file'])) {
329 return;
330 }
331 $this->addGeneralErrorsByTable('sys_file');
332
333 // fetch fresh storage records from database
334 $storageRecords = $this->fetchStorageRecords();
335
336 $defaultStorage = ResourceFactory::getInstance()->getDefaultStorage();
337
338 $sanitizedFolderMappings = [];
339
340 foreach ($this->dat['header']['records']['sys_file'] as $sysFileUid => $_) {
341 $fileRecord = $this->dat['records']['sys_file:' . $sysFileUid]['data'];
342
343 $temporaryFile = null;
344 // check if there is the right file already in the local folder
345 if ($this->filesPathForImport !== null) {
346 if (is_file($this->filesPathForImport . '/' . $fileRecord['sha1']) && sha1_file($this->filesPathForImport . '/' . $fileRecord['sha1']) === $fileRecord['sha1']) {
347 $temporaryFile = $this->filesPathForImport . '/' . $fileRecord['sha1'];
348 }
349 }
350
351 // save file to disk
352 if ($temporaryFile === null) {
353 $fileId = md5($fileRecord['storage'] . ':' . $fileRecord['identifier_hash']);
354 $temporaryFile = $this->writeTemporaryFileFromData($fileId);
355 if ($temporaryFile === null) {
356 // error on writing the file. Error message was already added
357 continue;
358 }
359 }
360
361 $originalStorageUid = $fileRecord['storage'];
362 $useStorageFromStorageRecords = false;
363
364 // replace storage id, if an alternative one was registered
365 if (isset($this->import_mapId['sys_file_storage'][$fileRecord['storage']])) {
366 $fileRecord['storage'] = $this->import_mapId['sys_file_storage'][$fileRecord['storage']];
367 $useStorageFromStorageRecords = true;
368 }
369
370 if (empty($fileRecord['storage']) && !$this->isFallbackStorage($fileRecord['storage'])) {
371 // no storage for the file is defined, mostly because of a missing default storage.
372 $this->error('Error: No storage for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $originalStorageUid . '"');
373 continue;
374 }
375
376 // using a storage from the local storage is only allowed, if the uid is present in the
377 // mapping. Only in this case we could be sure, that it's a local, online and writable storage.
378 if ($useStorageFromStorageRecords && isset($storageRecords[$fileRecord['storage']])) {
379 /** @var \TYPO3\CMS\Core\Resource\ResourceStorage $storage */
380 $storage = ResourceFactory::getInstance()->getStorageObject($fileRecord['storage'], $storageRecords[$fileRecord['storage']]);
381 } elseif ($this->isFallbackStorage($fileRecord['storage'])) {
382 $storage = ResourceFactory::getInstance()->getStorageObject(0);
383 } elseif ($defaultStorage !== null) {
384 $storage = $defaultStorage;
385 } else {
386 $this->error('Error: No storage available for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
387 continue;
388 }
389
390 $newFile = null;
391
392 // check, if there is an identical file
393 try {
394 if ($storage->hasFile($fileRecord['identifier'])) {
395 $file = $storage->getFile($fileRecord['identifier']);
396 if ($file->getSha1() === $fileRecord['sha1']) {
397 $newFile = $file;
398 }
399 }
400 } catch (Exception $e) {
401 }
402
403 if ($newFile === null) {
404 $folderName = PathUtility::dirname(ltrim($fileRecord['identifier'], '/'));
405 if (in_array($folderName, $sanitizedFolderMappings)) {
406 $folderName = $sanitizedFolderMappings[$folderName];
407 }
408 if (!$storage->hasFolder($folderName)) {
409 try {
410 $importFolder = $storage->createFolder($folderName);
411 if ($importFolder->getIdentifier() !== $folderName && !in_array($folderName, $sanitizedFolderMappings)) {
412 $sanitizedFolderMappings[$folderName] = $importFolder->getIdentifier();
413 }
414 } catch (Exception $e) {
415 $this->error('Error: Folder "' . $folderName . '" could not be created for file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
416 continue;
417 }
418 } else {
419 $importFolder = $storage->getFolder($folderName);
420 }
421
422 $this->callHook('before_addSysFileRecord', [
423 'fileRecord' => $fileRecord,
424 'importFolder' => $importFolder,
425 'temporaryFile' => $temporaryFile
426 ]);
427
428 try {
429 /** @var File $newFile */
430 $newFile = $storage->addFile($temporaryFile, $importFolder, $fileRecord['name']);
431 } catch (Exception $e) {
432 $this->error('Error: File could not be added to the storage: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
433 continue;
434 }
435
436 if ($newFile->getSha1() !== $fileRecord['sha1']) {
437 $this->error('Error: The hash of the written file is not identical to the import data! File could be corrupted! File: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
438 }
439 }
440
441 // save the new uid in the import id map
442 $this->import_mapId['sys_file'][$fileRecord['uid']] = $newFile->getUid();
443 $this->fixUidLocalInSysFileReferenceRecords($fileRecord['uid'], $newFile->getUid());
444 }
445
446 // unset the sys_file records to prevent an import in writeRecords_records
447 unset($this->dat['header']['records']['sys_file']);
448 // remove all sys_file_reference records that point to file records which are unknown
449 // in the system to prevent exceptions
450 $this->removeSysFileReferenceRecordsFromImportDataWithRelationToMissingFile();
451 }
452
453 /**
454 * Removes all sys_file_reference records from the import data array that are pointing to sys_file records which
455 * are missing not in the import data to prevent exceptions on checking the related file started by the Datahandler.
456 */
457 protected function removeSysFileReferenceRecordsFromImportDataWithRelationToMissingFile()
458 {
459 if (!isset($this->dat['header']['records']['sys_file_reference'])) {
460 return;
461 }
462
463 foreach ($this->dat['header']['records']['sys_file_reference'] as $sysFileReferenceUid => $_) {
464 $fileReferenceRecord = $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'];
465 if (!in_array($fileReferenceRecord['uid_local'], $this->import_mapId['sys_file'])) {
466 unset($this->dat['header']['records']['sys_file_reference'][$sysFileReferenceUid]);
467 unset($this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]);
468 $this->error(
469 'Error: sys_file_reference record ' . (int)$sysFileReferenceUid
470 . ' with relation to sys_file record ' . (int)$fileReferenceRecord['uid_local']
471 . ', which is not part of the import data, was not imported.'
472 );
473 }
474 }
475 }
476
477 /**
478 * Checks if the $storageId is the id of the fallback storage
479 *
480 * @param int|string $storageId
481 * @return bool
482 */
483 protected function isFallbackStorage($storageId)
484 {
485 return $storageId === 0 || $storageId === '0';
486 }
487
488 /**
489 * Normally the importer works like the following:
490 * Step 1: import the records with cleared field values of relation fields (see addSingle())
491 * Step 2: update the records with the right relation ids (see setRelations())
492 *
493 * In step 2 the saving fields of type "relation to sys_file_reference" checks the related sys_file_reference
494 * record (created in step 1) with the FileExtensionFilter for matching file extensions of the related file.
495 * To make this work correct, the uid_local of sys_file_reference records has to be not empty AND has to
496 * relate to the correct (imported) sys_file record uid!!!
497 *
498 * This is fixed here.
499 *
500 * @param int $oldFileUid
501 * @param int $newFileUid
502 */
503 protected function fixUidLocalInSysFileReferenceRecords($oldFileUid, $newFileUid)
504 {
505 if (!isset($this->dat['header']['records']['sys_file_reference'])) {
506 return;
507 }
508
509 foreach ($this->dat['header']['records']['sys_file_reference'] as $sysFileReferenceUid => $_) {
510 if (!isset($this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['hasBeenMapped'])
511 && $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data']['uid_local'] == $oldFileUid
512 ) {
513 $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['hasBeenMapped'] = true;
514 $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data']['uid_local'] = $newFileUid;
515 }
516 }
517 }
518
519 /**
520 * Fetched fresh storage records from database because the new imported
521 * ones are not in cached data of the StorageRepository
522 *
523 * @return bool|array
524 */
525 protected function fetchStorageRecords()
526 {
527 $result = GeneralUtility::makeInstance(ConnectionPool::class)
528 ->getQueryBuilderForTable('sys_file_storage')
529 ->select('*')
530 ->from('sys_file_storage')
531 ->orderBy('uid')
532 ->execute();
533 $rows = [];
534 while ($row = $result->fetch()) {
535 $rows[$row['uid']] = $row;
536 }
537 return $rows;
538 }
539
540 /**
541 * Writes the file from import array to temp dir and returns the filename of it.
542 *
543 * @param string $fileId
544 * @param string $dataKey
545 * @return string Absolute filename of the temporary filename of the file
546 */
547 protected function writeTemporaryFileFromData($fileId, $dataKey = 'files_fal')
548 {
549 $temporaryFilePath = null;
550 if (is_array($this->dat[$dataKey][$fileId])) {
551 $temporaryFilePathInternal = GeneralUtility::tempnam('import_temp_');
552 GeneralUtility::writeFile($temporaryFilePathInternal, $this->dat[$dataKey][$fileId]['content']);
553 clearstatcache();
554 if (@is_file($temporaryFilePathInternal)) {
555 $this->unlinkFiles[] = $temporaryFilePathInternal;
556 $temporaryFilePath = $temporaryFilePathInternal;
557 } else {
558 $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' was not written as it should have been!');
559 }
560 } else {
561 $this->error('Error: No file found for ID ' . $fileId);
562 }
563 return $temporaryFilePath;
564 }
565
566 /**
567 * Writing pagetree/pages to database:
568 *
569 * @param int $pid PID in which to import. If the operation is an update operation, the root of the page tree inside will be moved to this PID unless it is the same as the root page from the import
570 * @see writeRecords_records()
571 */
572 public function writeRecords_pages($pid)
573 {
574 // First, write page structure if any:
575 if (is_array($this->dat['header']['records']['pages'])) {
576 $this->addGeneralErrorsByTable('pages');
577 // $pageRecords is a copy of the pages array in the imported file. Records here are unset one by one when the addSingle function is called.
578 $pageRecords = $this->dat['header']['records']['pages'];
579 $this->import_data = [];
580 // First add page tree if any
581 if (is_array($this->dat['header']['pagetree'])) {
582 $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);
583 foreach ($pagesFromTree as $uid) {
584 $thisRec = $this->dat['header']['records']['pages'][$uid];
585 // PID: Set the main $pid, unless a NEW-id is found
586 $setPid = $this->import_newId_pids[$thisRec['pid']] ?? $pid;
587 $this->addSingle('pages', $uid, $setPid);
588 unset($pageRecords[$uid]);
589 }
590 }
591 // Then add all remaining pages not in tree on root level:
592 if (!empty($pageRecords)) {
593 $remainingPageUids = array_keys($pageRecords);
594 foreach ($remainingPageUids as $pUid) {
595 $this->addSingle('pages', $pUid, $pid);
596 }
597 }
598 // Now write to database:
599 $tce = $this->getNewTCE();
600 $tce->isImporting = true;
601 $this->callHook('before_writeRecordsPages', [
602 'tce' => &$tce,
603 'data' => &$this->import_data
604 ]);
605 $tce->suggestedInsertUids = $this->suggestedInsertUids;
606 $tce->start($this->import_data, []);
607 $tce->process_datamap();
608 $this->callHook('after_writeRecordsPages', [
609 'tce' => &$tce
610 ]);
611 // post-processing: Registering new ids (end all DataHandler sessions with this)
612 $this->addToMapId($tce->substNEWwithIDs);
613 // In case of an update, order pages from the page tree correctly:
614 if ($this->update && is_array($this->dat['header']['pagetree'])) {
615 $this->writeRecords_pages_order();
616 }
617 }
618 }
619
620 /**
621 * Organize all updated pages in page tree so they are related like in the import file
622 * Only used for updates and when $this->dat['header']['pagetree'] is an array.
623 *
624 * @internal
625 * @see writeRecords_pages()
626 * @see writeRecords_records_order()
627 */
628 public function writeRecords_pages_order()
629 {
630 $cmd_data = [];
631 // Get uid-pid relations and traverse them in order to map to possible new IDs
632 $pidsFromTree = $this->flatInversePageTree_pid($this->dat['header']['pagetree']);
633 foreach ($pidsFromTree as $origPid => $newPid) {
634 if ($newPid >= 0 && $this->dontIgnorePid('pages', $origPid)) {
635 // If the page had a new id (because it was created) use that instead!
636 if (strpos($this->import_newId_pids[$origPid], 'NEW') === 0) {
637 if ($this->import_mapId['pages'][$origPid]) {
638 $mappedPid = $this->import_mapId['pages'][$origPid];
639 $cmd_data['pages'][$mappedPid]['move'] = $newPid;
640 }
641 } else {
642 $cmd_data['pages'][$origPid]['move'] = $newPid;
643 }
644 }
645 }
646 // Execute the move commands if any:
647 if (!empty($cmd_data)) {
648 $tce = $this->getNewTCE();
649 $this->callHook('before_writeRecordsPagesOrder', [
650 'tce' => &$tce,
651 'data' => &$cmd_data
652 ]);
653 $tce->start([], $cmd_data);
654 $tce->process_cmdmap();
655 $this->callHook('after_writeRecordsPagesOrder', [
656 'tce' => &$tce
657 ]);
658 }
659 }
660
661 /**
662 * Recursively flattening the idH array, setting PIDs as values
663 *
664 * @param array $idH Page uid hierarchy
665 * @param array $a Accumulation array of pages (internal, don't set from outside)
666 * @param int $pid PID value (internal)
667 * @return array Array with uid-pid pairs for all pages in the page tree.
668 * @see ImportExport::flatInversePageTree()
669 */
670 public function flatInversePageTree_pid($idH, $a = [], $pid = -1)
671 {
672 if (is_array($idH)) {
673 $idH = array_reverse($idH);
674 foreach ($idH as $v) {
675 $a[$v['uid']] = $pid;
676 if (is_array($v['subrow'])) {
677 $a = $this->flatInversePageTree_pid($v['subrow'], $a, $v['uid']);
678 }
679 }
680 }
681 return $a;
682 }
683
684 /**
685 * Write all database records except pages (written in writeRecords_pages())
686 *
687 * @param int $pid Page id in which to import
688 * @see writeRecords_pages()
689 */
690 public function writeRecords_records($pid)
691 {
692 // Write the rest of the records
693 $this->import_data = [];
694 if (is_array($this->dat['header']['records'])) {
695 foreach ($this->dat['header']['records'] as $table => $recs) {
696 $this->addGeneralErrorsByTable($table);
697 if ($table !== 'pages') {
698 foreach ($recs as $uid => $thisRec) {
699 // PID: Set the main $pid, unless a NEW-id is found
700 $setPid = isset($this->import_mapId['pages'][$thisRec['pid']])
701 ? (int)$this->import_mapId['pages'][$thisRec['pid']]
702 : (int)$pid;
703 if (is_array($GLOBALS['TCA'][$table]) && isset($GLOBALS['TCA'][$table]['ctrl']['rootLevel'])) {
704 $rootLevelSetting = (int)$GLOBALS['TCA'][$table]['ctrl']['rootLevel'];
705 if ($rootLevelSetting === 1) {
706 $setPid = 0;
707 } elseif ($rootLevelSetting === 0 && $setPid === 0) {
708 $this->error('Error: Record type ' . $table . ' is not allowed on pid 0');
709 continue;
710 }
711 }
712 // Add record:
713 $this->addSingle($table, $uid, $setPid);
714 }
715 }
716 }
717 } else {
718 $this->error('Error: No records defined in internal data array.');
719 }
720 // Now write to database:
721 $tce = $this->getNewTCE();
722 $this->callHook('before_writeRecordsRecords', [
723 'tce' => &$tce,
724 'data' => &$this->import_data
725 ]);
726 $tce->suggestedInsertUids = $this->suggestedInsertUids;
727 // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.
728 $tce->reverseOrder = 1;
729 $tce->isImporting = true;
730 $tce->start($this->import_data, []);
731 $tce->process_datamap();
732 $this->callHook('after_writeRecordsRecords', [
733 'tce' => &$tce
734 ]);
735 // post-processing: Removing files and registering new ids (end all DataHandler sessions with this)
736 $this->addToMapId($tce->substNEWwithIDs);
737 // In case of an update, order pages from the page tree correctly:
738 if ($this->update) {
739 $this->writeRecords_records_order($pid);
740 }
741 }
742
743 /**
744 * Organize all updated record to their new positions.
745 * Only used for updates
746 *
747 * @param int $mainPid Main PID into which we import.
748 * @internal
749 * @see writeRecords_records()
750 * @see writeRecords_pages_order()
751 */
752 public function writeRecords_records_order($mainPid)
753 {
754 $cmd_data = [];
755 if (is_array($this->dat['header']['pagetree'])) {
756 $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);
757 } else {
758 $pagesFromTree = [];
759 }
760 if (is_array($this->dat['header']['pid_lookup'])) {
761 foreach ($this->dat['header']['pid_lookup'] as $pid => $recList) {
762 $newPid = $this->import_mapId['pages'][$pid] ?? $mainPid;
763 if (MathUtility::canBeInterpretedAsInteger($newPid)) {
764 foreach ($recList as $tableName => $uidList) {
765 // If $mainPid===$newPid then we are on root level and we can consider to move pages as well!
766 // (they will not be in the page tree!)
767 if (($tableName !== 'pages' || !$pagesFromTree[$pid]) && is_array($uidList)) {
768 $uidList = array_reverse(array_keys($uidList));
769 foreach ($uidList as $uid) {
770 if ($this->dontIgnorePid($tableName, $uid)) {
771 $cmd_data[$tableName][$uid]['move'] = $newPid;
772 }
773 }
774 }
775 }
776 }
777 }
778 }
779 // Execute the move commands if any:
780 if (!empty($cmd_data)) {
781 $tce = $this->getNewTCE();
782 $this->callHook('before_writeRecordsRecordsOrder', [
783 'tce' => &$tce,
784 'data' => &$cmd_data
785 ]);
786 $tce->start([], $cmd_data);
787 $tce->process_cmdmap();
788 $this->callHook('after_writeRecordsRecordsOrder', [
789 'tce' => &$tce
790 ]);
791 }
792 }
793
794 /**
795 * Adds a single record to the $importData array. Also copies files to tempfolder.
796 * However all File/DB-references and flexform field contents are set to blank for now!
797 * That is done with setRelations() later
798 *
799 * @param string $table Table name (from import memory)
800 * @param int $uid Record UID (from import memory)
801 * @param int $pid Page id
802 * @see writeRecords()
803 */
804 public function addSingle($table, $uid, $pid)
805 {
806 if ($this->import_mode[$table . ':' . $uid] === 'exclude') {
807 return;
808 }
809 $record = $this->dat['records'][$table . ':' . $uid]['data'];
810 if (is_array($record)) {
811 if ($this->update && $this->doesRecordExist($table, $uid) && $this->import_mode[$table . ':' . $uid] !== 'as_new') {
812 $ID = $uid;
813 } elseif ($table === 'sys_file_metadata' && $record['sys_language_uid'] == '0' && $this->import_mapId['sys_file'][$record['file']]) {
814 // on adding sys_file records the belonging sys_file_metadata record was also created
815 // if there is one the record need to be overwritten instead of creating a new one.
816 $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
817 ->getQueryBuilderForTable('sys_file_metadata');
818 $recordInDatabase = $queryBuilder->select('uid')
819 ->from('sys_file_metadata')
820 ->where(
821 $queryBuilder->expr()->eq(
822 'file',
823 $queryBuilder->createNamedParameter(
824 $this->import_mapId['sys_file'][$record['file']],
825 \PDO::PARAM_INT
826 )
827 ),
828 $queryBuilder->expr()->eq(
829 'sys_language_uid',
830 $queryBuilder->createNamedParameter(0, \PDO::PARAM_INT)
831 ),
832 $queryBuilder->expr()->eq(
833 'pid',
834 $queryBuilder->createNamedParameter(0, \PDO::PARAM_INT)
835 )
836 )
837 ->execute()
838 ->fetch();
839 // if no record could be found, $this->import_mapId['sys_file'][$record['file']] is pointing
840 // to a file, that was already there, thus a new metadata record should be created
841 if (is_array($recordInDatabase)) {
842 $this->import_mapId['sys_file_metadata'][$record['uid']] = $recordInDatabase['uid'];
843 $ID = $recordInDatabase['uid'];
844 } else {
845 $ID = StringUtility::getUniqueId('NEW');
846 }
847 } else {
848 $ID = StringUtility::getUniqueId('NEW');
849 }
850 $this->import_newId[$table . ':' . $ID] = ['table' => $table, 'uid' => $uid];
851 if ($table === 'pages') {
852 $this->import_newId_pids[$uid] = $ID;
853 }
854 // Set main record data:
855 $this->import_data[$table][$ID] = $record;
856 $this->import_data[$table][$ID]['tx_impexp_origuid'] = $this->import_data[$table][$ID]['uid'];
857 // Reset permission data:
858 if ($table === 'pages') {
859 // Have to reset the user/group IDs so pages are owned by importing user. Otherwise strange things may happen for non-admins!
860 unset($this->import_data[$table][$ID]['perms_userid']);
861 unset($this->import_data[$table][$ID]['perms_groupid']);
862 }
863 // PID and UID:
864 unset($this->import_data[$table][$ID]['uid']);
865 // Updates:
866 if (MathUtility::canBeInterpretedAsInteger($ID)) {
867 unset($this->import_data[$table][$ID]['pid']);
868 } else {
869 // Inserts:
870 $this->import_data[$table][$ID]['pid'] = $pid;
871 if (($this->import_mode[$table . ':' . $uid] === 'force_uid' && $this->update || $this->force_all_UIDS) && $this->getBackendUser()->isAdmin()) {
872 $this->import_data[$table][$ID]['uid'] = $uid;
873 $this->suggestedInsertUids[$table . ':' . $uid] = 'DELETE';
874 }
875 }
876 // Setting db/file blank:
877 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
878 switch ((string)$config['type']) {
879 case 'db':
880
881 case 'file':
882 // Fixed later in ->setRelations() [because we need to know ALL newly created IDs before we can map relations!]
883 // In the meantime we set NO values for relations.
884 //
885 // BUT for field uid_local of table sys_file_reference the relation MUST not be cleared here,
886 // because the value is already the uid of the right imported sys_file record.
887 // @see fixUidLocalInSysFileReferenceRecords()
888 // If it's empty or a uid to another record the FileExtensionFilter will throw an exception or
889 // delete the reference record if the file extension of the related record doesn't match.
890 if ($table !== 'sys_file_reference' && $field !== 'uid_local') {
891 $this->import_data[$table][$ID][$field] = '';
892 }
893 break;
894 case 'flex':
895 // Fixed later in setFlexFormRelations()
896 // In the meantime we set NO value for flexforms - this is mainly because file references
897 // inside will not be processed properly; In fact references will point to no file
898 // or existing files (in which case there will be double-references which is a big problem of course!)
899 $this->import_data[$table][$ID][$field] = '';
900 break;
901 }
902 }
903 } elseif ($table . ':' . $uid != 'pages:0') {
904 // On root level we don't want this error message.
905 $this->error('Error: no record was found in data array!');
906 }
907 }
908
909 /**
910 * Registers the substNEWids in memory.
911 *
912 * @param array $substNEWwithIDs From DataHandler to be merged into internal mapping variable in this object
913 * @see writeRecords()
914 */
915 public function addToMapId($substNEWwithIDs)
916 {
917 foreach ($this->import_data as $table => $recs) {
918 foreach ($recs as $id => $value) {
919 $old_uid = $this->import_newId[$table . ':' . $id]['uid'];
920 if (isset($substNEWwithIDs[$id])) {
921 $this->import_mapId[$table][$old_uid] = $substNEWwithIDs[$id];
922 } elseif ($this->update) {
923 // Map same ID to same ID....
924 $this->import_mapId[$table][$old_uid] = $id;
925 } else {
926 // if $this->import_mapId contains already the right mapping, skip the error msg.
927 // See special handling of sys_file_metadata in addSingle() => nothing to do
928 if (!($table === 'sys_file_metadata' && isset($this->import_mapId[$table][$old_uid]) && $this->import_mapId[$table][$old_uid] == $id)) {
929 $this->error('Possible error: ' . $table . ':' . $old_uid . ' had no new id assigned to it. This indicates that the record was not added to database during import. Please check changelog!');
930 }
931 }
932 }
933 }
934 }
935
936 /**
937 * Returns a new $TCE object
938 *
939 * @return DataHandler $TCE object
940 */
941 public function getNewTCE()
942 {
943 $tce = GeneralUtility::makeInstance(DataHandler::class);
944 $tce->dontProcessTransformations = 1;
945 $tce->enableLogging = $this->enableLogging;
946 return $tce;
947 }
948
949 /**
950 * Cleaning up all the temporary files stored in typo3temp/ folder
951 */
952 public function unlinkTempFiles()
953 {
954 foreach ($this->unlinkFiles as $fileName) {
955 if (GeneralUtility::isFirstPartOfStr($fileName, Environment::getPublicPath() . '/typo3temp/')) {
956 GeneralUtility::unlink_tempfile($fileName);
957 clearstatcache();
958 if (is_file($fileName)) {
959 $this->error('Error: ' . $fileName . ' was NOT unlinked as it should have been!');
960 }
961 } else {
962 $this->error('Error: ' . $fileName . ' was not in temp-path. Not removed!');
963 }
964 }
965 $this->unlinkFiles = [];
966 }
967
968 /***************************
969 * Import / Relations setting
970 ***************************/
971
972 /**
973 * At the end of the import process all file and DB relations should be set properly (that is relations
974 * to imported records are all re-created so imported records are correctly related again)
975 * Relations in flexform fields are processed in setFlexFormRelations() after this function
976 *
977 * @see setFlexFormRelations()
978 */
979 public function setRelations()
980 {
981 $updateData = [];
982 // import_newId contains a register of all records that was in the import memorys "records" key
983 foreach ($this->import_newId as $nId => $dat) {
984 $table = $dat['table'];
985 $uid = $dat['uid'];
986 // original UID - NOT the new one!
987 // If the record has been written and received a new id, then proceed:
988 if (is_array($this->import_mapId[$table]) && isset($this->import_mapId[$table][$uid])) {
989 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
990 if (is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {
991 // Traverse relation fields of each record
992 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
993 // uid_local of sys_file_reference needs no update because the correct reference uid was already written
994 // @see ImportExport::fixUidLocalInSysFileReferenceRecords()
995 if ($table === 'sys_file_reference' && $field === 'uid_local') {
996 continue;
997 }
998 switch ((string)$config['type']) {
999 case 'db':
1000 if (is_array($config['itemArray']) && !empty($config['itemArray'])) {
1001 $itemConfig = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
1002 $valArray = $this->setRelations_db($config['itemArray'], $itemConfig);
1003 $updateData[$table][$thisNewUid][$field] = implode(',', $valArray);
1004 }
1005 break;
1006 case 'file':
1007 if (is_array($config['newValueFiles']) && !empty($config['newValueFiles'])) {
1008 $valArr = [];
1009 foreach ($config['newValueFiles'] as $fI) {
1010 $valArr[] = $this->import_addFileNameToBeCopied($fI);
1011 }
1012 $updateData[$table][$thisNewUid][$field] = implode(',', $valArr);
1013 }
1014 break;
1015 }
1016 }
1017 } else {
1018 $this->error('Error: no record was found in data array!');
1019 }
1020 } else {
1021 $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')');
1022 }
1023 }
1024 if (!empty($updateData)) {
1025 $tce = $this->getNewTCE();
1026 $tce->isImporting = true;
1027 $this->callHook('before_setRelation', [
1028 'tce' => &$tce,
1029 'data' => &$updateData
1030 ]);
1031 $tce->start($updateData, []);
1032 $tce->process_datamap();
1033 $this->callHook('after_setRelations', [
1034 'tce' => &$tce
1035 ]);
1036 }
1037 }
1038
1039 /**
1040 * Maps relations for database
1041 *
1042 * @param array $itemArray Array of item sets (table/uid) from a dbAnalysis object
1043 * @param array $itemConfig Array of TCA config of the field the relation to be set on
1044 * @return array Array with values [table]_[uid] or [uid] for field of type group / internal_type file_reference. These values have the regular DataHandler-input group/select type which means they will automatically be processed into a uid-list or MM relations.
1045 */
1046 public function setRelations_db($itemArray, $itemConfig)
1047 {
1048 $valArray = [];
1049 foreach ($itemArray as $relDat) {
1050 if (is_array($this->import_mapId[$relDat['table']]) && isset($this->import_mapId[$relDat['table']][$relDat['id']])) {
1051 if ($itemConfig['type'] === 'input' && isset($itemConfig['wizards']['link'])) {
1052 // If an input field has a relation to a sys_file record this need to be converted back to
1053 // the public path. But use getPublicUrl here, because could normally only be a local file path.
1054 $fileUid = $this->import_mapId[$relDat['table']][$relDat['id']];
1055 // Fallback value
1056 $value = 'file:' . $fileUid;
1057 try {
1058 $file = ResourceFactory::getInstance()->retrieveFileOrFolderObject($fileUid);
1059 } catch (\Exception $e) {
1060 $file = null;
1061 }
1062 if ($file instanceof FileInterface) {
1063 $value = $file->getPublicUrl();
1064 }
1065 } else {
1066 $value = $relDat['table'] . '_' . $this->import_mapId[$relDat['table']][$relDat['id']];
1067 }
1068 $valArray[] = $value;
1069 } elseif ($this->isTableStatic($relDat['table']) || $this->isExcluded($relDat['table'], $relDat['id']) || $relDat['id'] < 0) {
1070 // Checking for less than zero because some select types could contain negative values,
1071 // eg. fe_groups (-1, -2) and sys_language (-1 = ALL languages). This must be handled on both export and import.
1072 $valArray[] = $relDat['table'] . '_' . $relDat['id'];
1073 } else {
1074 $this->error('Lost relation: ' . $relDat['table'] . ':' . $relDat['id']);
1075 }
1076 }
1077 return $valArray;
1078 }
1079
1080 /**
1081 * Writes the file from import array to temp dir and returns the filename of it.
1082 *
1083 * @param array $fI File information with three keys: "filename" = filename without path, "ID_absFile" = absolute filepath to the file (including the filename), "ID" = md5 hash of "ID_absFile
1084 * @return string|null Absolute filename of the temporary filename of the file.
1085 */
1086 public function import_addFileNameToBeCopied($fI)
1087 {
1088 if (is_array($this->dat['files'][$fI['ID']])) {
1089 $tmpFile = null;
1090 // check if there is the right file already in the local folder
1091 if ($this->filesPathForImport !== null) {
1092 if (is_file($this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5']) &&
1093 md5_file($this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5']) === $this->dat['files'][$fI['ID']]['content_md5']) {
1094 $tmpFile = $this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5'];
1095 }
1096 }
1097 if ($tmpFile === null) {
1098 $tmpFile = GeneralUtility::tempnam('import_temp_');
1099 GeneralUtility::writeFile($tmpFile, $this->dat['files'][$fI['ID']]['content']);
1100 }
1101 clearstatcache();
1102 if (@is_file($tmpFile)) {
1103 $this->unlinkFiles[] = $tmpFile;
1104 return $tmpFile;
1105 }
1106 $this->error('Error: temporary file ' . $tmpFile . ' was not written as it should have been!');
1107 } else {
1108 $this->error('Error: No file found for ID ' . $fI['ID']);
1109 }
1110 return null;
1111 }
1112
1113 /**
1114 * After all DB relations has been set in the end of the import (see setRelations()) then it is time to correct all relations inside of FlexForm fields.
1115 * The reason for doing this after is that the setting of relations may affect (quite often!) which data structure is used for the flexforms field!
1116 *
1117 * @see setRelations()
1118 */
1119 public function setFlexFormRelations()
1120 {
1121 $updateData = [];
1122 // import_newId contains a register of all records that was in the import memorys "records" key
1123 foreach ($this->import_newId as $nId => $dat) {
1124 $table = $dat['table'];
1125 $uid = $dat['uid'];
1126 // original UID - NOT the new one!
1127 // If the record has been written and received a new id, then proceed:
1128 if (!isset($this->import_mapId[$table][$uid])) {
1129 $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')');
1130 continue;
1131 }
1132
1133 if (!is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {
1134 $this->error('Error: no record was found in data array!');
1135 continue;
1136 }
1137 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
1138 // Traverse relation fields of each record
1139 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
1140 switch ((string)$config['type']) {
1141 case 'flex':
1142 // Get XML content and set as default value (string, non-processed):
1143 $updateData[$table][$thisNewUid][$field] = $this->dat['records'][$table . ':' . $uid]['data'][$field];
1144 // If there has been registered relations inside the flex form field, run processing on the content:
1145 if (!empty($config['flexFormRels']['db']) || !empty($config['flexFormRels']['file'])) {
1146 $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');
1147 // This will fetch the new row for the element (which should be updated with any references to data structures etc.)
1148 $fieldTca = $GLOBALS['TCA'][$table]['columns'][$field];
1149 if (is_array($origRecordRow) && is_array($fieldTca['config']) && $fieldTca['config']['type'] === 'flex') {
1150 // Get current data structure and value array:
1151 $flexFormTools = GeneralUtility::makeInstance(FlexFormTools::class);
1152 $dataStructureIdentifier = $flexFormTools->getDataStructureIdentifier(
1153 $fieldTca,
1154 $table,
1155 $field,
1156 $origRecordRow
1157 );
1158 $dataStructureArray = $flexFormTools->parseDataStructureByIdentifier($dataStructureIdentifier);
1159 $currentValueArray = GeneralUtility::xml2array($updateData[$table][$thisNewUid][$field]);
1160 // Do recursive processing of the XML data:
1161 $iteratorObj = GeneralUtility::makeInstance(DataHandler::class);
1162 $iteratorObj->callBackObj = $this;
1163 $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData(
1164 $currentValueArray['data'],
1165 [],
1166 [],
1167 $dataStructureArray,
1168 [$table, $thisNewUid, $field, $config],
1169 'remapListedDBRecords_flexFormCallBack'
1170 );
1171 // The return value is set as an array which means it will be processed by DataHandler for file and DB references!
1172 if (is_array($currentValueArray['data'])) {
1173 $updateData[$table][$thisNewUid][$field] = $currentValueArray;
1174 }
1175 }
1176 }
1177 break;
1178 }
1179 }
1180 }
1181 if (!empty($updateData)) {
1182 $tce = $this->getNewTCE();
1183 $tce->isImporting = true;
1184 $this->callHook('before_setFlexFormRelations', [
1185 'tce' => &$tce,
1186 'data' => &$updateData
1187 ]);
1188 $tce->start($updateData, []);
1189 $tce->process_datamap();
1190 $this->callHook('after_setFlexFormRelations', [
1191 'tce' => &$tce
1192 ]);
1193 }
1194 }
1195
1196 /**
1197 * Callback function for traversing the FlexForm structure in relation to remapping database relations
1198 *
1199 * @param array $pParams Set of parameters in numeric array: table, uid, field
1200 * @param array $dsConf TCA config for field (from Data Structure of course)
1201 * @param string $dataValue Field value (from FlexForm XML)
1202 * @param string $dataValue_ext1 Not used
1203 * @param string $dataValue_ext2 Not used
1204 * @param string $path Path of where the data structure of the element is found
1205 * @return array Array where the "value" key carries the value.
1206 * @see setFlexFormRelations()
1207 */
1208 public function remapListedDBRecords_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path)
1209 {
1210 // Extract parameters:
1211 list(, , , $config) = $pParams;
1212 // In case the $path is used as index without a trailing slash we will remove that
1213 if (!is_array($config['flexFormRels']['db'][$path]) && is_array($config['flexFormRels']['db'][rtrim($path, '/')])) {
1214 $path = rtrim($path, '/');
1215 }
1216 if (is_array($config['flexFormRels']['db'][$path])) {
1217 $valArray = $this->setRelations_db($config['flexFormRels']['db'][$path], $dsConf);
1218 $dataValue = implode(',', $valArray);
1219 }
1220 if (is_array($config['flexFormRels']['file'][$path])) {
1221 $valArr = [];
1222 foreach ($config['flexFormRels']['file'][$path] as $fI) {
1223 $valArr[] = $this->import_addFileNameToBeCopied($fI);
1224 }
1225 $dataValue = implode(',', $valArr);
1226 }
1227 return ['value' => $dataValue];
1228 }
1229
1230 /**************************
1231 * Import / Soft References
1232 *************************/
1233
1234 /**
1235 * Processing of soft references
1236 */
1237 public function processSoftReferences()
1238 {
1239 // Initialize:
1240 $inData = [];
1241 // Traverse records:
1242 if (is_array($this->dat['header']['records'])) {
1243 foreach ($this->dat['header']['records'] as $table => $recs) {
1244 foreach ($recs as $uid => $thisRec) {
1245 // If there are soft references defined, traverse those:
1246 if (isset($GLOBALS['TCA'][$table]) && is_array($thisRec['softrefs'])) {
1247 // First traversal is to collect softref configuration and split them up based on fields.
1248 // This could probably also have been done with the "records" key instead of the header.
1249 $fieldsIndex = [];
1250 foreach ($thisRec['softrefs'] as $softrefDef) {
1251 // If a substitution token is set:
1252 if ($softrefDef['field'] && is_array($softrefDef['subst']) && $softrefDef['subst']['tokenID']) {
1253 $fieldsIndex[$softrefDef['field']][$softrefDef['subst']['tokenID']] = $softrefDef;
1254 }
1255 }
1256 // The new id:
1257 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
1258 // Now, if there are any fields that require substitution to be done, lets go for that:
1259 foreach ($fieldsIndex as $field => $softRefCfgs) {
1260 if (is_array($GLOBALS['TCA'][$table]['columns'][$field])) {
1261 if ($GLOBALS['TCA'][$table]['columns'][$field]['config']['type'] === 'flex') {
1262 // This will fetch the new row for the element (which should be updated with any references to data structures etc.)
1263 $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');
1264 if (is_array($origRecordRow)) {
1265 // Get current data structure and value array:
1266 $flexFormTools = GeneralUtility::makeInstance(FlexFormTools::class);
1267 $dataStructureIdentifier = $flexFormTools->getDataStructureIdentifier(
1268 $GLOBALS['TCA'][$table]['columns'][$field],
1269 $table,
1270 $field,
1271 $origRecordRow
1272 );
1273 $dataStructureArray = $flexFormTools->parseDataStructureByIdentifier($dataStructureIdentifier);
1274 $currentValueArray = GeneralUtility::xml2array($origRecordRow[$field]);
1275 // Do recursive processing of the XML data:
1276 /** @var DataHandler $iteratorObj */
1277 $iteratorObj = GeneralUtility::makeInstance(DataHandler::class);
1278 $iteratorObj->callBackObj = $this;
1279 $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData($currentValueArray['data'], [], [], $dataStructureArray, [$table, $uid, $field, $softRefCfgs], 'processSoftReferences_flexFormCallBack');
1280 // The return value is set as an array which means it will be processed by DataHandler for file and DB references!
1281 if (is_array($currentValueArray['data'])) {
1282 $inData[$table][$thisNewUid][$field] = $currentValueArray;
1283 }
1284 }
1285 } else {
1286 // Get tokenizedContent string and proceed only if that is not blank:
1287 $tokenizedContent = $this->dat['records'][$table . ':' . $uid]['rels'][$field]['softrefs']['tokenizedContent'];
1288 if (strlen($tokenizedContent) && is_array($softRefCfgs)) {
1289 $inData[$table][$thisNewUid][$field] = $this->processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid);
1290 }
1291 }
1292 }
1293 }
1294 }
1295 }
1296 }
1297 }
1298 // Now write to database:
1299 $tce = $this->getNewTCE();
1300 $tce->isImporting = true;
1301 $this->callHook('before_processSoftReferences', [
1302 'tce' => $tce,
1303 'data' => &$inData
1304 ]);
1305 $tce->enableLogging = true;
1306 $tce->start($inData, []);
1307 $tce->process_datamap();
1308 $this->callHook('after_processSoftReferences', [
1309 'tce' => $tce
1310 ]);
1311 }
1312
1313 /**
1314 * Callback function for traversing the FlexForm structure in relation to remapping softreference relations
1315 *
1316 * @param array $pParams Set of parameters in numeric array: table, uid, field
1317 * @param array $dsConf TCA config for field (from Data Structure of course)
1318 * @param string $dataValue Field value (from FlexForm XML)
1319 * @param string $dataValue_ext1 Not used
1320 * @param string $dataValue_ext2 Not used
1321 * @param string $path Path of where the data structure where the element is found
1322 * @return array Array where the "value" key carries the value.
1323 * @see setFlexFormRelations()
1324 */
1325 public function processSoftReferences_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path)
1326 {
1327 // Extract parameters:
1328 list($table, $origUid, $field, $softRefCfgs) = $pParams;
1329 if (is_array($softRefCfgs)) {
1330 // First, find all soft reference configurations for this structure path (they are listed flat in the header):
1331 $thisSoftRefCfgList = [];
1332 foreach ($softRefCfgs as $sK => $sV) {
1333 if ($sV['structurePath'] === $path) {
1334 $thisSoftRefCfgList[$sK] = $sV;
1335 }
1336 }
1337 // If any was found, do processing:
1338 if (!empty($thisSoftRefCfgList)) {
1339 // Get tokenizedContent string and proceed only if that is not blank:
1340 $tokenizedContent = $this->dat['records'][$table . ':' . $origUid]['rels'][$field]['flexFormRels']['softrefs'][$path]['tokenizedContent'];
1341 if (strlen($tokenizedContent)) {
1342 $dataValue = $this->processSoftReferences_substTokens($tokenizedContent, $thisSoftRefCfgList, $table, $origUid);
1343 }
1344 }
1345 }
1346 // Return
1347 return ['value' => $dataValue];
1348 }
1349
1350 /**
1351 * Substition of softreference tokens
1352 *
1353 * @param string $tokenizedContent Content of field with soft reference tokens in.
1354 * @param array $softRefCfgs Soft reference configurations
1355 * @param string $table Table for which the processing occurs
1356 * @param string $uid UID of record from table
1357 * @return string The input content with tokens substituted according to entries in softRefCfgs
1358 */
1359 public function processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid)
1360 {
1361 // traverse each softref type for this field:
1362 foreach ($softRefCfgs as $cfg) {
1363 // Get token ID:
1364 $tokenID = $cfg['subst']['tokenID'];
1365 // Default is current token value:
1366 $insertValue = $cfg['subst']['tokenValue'];
1367 // Based on mode:
1368 switch ((string)$this->softrefCfg[$tokenID]['mode']) {
1369 case 'exclude':
1370 // Exclude is a simple passthrough of the value
1371 break;
1372 case 'editable':
1373 // Editable always picks up the value from this input array:
1374 $insertValue = $this->softrefInputValues[$tokenID];
1375 break;
1376 default:
1377 // Mapping IDs/creating files: Based on type, look up new value:
1378 switch ((string)$cfg['subst']['type']) {
1379 case 'file':
1380 // Create / Overwrite file:
1381 $insertValue = $this->processSoftReferences_saveFile($cfg['subst']['relFileName'], $cfg, $table, $uid);
1382 break;
1383 case 'db':
1384 default:
1385 // Trying to map database element if found in the mapID array:
1386 list($tempTable, $tempUid) = explode(':', $cfg['subst']['recordRef']);
1387 if (isset($this->import_mapId[$tempTable][$tempUid])) {
1388 $insertValue = BackendUtility::wsMapId($tempTable, $this->import_mapId[$tempTable][$tempUid]);
1389 if (strpos($cfg['subst']['tokenValue'], ':') !== false) {
1390 list($tokenKey) = explode(':', $cfg['subst']['tokenValue']);
1391 $insertValue = $tokenKey . ':' . $insertValue;
1392 }
1393 }
1394 }
1395 }
1396 // Finally, swap the soft reference token in tokenized content with the insert value:
1397 $tokenizedContent = str_replace('{softref:' . $tokenID . '}', $insertValue, $tokenizedContent);
1398 }
1399 return $tokenizedContent;
1400 }
1401
1402 /**
1403 * Process a soft reference file
1404 *
1405 * @param string $relFileName Old Relative filename
1406 * @param array $cfg soft reference configuration array
1407 * @param string $table Table for which the processing occurs
1408 * @param string $uid UID of record from table
1409 * @return string New relative filename (value to insert instead of the softref token)
1410 */
1411 public function processSoftReferences_saveFile($relFileName, $cfg, $table, $uid)
1412 {
1413 if ($fileHeaderInfo = $this->dat['header']['files'][$cfg['file_ID']]) {
1414 // Initialize; Get directory prefix for file and find possible RTE filename
1415 $dirPrefix = PathUtility::dirname($relFileName) . '/';
1416 if (GeneralUtility::isFirstPartOfStr($dirPrefix, $this->fileadminFolderName . '/')) {
1417 // File in fileadmin/ folder:
1418 // Create file (and possible resources)
1419 $newFileName = $this->processSoftReferences_saveFile_createRelFile($dirPrefix, PathUtility::basename($relFileName), $cfg['file_ID'], $table, $uid);
1420 if (strlen($newFileName)) {
1421 $relFileName = $newFileName;
1422 } else {
1423 $this->error('ERROR: No new file created for "' . $relFileName . '"');
1424 }
1425 } else {
1426 $this->error('ERROR: Sorry, cannot operate on non-RTE files which are outside the fileadmin folder.');
1427 }
1428 } else {
1429 $this->error('ERROR: Could not find file ID in header.');
1430 }
1431 // Return (new) filename relative to public web path
1432 return $relFileName;
1433 }
1434
1435 /**
1436 * Create file in directory and return the new (unique) filename
1437 *
1438 * @param string $origDirPrefix Directory prefix, relative, with trailing slash
1439 * @param string $fileName Filename (without path)
1440 * @param string $fileID File ID from import memory
1441 * @param string $table Table for which the processing occurs
1442 * @param string $uid UID of record from table
1443 * @return string|null New relative filename, if any
1444 */
1445 public function processSoftReferences_saveFile_createRelFile($origDirPrefix, $fileName, $fileID, $table, $uid)
1446 {
1447 // If the fileID map contains an entry for this fileID then just return the relative filename of that entry;
1448 // we don't want to write another unique filename for this one!
1449 if (isset($this->fileIDMap[$fileID])) {
1450 return PathUtility::stripPathSitePrefix($this->fileIDMap[$fileID]);
1451 }
1452 // Verify FileMount access to dir-prefix. Returns the best alternative relative path if any
1453 $dirPrefix = $this->verifyFolderAccess($origDirPrefix);
1454 if ($dirPrefix && (!$this->update || $origDirPrefix === $dirPrefix) && $this->checkOrCreateDir($dirPrefix)) {
1455 $fileHeaderInfo = $this->dat['header']['files'][$fileID];
1456 $updMode = $this->update && $this->import_mapId[$table][$uid] === $uid && $this->import_mode[$table . ':' . $uid] !== 'as_new';
1457 // Create new name for file:
1458 // Must have same ID in map array (just for security, is not really needed) and NOT be set "as_new".
1459
1460 // Write main file:
1461 if ($updMode) {
1462 $newName = Environment::getPublicPath() . '/' . $dirPrefix . $fileName;
1463 } else {
1464 // Create unique filename:
1465 $fileProcObj = $this->getFileProcObj();
1466 $newName = $fileProcObj->getUniqueName($fileName, Environment::getPublicPath() . '/' . $dirPrefix);
1467 }
1468 if ($this->writeFileVerify($newName, $fileID)) {
1469 // If the resource was an HTML/CSS file with resources attached, we will write those as well!
1470 if (is_array($fileHeaderInfo['EXT_RES_ID'])) {
1471 $tokenizedContent = $this->dat['files'][$fileID]['tokenizedContent'];
1472 $tokenSubstituted = false;
1473 $fileProcObj = $this->getFileProcObj();
1474 if ($updMode) {
1475 foreach ($fileHeaderInfo['EXT_RES_ID'] as $res_fileID) {
1476 if ($this->dat['files'][$res_fileID]['filename']) {
1477 // Resolve original filename:
1478 $relResourceFileName = $this->dat['files'][$res_fileID]['parentRelFileName'];
1479 $absResourceFileName = GeneralUtility::resolveBackPath(Environment::getPublicPath() . '/' . $origDirPrefix . $relResourceFileName);
1480 $absResourceFileName = GeneralUtility::getFileAbsFileName($absResourceFileName);
1481 if ($absResourceFileName && GeneralUtility::isFirstPartOfStr($absResourceFileName, Environment::getPublicPath() . '/' . $this->fileadminFolderName . '/')) {
1482 $destDir = PathUtility::stripPathSitePrefix(PathUtility::dirname($absResourceFileName) . '/');
1483 if ($this->verifyFolderAccess($destDir, true) && $this->checkOrCreateDir($destDir)) {
1484 $this->writeFileVerify($absResourceFileName, $res_fileID);
1485 } else {
1486 $this->error('ERROR: Could not create file in directory "' . $destDir . '"');
1487 }
1488 } else {
1489 $this->error('ERROR: Could not resolve path for "' . $relResourceFileName . '"');
1490 }
1491 $tokenizedContent = str_replace('{EXT_RES_ID:' . $res_fileID . '}', $relResourceFileName, $tokenizedContent);
1492 $tokenSubstituted = true;
1493 }
1494 }
1495 } else {
1496 // Create the ressource's directory name (filename without extension, suffixed "_FILES")
1497 $resourceDir = PathUtility::dirname($newName) . '/' . preg_replace('/\\.[^.]*$/', '', PathUtility::basename($newName)) . '_FILES';
1498 if (GeneralUtility::mkdir($resourceDir)) {
1499 foreach ($fileHeaderInfo['EXT_RES_ID'] as $res_fileID) {
1500 if ($this->dat['files'][$res_fileID]['filename']) {
1501 $absResourceFileName = $fileProcObj->getUniqueName($this->dat['files'][$res_fileID]['filename'], $resourceDir);
1502 $relResourceFileName = substr($absResourceFileName, strlen(PathUtility::dirname($resourceDir)) + 1);
1503 $this->writeFileVerify($absResourceFileName, $res_fileID);
1504 $tokenizedContent = str_replace('{EXT_RES_ID:' . $res_fileID . '}', $relResourceFileName, $tokenizedContent);
1505 $tokenSubstituted = true;
1506 }
1507 }
1508 }
1509 }
1510 // If substitutions has been made, write the content to the file again:
1511 if ($tokenSubstituted) {
1512 GeneralUtility::writeFile($newName, $tokenizedContent);
1513 }
1514 }
1515 return PathUtility::stripPathSitePrefix($newName);
1516 }
1517 }
1518 return null;
1519 }
1520
1521 /**
1522 * Writes a file from the import memory having $fileID to file name $fileName which must be an absolute path inside public web path
1523 *
1524 * @param string $fileName Absolute filename inside public web path to write to
1525 * @param string $fileID File ID from import memory
1526 * @param bool $bypassMountCheck Bypasses the checking against filemounts - only for RTE files!
1527 * @return bool Returns TRUE if it went well. Notice that the content of the file is read again, and md5 from import memory is validated.
1528 */
1529 public function writeFileVerify($fileName, $fileID, $bypassMountCheck = false)
1530 {
1531 $fileProcObj = $this->getFileProcObj();
1532 if (!$fileProcObj->actionPerms['addFile']) {
1533 $this->error('ERROR: You did not have sufficient permissions to write the file "' . $fileName . '"');
1534 return false;
1535 }
1536 // Just for security, check again. Should actually not be necessary.
1537 if (!$bypassMountCheck) {
1538 try {
1539 ResourceFactory::getInstance()->getFolderObjectFromCombinedIdentifier(PathUtility::dirname($fileName));
1540 } catch (InsufficientFolderAccessPermissionsException $e) {
1541 $this->error('ERROR: Filename "' . $fileName . '" was not allowed in destination path!');
1542 return false;
1543 }
1544 }
1545 $fI = GeneralUtility::split_fileref($fileName);
1546 if (!$fileProcObj->checkIfAllowed($fI['fileext'], $fI['path'], $fI['file'])) {
1547 $this->error('ERROR: Filename "' . $fileName . '" failed against extension check or deny-pattern!');
1548 return false;
1549 }
1550 if (!GeneralUtility::getFileAbsFileName($fileName)) {
1551 $this->error('ERROR: Filename "' . $fileName . '" was not a valid relative file path!');
1552 return false;
1553 }
1554 if (!$this->dat['files'][$fileID]) {
1555 $this->error('ERROR: File ID "' . $fileID . '" could not be found');
1556 return false;
1557 }
1558 GeneralUtility::writeFile($fileName, $this->dat['files'][$fileID]['content']);
1559 $this->fileIDMap[$fileID] = $fileName;
1560 if (md5(file_get_contents($fileName)) == $this->dat['files'][$fileID]['content_md5']) {
1561 return true;
1562 }
1563 $this->error('ERROR: File content "' . $fileName . '" was corrupted');
1564 return false;
1565 }
1566
1567 /**
1568 * Returns TRUE if directory exists and if it doesn't it will create directory and return TRUE if that succeeded.
1569 *
1570 * @param string $dirPrefix Directory to create. Having a trailing slash. Must be in fileadmin/. Relative to public web path
1571 * @return bool TRUE, if directory exists (was created)
1572 */
1573 public function checkOrCreateDir($dirPrefix)
1574 {
1575 // Split dir path and remove first directory (which should be "fileadmin")
1576 $filePathParts = explode('/', $dirPrefix);
1577 $firstDir = array_shift($filePathParts);
1578 if ($firstDir === $this->fileadminFolderName && GeneralUtility::getFileAbsFileName($dirPrefix)) {
1579 $pathAcc = '';
1580 foreach ($filePathParts as $dirname) {
1581 $pathAcc .= '/' . $dirname;
1582 if (strlen($dirname)) {
1583 if (!@is_dir(Environment::getPublicPath() . '/' . $this->fileadminFolderName . $pathAcc)) {
1584 if (!GeneralUtility::mkdir(Environment::getPublicPath() . '/' . $this->fileadminFolderName . $pathAcc)) {
1585 $this->error('ERROR: Directory could not be created....B');
1586 return false;
1587 }
1588 }
1589 } elseif ($dirPrefix === $this->fileadminFolderName . $pathAcc) {
1590 return true;
1591 } else {
1592 $this->error('ERROR: Directory could not be created....A');
1593 }
1594 }
1595 }
1596 return false;
1597 }
1598
1599 /**************************
1600 * File Input
1601 *************************/
1602
1603 /**
1604 * Loads the header section/all of the $filename into memory
1605 *
1606 * @param string $filename Filename, absolute
1607 * @param bool $all If set, all information is loaded (header, records and files). Otherwise the default is to read only the header information
1608 * @return bool TRUE if the operation went well
1609 */
1610 public function loadFile($filename, $all = false)
1611 {
1612 if (!@is_file($filename)) {
1613 $this->error('Filename not found: ' . $filename);
1614 return false;
1615 }
1616 $fI = pathinfo($filename);
1617 if (@is_dir($filename . '.files')) {
1618 if (GeneralUtility::isAllowedAbsPath($filename . '.files')) {
1619 // copy the folder lowlevel to typo3temp, because the files would be deleted after import
1620 $temporaryFolderName = $this->getTemporaryFolderName();
1621 GeneralUtility::copyDirectory($filename . '.files', $temporaryFolderName);
1622 $this->filesPathForImport = $temporaryFolderName;
1623 } else {
1624 $this->error('External import files for the given import source is currently not supported.');
1625 }
1626 }
1627 if (strtolower($fI['extension']) === 'xml') {
1628 // XML:
1629 $xmlContent = file_get_contents($filename);
1630 if (strlen($xmlContent)) {
1631 $this->dat = GeneralUtility::xml2array($xmlContent, '', true);
1632 if (is_array($this->dat)) {
1633 if ($this->dat['_DOCUMENT_TAG'] === 'T3RecordDocument' && is_array($this->dat['header']) && is_array($this->dat['records'])) {
1634 $this->loadInit();
1635 return true;
1636 }
1637 $this->error('XML file did not contain proper XML for TYPO3 Import');
1638 } else {
1639 $this->error('XML could not be parsed: ' . $this->dat);
1640 }
1641 } else {
1642 $this->error('Error opening file: ' . $filename);
1643 }
1644 } else {
1645 // T3D
1646 if ($fd = fopen($filename, 'rb')) {
1647 $this->dat['header'] = $this->getNextFilePart($fd, 1, 'header');
1648 if ($all) {
1649 $this->dat['records'] = $this->getNextFilePart($fd, 1, 'records');
1650 $this->dat['files'] = $this->getNextFilePart($fd, 1, 'files');
1651 $this->dat['files_fal'] = $this->getNextFilePart($fd, 1, 'files_fal');
1652 }
1653 $this->loadInit();
1654 return true;
1655 }
1656 $this->error('Error opening file: ' . $filename);
1657
1658 fclose($fd);
1659 }
1660 return false;
1661 }
1662
1663 /**
1664 * Returns the next content part form the fileresource (t3d), $fd
1665 *
1666 * @param resource $fd File pointer
1667 * @param bool $unserialize If set, the returned content is unserialized into an array, otherwise you get the raw string
1668 * @param string $name For error messages this indicates the section of the problem.
1669 * @return string|null Data string or NULL in case of an error
1670 * @internal
1671 * @see loadFile()
1672 */
1673 public function getNextFilePart($fd, $unserialize = false, $name = '')
1674 {
1675 $initStrLen = 32 + 1 + 1 + 1 + 10 + 1;
1676 // Getting header data
1677 $initStr = fread($fd, $initStrLen);
1678 if (empty($initStr)) {
1679 $this->error('File does not contain data for "' . $name . '"');
1680 return null;
1681 }
1682 $initStrDat = explode(':', $initStr);
1683 if (strpos($initStrDat[0], 'Warning') !== false) {
1684 $this->error('File read error: Warning message in file. (' . $initStr . fgets($fd) . ')');
1685 return null;
1686 }
1687 if ((string)$initStrDat[3] !== '') {
1688 $this->error('File read error: InitString had a wrong length. (' . $name . ')');
1689 return null;
1690 }
1691 $datString = fread($fd, (int)$initStrDat[2]);
1692 fread($fd, 1);
1693 if (md5($datString) === $initStrDat[0]) {
1694 if ($initStrDat[1]) {
1695 if ($this->compress) {
1696 $datString = gzuncompress($datString);
1697 } else {
1698 $this->error('Content read error: This file requires decompression, but this server does not offer gzcompress()/gzuncompress() functions.');
1699 return null;
1700 }
1701 }
1702 return $unserialize ? unserialize($datString, ['allowed_classes' => false]) : $datString;
1703 }
1704 $this->error('MD5 check failed (' . $name . ')');
1705
1706 return null;
1707 }
1708
1709 /**
1710 * Loads T3D file content into the $this->dat array
1711 * (This function can be used to test the output strings from ->compileMemoryToFileContent())
1712 *
1713 * @param string $filecontent File content
1714 */
1715 public function loadContent($filecontent)
1716 {
1717 $pointer = 0;
1718 $this->dat['header'] = $this->getNextContentPart($filecontent, $pointer, 1, 'header');
1719 $this->dat['records'] = $this->getNextContentPart($filecontent, $pointer, 1, 'records');
1720 $this->dat['files'] = $this->getNextContentPart($filecontent, $pointer, 1, 'files');
1721 $this->loadInit();
1722 }
1723
1724 /**
1725 * Returns the next content part from the $filecontent
1726 *
1727 * @param string $filecontent File content string
1728 * @param int $pointer File pointer (where to read from)
1729 * @param bool $unserialize If set, the returned content is unserialized into an array, otherwise you get the raw string
1730 * @param string $name For error messages this indicates the section of the problem.
1731 * @return string|null Data string
1732 */
1733 public function getNextContentPart($filecontent, &$pointer, $unserialize = false, $name = '')
1734 {
1735 $initStrLen = 32 + 1 + 1 + 1 + 10 + 1;
1736 // getting header data
1737 $initStr = substr($filecontent, $pointer, $initStrLen);
1738 $pointer += $initStrLen;
1739 $initStrDat = explode(':', $initStr);
1740 if ((string)$initStrDat[3] !== '') {
1741 $this->error('Content read error: InitString had a wrong length. (' . $name . ')');
1742 return null;
1743 }
1744 $datString = substr($filecontent, $pointer, (int)$initStrDat[2]);
1745 $pointer += (int)$initStrDat[2] + 1;
1746 if (md5($datString) === $initStrDat[0]) {
1747 if ($initStrDat[1]) {
1748 if ($this->compress) {
1749 $datString = gzuncompress($datString);
1750 return $unserialize ? unserialize($datString, ['allowed_classes' => false]) : $datString;
1751 }
1752 $this->error('Content read error: This file requires decompression, but this server does not offer gzcompress()/gzuncompress() functions.');
1753 }
1754 } else {
1755 $this->error('MD5 check failed (' . $name . ')');
1756 }
1757 return null;
1758 }
1759
1760 /**
1761 * Setting up the object based on the recently loaded ->dat array
1762 */
1763 public function loadInit()
1764 {
1765 $this->relStaticTables = (array)$this->dat['header']['relStaticTables'];
1766 $this->excludeMap = (array)$this->dat['header']['excludeMap'];
1767 $this->softrefCfg = (array)$this->dat['header']['softrefCfg'];
1768 }
1769 }