[BUGFIX] Ensure t3d compatibility for supported TYPO3 version
[Packages/TYPO3.CMS.git] / typo3 / sysext / impexp / Classes / Import.php
1 <?php
2 namespace TYPO3\CMS\Impexp;
3
4 /*
5 * This file is part of the TYPO3 CMS project.
6 *
7 * It is free software; you can redistribute it and/or modify it under
8 * the terms of the GNU General Public License, either version 2
9 * of the License, or any later version.
10 *
11 * For the full copyright and license information, please read the
12 * LICENSE.txt file that was distributed with this source code.
13 *
14 * The TYPO3 project - inspiring people to share!
15 */
16
17 use TYPO3\CMS\Backend\Utility\BackendUtility;
18 use TYPO3\CMS\Core\Charset\CharsetConverter;
19 use TYPO3\CMS\Core\DataHandling\DataHandler;
20 use TYPO3\CMS\Core\Exception;
21 use TYPO3\CMS\Core\Resource\DuplicationBehavior;
22 use TYPO3\CMS\Core\Resource\File;
23 use TYPO3\CMS\Core\Resource\FileInterface;
24 use TYPO3\CMS\Core\Resource\ResourceFactory;
25 use TYPO3\CMS\Core\Resource\ResourceStorage;
26 use TYPO3\CMS\Core\Resource\StorageRepository;
27 use TYPO3\CMS\Core\Utility\ExtensionManagementUtility;
28 use TYPO3\CMS\Core\Utility\GeneralUtility;
29 use TYPO3\CMS\Core\Utility\MathUtility;
30 use TYPO3\CMS\Core\Utility\PathUtility;
31 use TYPO3\CMS\Core\Utility\StringUtility;
32 use TYPO3\CMS\Core\Utility\VersionNumberUtility;
33
34 /**
35 * T3D file Import library (TYPO3 Record Document)
36 */
37 class Import extends ImportExport
38 {
39 /**
40 * Used to register the forged UID values for imported records that we want
41 * to create with the same UIDs as in the import file. Admin-only feature.
42 *
43 * @var array
44 */
45 public $suggestedInsertUids = array();
46
47 /**
48 * Disable logging when importing
49 *
50 * @var bool
51 */
52 public $enableLogging = false;
53
54 /**
55 * Keys are [tablename]:[new NEWxxx ids (or when updating it is uids)]
56 * while values are arrays with table/uid of the original record it is based on.
57 * With the array keys the new ids can be looked up inside tcemain
58 *
59 * @var array
60 */
61 public $import_newId = array();
62
63 /**
64 * Page id map for page tree (import)
65 *
66 * @var array
67 */
68 public $import_newId_pids = array();
69
70 /**
71 * Internal data accumulation for writing records during import
72 *
73 * @var array
74 */
75 public $import_data = array();
76
77 /**
78 * Array of current registered storage objects
79 *
80 * @var ResourceStorage[]
81 */
82 protected $storageObjects = array();
83
84 /**
85 * Is set, if the import file has a TYPO3 version below 6.0
86 *
87 * @var bool
88 */
89 protected $legacyImport = false;
90
91 /**
92 * @var \TYPO3\CMS\Core\Resource\Folder
93 */
94 protected $legacyImportFolder = null;
95
96 /**
97 * Related to the default storage root
98 *
99 * @var string
100 */
101 protected $legacyImportTargetPath = '_imported/';
102
103 /**
104 * Table fields to migrate
105 *
106 * @var array
107 */
108 protected $legacyImportMigrationTables = array(
109 'tt_content' => array(
110 'image' => array(
111 'titleTexts' => 'titleText',
112 'description' => 'imagecaption',
113 'links' => 'image_link',
114 'alternativeTexts' => 'altText'
115 ),
116 'media' => array(
117 'description' => 'imagecaption',
118 )
119 ),
120 'pages' => array(
121 'media' => array()
122 ),
123 'pages_language_overlay' => array(
124 'media' => array()
125 )
126 );
127
128 /**
129 * Records to be migrated after all
130 * Multidimensional array [table][uid][field] = array([related sys_file_reference uids])
131 *
132 * @var array
133 */
134 protected $legacyImportMigrationRecords = array();
135
136 /**
137 * @var NULL|string
138 */
139 protected $filesPathForImport = null;
140
141 /**
142 * @var array
143 */
144 protected $unlinkFiles = array();
145
146 /**
147 * @var array
148 */
149 protected $alternativeFileName = array();
150
151 /**
152 * @var array
153 */
154 protected $alternativeFilePath = array();
155
156 /**
157 * @var array
158 */
159 protected $filePathMap = array();
160
161 /**************************
162 * Initialize
163 *************************/
164
165 /**
166 * Init the object
167 *
168 * @return void
169 */
170 public function init()
171 {
172 parent::init();
173 $this->mode = 'import';
174 }
175
176 /***********************
177 * Import
178 ***********************/
179
180 /**
181 * Initialize all settings for the import
182 *
183 * @return void
184 */
185 protected function initializeImport()
186 {
187 // Set this flag to indicate that an import is being/has been done.
188 $this->doesImport = 1;
189 // Initialize:
190 // These vars MUST last for the whole section not being cleared. They are used by the method setRelations() which are called at the end of the import session.
191 $this->import_mapId = array();
192 $this->import_newId = array();
193 $this->import_newId_pids = array();
194 // Temporary files stack initialized:
195 $this->unlinkFiles = array();
196 $this->alternativeFileName = array();
197 $this->alternativeFilePath = array();
198
199 $this->initializeStorageObjects();
200 }
201
202 /**
203 * Initialize the all present storage objects
204 *
205 * @return void
206 */
207 protected function initializeStorageObjects()
208 {
209 /** @var $storageRepository StorageRepository */
210 $storageRepository = GeneralUtility::makeInstance(StorageRepository::class);
211 $this->storageObjects = $storageRepository->findAll();
212 }
213
214 /**
215 * Imports the internal data array to $pid.
216 *
217 * @param int $pid Page ID in which to import the content
218 * @return void
219 */
220 public function importData($pid)
221 {
222 $this->initializeImport();
223
224 // Write sys_file_storages first
225 $this->writeSysFileStorageRecords();
226 // Write sys_file records and write the binary file data
227 $this->writeSysFileRecords();
228 // Write records, first pages, then the rest
229 // Fields with "hard" relations to database, files and flexform fields are kept empty during this run
230 $this->writeRecords_pages($pid);
231 $this->writeRecords_records($pid);
232 // Finally all the file and DB record references must be fixed. This is done after all records have supposedly been written to database:
233 // $this->import_mapId will indicate two things: 1) that a record WAS written to db and 2) that it has got a new id-number.
234 $this->setRelations();
235 // And when all DB relations are in place, we can fix file and DB relations in flexform fields (since data structures often depends on relations to a DS record):
236 $this->setFlexFormRelations();
237 // Unlink temporary files:
238 $this->unlinkTempFiles();
239 // Finally, traverse all records and process softreferences with substitution attributes.
240 $this->processSoftReferences();
241 // After all migrate records using sys_file_reference now
242 if ($this->legacyImport) {
243 $this->migrateLegacyImportRecords();
244 }
245 }
246
247 /**
248 * Imports the sys_file_storage records from internal data array.
249 *
250 * @return void
251 */
252 protected function writeSysFileStorageRecords()
253 {
254 if (!isset($this->dat['header']['records']['sys_file_storage'])) {
255 return;
256 }
257 $sysFileStorageUidsToBeResetToDefaultStorage = array();
258 foreach ($this->dat['header']['records']['sys_file_storage'] as $sysFileStorageUid => $_) {
259 $storageRecord = $this->dat['records']['sys_file_storage:' . $sysFileStorageUid]['data'];
260 // continue with Local, writable and online storage only
261 if ($storageRecord['driver'] === 'Local' && $storageRecord['is_writable'] && $storageRecord['is_online']) {
262 foreach ($this->storageObjects as $localStorage) {
263 if ($this->isEquivalentObjectStorage($localStorage, $storageRecord)) {
264 $this->import_mapId['sys_file_storage'][$sysFileStorageUid] = $localStorage->getUid();
265 break;
266 }
267 }
268
269 if (!isset($this->import_mapId['sys_file_storage'][$sysFileStorageUid])) {
270 // Local, writable and online storage. Is allowed to be used to later write files in.
271 // Does currently not exist so add the record.
272 $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);
273 }
274 } else {
275 // Storage with non Local drivers could be imported but must not be used to saves files in, because you
276 // could not be sure, that this is supported. The default storage will be used in this case.
277 // It could happen that non writable and non online storage will be created as dupes because you could not
278 // check the detailed configuration options at this point
279 $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);
280 $sysFileStorageUidsToBeResetToDefaultStorage[] = $sysFileStorageUid;
281 }
282 }
283
284 // Importing the added ones
285 $tce = $this->getNewTCE();
286 // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.
287 $tce->reverseOrder = 1;
288 $tce->isImporting = true;
289 $tce->start($this->import_data, array());
290 $tce->process_datamap();
291 $this->addToMapId($tce->substNEWwithIDs);
292
293 $defaultStorageUid = null;
294 // get default storage
295 $defaultStorage = ResourceFactory::getInstance()->getDefaultStorage();
296 if ($defaultStorage !== null) {
297 $defaultStorageUid = $defaultStorage->getUid();
298 }
299 foreach ($sysFileStorageUidsToBeResetToDefaultStorage as $sysFileStorageUidToBeResetToDefaultStorage) {
300 $this->import_mapId['sys_file_storage'][$sysFileStorageUidToBeResetToDefaultStorage] = $defaultStorageUid;
301 }
302
303 // unset the sys_file_storage records to prevent an import in writeRecords_records
304 unset($this->dat['header']['records']['sys_file_storage']);
305 }
306
307 /**
308 * Determines whether the passed storage object and record (sys_file_storage) can be
309 * seen as equivalent during import.
310 *
311 * @param ResourceStorage $storageObject The storage object which should get compared
312 * @param array $storageRecord The storage record which should get compared
313 * @return bool Returns TRUE when both object storages can be seen as equivalent
314 */
315 protected function isEquivalentObjectStorage(ResourceStorage $storageObject, array $storageRecord)
316 {
317 // compare the properties: driver, writable and online
318 if ($storageObject->getDriverType() === $storageRecord['driver']
319 && (bool)$storageObject->isWritable() === (bool)$storageRecord['is_writable']
320 && (bool)$storageObject->isOnline() === (bool)$storageRecord['is_online']
321 ) {
322 $storageRecordConfiguration = ResourceFactory::getInstance()->convertFlexFormDataToConfigurationArray($storageRecord['configuration']);
323 $storageObjectConfiguration = $storageObject->getConfiguration();
324 // compare the properties: pathType and basePath
325 if ($storageRecordConfiguration['pathType'] === $storageObjectConfiguration['pathType']
326 && $storageRecordConfiguration['basePath'] === $storageObjectConfiguration['basePath']
327 ) {
328 return true;
329 }
330 }
331 return false;
332 }
333
334 /**
335 * Checks any prerequisites necessary to get fullfilled before import
336 *
337 * @return array Messages explaining issues which need to get resolved before import
338 */
339 public function checkImportPrerequisites()
340 {
341 $messages = array();
342
343 // Check #1: Extension dependencies
344 $extKeysToInstall = array();
345 foreach ($this->dat['header']['extensionDependencies'] as $extKey) {
346 if (!empty($extKey) && !ExtensionManagementUtility::isLoaded($extKey)) {
347 $extKeysToInstall[] = $extKey;
348 }
349 }
350 if (!empty($extKeysToInstall)) {
351 $messages['missingExtensions'] = 'Before you can install this T3D file you need to install the extensions "'
352 . implode('", "', $extKeysToInstall) . '".';
353 }
354
355 // Check #2: If the path for every local storage object exists.
356 // Else files can't get moved into a newly imported storage.
357 if (!empty($this->dat['header']['records']['sys_file_storage'])) {
358 foreach ($this->dat['header']['records']['sys_file_storage'] as $sysFileStorageUid => $_) {
359 $storageRecord = $this->dat['records']['sys_file_storage:' . $sysFileStorageUid]['data'];
360 // continue with Local, writable and online storage only
361 if ($storageRecord['driver'] === 'Local'
362 && $storageRecord['is_writable']
363 && $storageRecord['is_online']
364 ) {
365 foreach ($this->storageObjects as $localStorage) {
366 if ($this->isEquivalentObjectStorage($localStorage, $storageRecord)) {
367 // There is already an existing storage
368 break;
369 }
370
371 // The storage from the import does not have an equivalent storage
372 // in the current instance (same driver, same path, etc.). Before
373 // the storage record can get inserted later on take care the path
374 // it points to really exists and is accessible.
375 $storageRecordUid = $storageRecord['uid'];
376 // Unset the storage record UID when trying to create the storage object
377 // as the record does not already exist in DB. The constructor of the
378 // storage object will check whether the target folder exists and set the
379 // isOnline flag depending on the outcome.
380 $storageRecord['uid'] = 0;
381 $resourceStorage = ResourceFactory::getInstance()->createStorageObject($storageRecord);
382 if (!$resourceStorage->isOnline()) {
383 $configuration = $resourceStorage->getConfiguration();
384 $messages['resourceStorageFolderMissing_' . $storageRecordUid] =
385 'The resource storage "'
386 . $resourceStorage->getName()
387 . $configuration['basePath']
388 . '" does not exist. Please create the directory prior to starting the import!';
389 }
390 }
391 }
392 }
393 }
394
395 return $messages;
396 }
397
398 /**
399 * Imports the sys_file records and the binary files data from internal data array.
400 *
401 * @return void
402 */
403 protected function writeSysFileRecords()
404 {
405 if (!isset($this->dat['header']['records']['sys_file'])) {
406 return;
407 }
408 $this->addGeneralErrorsByTable('sys_file');
409
410 // fetch fresh storage records from database
411 $storageRecords = $this->fetchStorageRecords();
412
413 $defaultStorage = ResourceFactory::getInstance()->getDefaultStorage();
414
415 $sanitizedFolderMappings = array();
416
417 foreach ($this->dat['header']['records']['sys_file'] as $sysFileUid => $_) {
418 $fileRecord = $this->dat['records']['sys_file:' . $sysFileUid]['data'];
419
420 $temporaryFile = null;
421 // check if there is the right file already in the local folder
422 if ($this->filesPathForImport !== null) {
423 if (is_file($this->filesPathForImport . '/' . $fileRecord['sha1']) && sha1_file($this->filesPathForImport . '/' . $fileRecord['sha1']) === $fileRecord['sha1']) {
424 $temporaryFile = $this->filesPathForImport . '/' . $fileRecord['sha1'];
425 }
426 }
427
428 // save file to disk
429 if ($temporaryFile === null) {
430 $fileId = md5($fileRecord['storage'] . ':' . $fileRecord['identifier_hash']);
431 $temporaryFile = $this->writeTemporaryFileFromData($fileId);
432 if ($temporaryFile === null) {
433 // error on writing the file. Error message was already added
434 continue;
435 }
436 }
437
438 $originalStorageUid = $fileRecord['storage'];
439 $useStorageFromStorageRecords = false;
440
441 // replace storage id, if an alternative one was registered
442 if (isset($this->import_mapId['sys_file_storage'][$fileRecord['storage']])) {
443 $fileRecord['storage'] = $this->import_mapId['sys_file_storage'][$fileRecord['storage']];
444 $useStorageFromStorageRecords = true;
445 }
446
447 if (empty($fileRecord['storage']) && !$this->isFallbackStorage($fileRecord['storage'])) {
448 // no storage for the file is defined, mostly because of a missing default storage.
449 $this->error('Error: No storage for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $originalStorageUid . '"');
450 continue;
451 }
452
453 // using a storage from the local storage is only allowed, if the uid is present in the
454 // mapping. Only in this case we could be sure, that it's a local, online and writable storage.
455 if ($useStorageFromStorageRecords && isset($storageRecords[$fileRecord['storage']])) {
456 /** @var $storage \TYPO3\CMS\Core\Resource\ResourceStorage */
457 $storage = ResourceFactory::getInstance()->getStorageObject($fileRecord['storage'], $storageRecords[$fileRecord['storage']]);
458 } elseif ($this->isFallbackStorage($fileRecord['storage'])) {
459 $storage = ResourceFactory::getInstance()->getStorageObject(0);
460 } elseif ($defaultStorage !== null) {
461 $storage = $defaultStorage;
462 } else {
463 $this->error('Error: No storage available for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
464 continue;
465 }
466
467 $newFile = null;
468
469 // check, if there is an identical file
470 try {
471 if ($storage->hasFile($fileRecord['identifier'])) {
472 $file = $storage->getFile($fileRecord['identifier']);
473 if ($file->getSha1() === $fileRecord['sha1']) {
474 $newFile = $file;
475 }
476 }
477 } catch (Exception $e) {
478 }
479
480 if ($newFile === null) {
481 $folderName = PathUtility::dirname(ltrim($fileRecord['identifier'], '/'));
482 if (in_array($folderName, $sanitizedFolderMappings)) {
483 $folderName = $sanitizedFolderMappings[$folderName];
484 }
485 if (!$storage->hasFolder($folderName)) {
486 try {
487 $importFolder = $storage->createFolder($folderName);
488 if ($importFolder->getIdentifier() !== $folderName && !in_array($folderName, $sanitizedFolderMappings)) {
489 $sanitizedFolderMappings[$folderName] = $importFolder->getIdentifier();
490 }
491 } catch (Exception $e) {
492 $this->error('Error: Folder "' . $folderName . '" could not be created for file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
493 continue;
494 }
495 } else {
496 $importFolder = $storage->getFolder($folderName);
497 }
498
499 try {
500 /** @var $newFile File */
501 $newFile = $storage->addFile($temporaryFile, $importFolder, $fileRecord['name']);
502 } catch (Exception $e) {
503 $this->error('Error: File could not be added to the storage: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
504 continue;
505 }
506
507 if ($newFile->getSha1() !== $fileRecord['sha1']) {
508 $this->error('Error: The hash of the written file is not identical to the import data! File could be corrupted! File: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
509 }
510 }
511
512 // save the new uid in the import id map
513 $this->import_mapId['sys_file'][$fileRecord['uid']] = $newFile->getUid();
514 $this->fixUidLocalInSysFileReferenceRecords($fileRecord['uid'], $newFile->getUid());
515 }
516
517 // unset the sys_file records to prevent an import in writeRecords_records
518 unset($this->dat['header']['records']['sys_file']);
519 // remove all sys_file_reference records that point to file records which are unknown
520 // in the system to prevent exceptions
521 $this->removeSysFileReferenceRecordsFromImportDataWithRelationToMissingFile();
522 }
523
524 /**
525 * Removes all sys_file_reference records from the import data array that are pointing to sys_file records which
526 * are missing not in the import data to prevent exceptions on checking the related file started by the Datahandler.
527 *
528 * @return void
529 */
530 protected function removeSysFileReferenceRecordsFromImportDataWithRelationToMissingFile()
531 {
532 if (!isset($this->dat['header']['records']['sys_file_reference'])) {
533 return;
534 }
535
536 foreach ($this->dat['header']['records']['sys_file_reference'] as $sysFileReferenceUid => $_) {
537 $fileReferenceRecord = $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'];
538 if (!in_array($fileReferenceRecord['uid_local'], $this->import_mapId['sys_file'])) {
539 unset($this->dat['header']['records']['sys_file_reference'][$sysFileReferenceUid]);
540 unset($this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]);
541 $this->error('Error: sys_file_reference record ' . (int)$sysFileReferenceUid
542 . ' with relation to sys_file record ' . (int)$fileReferenceRecord['uid_local']
543 . ', which is not part of the import data, was not imported.'
544 );
545 }
546 }
547 }
548
549 /**
550 * Checks if the $storageId is the id of the fallback storage
551 *
552 * @param int|string $storageId
553 * @return bool
554 */
555 protected function isFallbackStorage($storageId)
556 {
557 return $storageId === 0 || $storageId === '0';
558 }
559
560 /**
561 * Normally the importer works like the following:
562 * Step 1: import the records with cleared field values of relation fields (see addSingle())
563 * Step 2: update the records with the right relation ids (see setRelations())
564 *
565 * In step 2 the saving fields of type "relation to sys_file_reference" checks the related sys_file_reference
566 * record (created in step 1) with the FileExtensionFilter for matching file extensions of the related file.
567 * To make this work correct, the uid_local of sys_file_reference records has to be not empty AND has to
568 * relate to the correct (imported) sys_file record uid!!!
569 *
570 * This is fixed here.
571 *
572 * @param int $oldFileUid
573 * @param int $newFileUid
574 * @return void
575 */
576 protected function fixUidLocalInSysFileReferenceRecords($oldFileUid, $newFileUid)
577 {
578 if (!isset($this->dat['header']['records']['sys_file_reference'])) {
579 return;
580 }
581
582 foreach ($this->dat['header']['records']['sys_file_reference'] as $sysFileReferenceUid => $_) {
583 $fileReferenceRecord = $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'];
584 if ($fileReferenceRecord['uid_local'] == $oldFileUid) {
585 $fileReferenceRecord['uid_local'] = $newFileUid;
586 $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'] = $fileReferenceRecord;
587 }
588 }
589 }
590
591 /**
592 * Initializes the folder for legacy imports as subfolder of backend users default upload folder
593 *
594 * @return void
595 */
596 protected function initializeLegacyImportFolder()
597 {
598 /** @var \TYPO3\CMS\Core\Resource\Folder $folder */
599 $folder = $this->getBackendUser()->getDefaultUploadFolder();
600 if ($folder === false) {
601 $this->error('Error: the backend users default upload folder is missing! No files will be imported!');
602 }
603 if (!$folder->hasFolder($this->legacyImportTargetPath)) {
604 try {
605 $this->legacyImportFolder = $folder->createFolder($this->legacyImportTargetPath);
606 } catch (Exception $e) {
607 $this->error('Error: the import folder in the default upload folder could not be created! No files will be imported!');
608 }
609 } else {
610 $this->legacyImportFolder = $folder->getSubfolder($this->legacyImportTargetPath);
611 }
612 }
613
614 /**
615 * Fetched fresh storage records from database because the new imported
616 * ones are not in cached data of the StorageRepository
617 *
618 * @return bool|array
619 */
620 protected function fetchStorageRecords()
621 {
622 $whereClause = BackendUtility::BEenableFields('sys_file_storage');
623 $whereClause .= BackendUtility::deleteClause('sys_file_storage');
624
625 $rows = $this->getDatabaseConnection()->exec_SELECTgetRows(
626 '*',
627 'sys_file_storage',
628 '1=1' . $whereClause,
629 '',
630 '',
631 '',
632 'uid'
633 );
634
635 return $rows;
636 }
637
638 /**
639 * Writes the file from import array to temp dir and returns the filename of it.
640 *
641 * @param string $fileId
642 * @param string $dataKey
643 * @return string Absolute filename of the temporary filename of the file
644 */
645 protected function writeTemporaryFileFromData($fileId, $dataKey = 'files_fal')
646 {
647 $temporaryFilePath = null;
648 if (is_array($this->dat[$dataKey][$fileId])) {
649 $temporaryFilePathInternal = GeneralUtility::tempnam('import_temp_');
650 GeneralUtility::writeFile($temporaryFilePathInternal, $this->dat[$dataKey][$fileId]['content']);
651 clearstatcache();
652 if (@is_file($temporaryFilePathInternal)) {
653 $this->unlinkFiles[] = $temporaryFilePathInternal;
654 if (filesize($temporaryFilePathInternal) == $this->dat[$dataKey][$fileId]['filesize']) {
655 $temporaryFilePath = $temporaryFilePathInternal;
656 } else {
657 $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' had a size (' . filesize($temporaryFilePathInternal) . ') different from the original (' . $this->dat[$dataKey][$fileId]['filesize'] . ')');
658 }
659 } else {
660 $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' was not written as it should have been!');
661 }
662 } else {
663 $this->error('Error: No file found for ID ' . $fileId);
664 }
665 return $temporaryFilePath;
666 }
667
668 /**
669 * Writing pagetree/pages to database:
670 *
671 * @param int $pid PID in which to import. If the operation is an update operation, the root of the page tree inside will be moved to this PID unless it is the same as the root page from the import
672 * @return void
673 * @see writeRecords_records()
674 */
675 public function writeRecords_pages($pid)
676 {
677 // First, write page structure if any:
678 if (is_array($this->dat['header']['records']['pages'])) {
679 $this->addGeneralErrorsByTable('pages');
680 // $pageRecords is a copy of the pages array in the imported file. Records here are unset one by one when the addSingle function is called.
681 $pageRecords = $this->dat['header']['records']['pages'];
682 $this->import_data = array();
683 // First add page tree if any
684 if (is_array($this->dat['header']['pagetree'])) {
685 $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);
686 foreach ($pagesFromTree as $uid) {
687 $thisRec = $this->dat['header']['records']['pages'][$uid];
688 // PID: Set the main $pid, unless a NEW-id is found
689 $setPid = isset($this->import_newId_pids[$thisRec['pid']]) ? $this->import_newId_pids[$thisRec['pid']] : $pid;
690 $this->addSingle('pages', $uid, $setPid);
691 unset($pageRecords[$uid]);
692 }
693 }
694 // Then add all remaining pages not in tree on root level:
695 if (!empty($pageRecords)) {
696 $remainingPageUids = array_keys($pageRecords);
697 foreach ($remainingPageUids as $pUid) {
698 $this->addSingle('pages', $pUid, $pid);
699 }
700 }
701 // Now write to database:
702 $tce = $this->getNewTCE();
703 $tce->isImporting = true;
704 $this->callHook('before_writeRecordsPages', array(
705 'tce' => &$tce,
706 'data' => &$this->import_data
707 ));
708 $tce->suggestedInsertUids = $this->suggestedInsertUids;
709 $tce->start($this->import_data, array());
710 $tce->process_datamap();
711 $this->callHook('after_writeRecordsPages', array(
712 'tce' => &$tce
713 ));
714 // post-processing: Registering new ids (end all tcemain sessions with this)
715 $this->addToMapId($tce->substNEWwithIDs);
716 // In case of an update, order pages from the page tree correctly:
717 if ($this->update && is_array($this->dat['header']['pagetree'])) {
718 $this->writeRecords_pages_order();
719 }
720 }
721 }
722
723 /**
724 * Organize all updated pages in page tree so they are related like in the import file
725 * Only used for updates and when $this->dat['header']['pagetree'] is an array.
726 *
727 * @return void
728 * @access private
729 * @see writeRecords_pages(), writeRecords_records_order()
730 */
731 public function writeRecords_pages_order()
732 {
733 $cmd_data = array();
734 // Get uid-pid relations and traverse them in order to map to possible new IDs
735 $pidsFromTree = $this->flatInversePageTree_pid($this->dat['header']['pagetree']);
736 foreach ($pidsFromTree as $origPid => $newPid) {
737 if ($newPid >= 0 && $this->dontIgnorePid('pages', $origPid)) {
738 // If the page had a new id (because it was created) use that instead!
739 if (substr($this->import_newId_pids[$origPid], 0, 3) === 'NEW') {
740 if ($this->import_mapId['pages'][$origPid]) {
741 $mappedPid = $this->import_mapId['pages'][$origPid];
742 $cmd_data['pages'][$mappedPid]['move'] = $newPid;
743 }
744 } else {
745 $cmd_data['pages'][$origPid]['move'] = $newPid;
746 }
747 }
748 }
749 // Execute the move commands if any:
750 if (!empty($cmd_data)) {
751 $tce = $this->getNewTCE();
752 $this->callHook('before_writeRecordsPagesOrder', array(
753 'tce' => &$tce,
754 'data' => &$cmd_data
755 ));
756 $tce->start(array(), $cmd_data);
757 $tce->process_cmdmap();
758 $this->callHook('after_writeRecordsPagesOrder', array(
759 'tce' => &$tce
760 ));
761 }
762 }
763
764 /**
765 * Recursively flattening the idH array, setting PIDs as values
766 *
767 * @param array $idH Page uid hierarchy
768 * @param array $a Accumulation array of pages (internal, don't set from outside)
769 * @param int $pid PID value (internal)
770 * @return array Array with uid-pid pairs for all pages in the page tree.
771 * @see ImportExport::flatInversePageTree()
772 */
773 public function flatInversePageTree_pid($idH, $a = array(), $pid = -1)
774 {
775 if (is_array($idH)) {
776 $idH = array_reverse($idH);
777 foreach ($idH as $v) {
778 $a[$v['uid']] = $pid;
779 if (is_array($v['subrow'])) {
780 $a = $this->flatInversePageTree_pid($v['subrow'], $a, $v['uid']);
781 }
782 }
783 }
784 return $a;
785 }
786
787 /**
788 * Write all database records except pages (writtein in writeRecords_pages())
789 *
790 * @param int $pid Page id in which to import
791 * @return void
792 * @see writeRecords_pages()
793 */
794 public function writeRecords_records($pid)
795 {
796 // Write the rest of the records
797 $this->import_data = array();
798 if (is_array($this->dat['header']['records'])) {
799 foreach ($this->dat['header']['records'] as $table => $recs) {
800 $this->addGeneralErrorsByTable($table);
801 if ($table != 'pages') {
802 foreach ($recs as $uid => $thisRec) {
803 // PID: Set the main $pid, unless a NEW-id is found
804 $setPid = isset($this->import_mapId['pages'][$thisRec['pid']])
805 ? (int)$this->import_mapId['pages'][$thisRec['pid']]
806 : (int)$pid;
807 if (is_array($GLOBALS['TCA'][$table]) && isset($GLOBALS['TCA'][$table]['ctrl']['rootLevel'])) {
808 $rootLevelSetting = (int)$GLOBALS['TCA'][$table]['ctrl']['rootLevel'];
809 if ($rootLevelSetting === 1) {
810 $setPid = 0;
811 } elseif ($rootLevelSetting === 0 && $setPid === 0) {
812 $this->error('Error: Record type ' . $table . ' is not allowed on pid 0');
813 continue;
814 }
815 }
816 // Add record:
817 $this->addSingle($table, $uid, $setPid);
818 }
819 }
820 }
821 } else {
822 $this->error('Error: No records defined in internal data array.');
823 }
824 // Now write to database:
825 $tce = $this->getNewTCE();
826 $this->callHook('before_writeRecordsRecords', array(
827 'tce' => &$tce,
828 'data' => &$this->import_data
829 ));
830 $tce->suggestedInsertUids = $this->suggestedInsertUids;
831 // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.
832 $tce->reverseOrder = 1;
833 $tce->isImporting = true;
834 $tce->start($this->import_data, array());
835 $tce->process_datamap();
836 $this->callHook('after_writeRecordsRecords', array(
837 'tce' => &$tce
838 ));
839 // post-processing: Removing files and registering new ids (end all tcemain sessions with this)
840 $this->addToMapId($tce->substNEWwithIDs);
841 // In case of an update, order pages from the page tree correctly:
842 if ($this->update) {
843 $this->writeRecords_records_order($pid);
844 }
845 }
846
847 /**
848 * Organize all updated record to their new positions.
849 * Only used for updates
850 *
851 * @param int $mainPid Main PID into which we import.
852 * @return void
853 * @access private
854 * @see writeRecords_records(), writeRecords_pages_order()
855 */
856 public function writeRecords_records_order($mainPid)
857 {
858 $cmd_data = array();
859 if (is_array($this->dat['header']['pagetree'])) {
860 $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);
861 } else {
862 $pagesFromTree = array();
863 }
864 if (is_array($this->dat['header']['pid_lookup'])) {
865 foreach ($this->dat['header']['pid_lookup'] as $pid => $recList) {
866 $newPid = isset($this->import_mapId['pages'][$pid]) ? $this->import_mapId['pages'][$pid] : $mainPid;
867 if (MathUtility::canBeInterpretedAsInteger($newPid)) {
868 foreach ($recList as $tableName => $uidList) {
869 // If $mainPid===$newPid then we are on root level and we can consider to move pages as well!
870 // (they will not be in the page tree!)
871 if (($tableName != 'pages' || !$pagesFromTree[$pid]) && is_array($uidList)) {
872 $uidList = array_reverse(array_keys($uidList));
873 foreach ($uidList as $uid) {
874 if ($this->dontIgnorePid($tableName, $uid)) {
875 $cmd_data[$tableName][$uid]['move'] = $newPid;
876 } else {
877 }
878 }
879 }
880 }
881 }
882 }
883 }
884 // Execute the move commands if any:
885 if (!empty($cmd_data)) {
886 $tce = $this->getNewTCE();
887 $this->callHook('before_writeRecordsRecordsOrder', array(
888 'tce' => &$tce,
889 'data' => &$cmd_data
890 ));
891 $tce->start(array(), $cmd_data);
892 $tce->process_cmdmap();
893 $this->callHook('after_writeRecordsRecordsOrder', array(
894 'tce' => &$tce
895 ));
896 }
897 }
898
899 /**
900 * Adds a single record to the $importData array. Also copies files to tempfolder.
901 * However all File/DB-references and flexform field contents are set to blank for now!
902 * That is done with setRelations() later
903 *
904 * @param string $table Table name (from import memory)
905 * @param int $uid Record UID (from import memory)
906 * @param int $pid Page id
907 * @return void
908 * @see writeRecords()
909 */
910 public function addSingle($table, $uid, $pid)
911 {
912 if ($this->import_mode[$table . ':' . $uid] === 'exclude') {
913 return;
914 }
915 $record = $this->dat['records'][$table . ':' . $uid]['data'];
916 if (is_array($record)) {
917 if ($this->update && $this->doesRecordExist($table, $uid) && $this->import_mode[$table . ':' . $uid] !== 'as_new') {
918 $ID = $uid;
919 } elseif ($table === 'sys_file_metadata' && $record['sys_language_uid'] == '0' && $this->import_mapId['sys_file'][$record['file']]) {
920 // on adding sys_file records the belonging sys_file_metadata record was also created
921 // if there is one the record need to be overwritten instead of creating a new one.
922 $recordInDatabase = $this->getDatabaseConnection()->exec_SELECTgetSingleRow(
923 'uid',
924 'sys_file_metadata',
925 'file = ' . $this->import_mapId['sys_file'][$record['file']] . ' AND sys_language_uid = 0 AND pid = 0'
926 );
927 // if no record could be found, $this->import_mapId['sys_file'][$record['file']] is pointing
928 // to a file, that was already there, thus a new metadata record should be created
929 if (is_array($recordInDatabase)) {
930 $this->import_mapId['sys_file_metadata'][$record['uid']] = $recordInDatabase['uid'];
931 $ID = $recordInDatabase['uid'];
932 } else {
933 $ID = StringUtility::getUniqueId('NEW');
934 }
935 } else {
936 $ID = StringUtility::getUniqueId('NEW');
937 }
938 $this->import_newId[$table . ':' . $ID] = array('table' => $table, 'uid' => $uid);
939 if ($table == 'pages') {
940 $this->import_newId_pids[$uid] = $ID;
941 }
942 // Set main record data:
943 $this->import_data[$table][$ID] = $record;
944 $this->import_data[$table][$ID]['tx_impexp_origuid'] = $this->import_data[$table][$ID]['uid'];
945 // Reset permission data:
946 if ($table === 'pages') {
947 // Have to reset the user/group IDs so pages are owned by importing user. Otherwise strange things may happen for non-admins!
948 unset($this->import_data[$table][$ID]['perms_userid']);
949 unset($this->import_data[$table][$ID]['perms_groupid']);
950 }
951 // PID and UID:
952 unset($this->import_data[$table][$ID]['uid']);
953 // Updates:
954 if (MathUtility::canBeInterpretedAsInteger($ID)) {
955 unset($this->import_data[$table][$ID]['pid']);
956 } else {
957 // Inserts:
958 $this->import_data[$table][$ID]['pid'] = $pid;
959 if (($this->import_mode[$table . ':' . $uid] === 'force_uid' && $this->update || $this->force_all_UIDS) && $this->getBackendUser()->isAdmin()) {
960 $this->import_data[$table][$ID]['uid'] = $uid;
961 $this->suggestedInsertUids[$table . ':' . $uid] = 'DELETE';
962 }
963 }
964 // Setting db/file blank:
965 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
966 switch ((string)$config['type']) {
967 case 'db':
968
969 case 'file':
970 // Fixed later in ->setRelations() [because we need to know ALL newly created IDs before we can map relations!]
971 // In the meantime we set NO values for relations.
972 //
973 // BUT for field uid_local of table sys_file_reference the relation MUST not be cleared here,
974 // because the value is already the uid of the right imported sys_file record.
975 // @see fixUidLocalInSysFileReferenceRecords()
976 // If it's empty or a uid to another record the FileExtensionFilter will throw an exception or
977 // delete the reference record if the file extension of the related record doesn't match.
978 if ($table !== 'sys_file_reference' && $field !== 'uid_local') {
979 $this->import_data[$table][$ID][$field] = '';
980 }
981 break;
982 case 'flex':
983 // Fixed later in setFlexFormRelations()
984 // In the meantime we set NO value for flexforms - this is mainly because file references
985 // inside will not be processed properly; In fact references will point to no file
986 // or existing files (in which case there will be double-references which is a big problem of course!)
987 $this->import_data[$table][$ID][$field] = '';
988 break;
989 }
990 }
991 } elseif ($table . ':' . $uid != 'pages:0') {
992 // On root level we don't want this error message.
993 $this->error('Error: no record was found in data array!');
994 }
995 }
996
997 /**
998 * Registers the substNEWids in memory.
999 *
1000 * @param array $substNEWwithIDs From tcemain to be merged into internal mapping variable in this object
1001 * @return void
1002 * @see writeRecords()
1003 */
1004 public function addToMapId($substNEWwithIDs)
1005 {
1006 foreach ($this->import_data as $table => $recs) {
1007 foreach ($recs as $id => $value) {
1008 $old_uid = $this->import_newId[$table . ':' . $id]['uid'];
1009 if (isset($substNEWwithIDs[$id])) {
1010 $this->import_mapId[$table][$old_uid] = $substNEWwithIDs[$id];
1011 } elseif ($this->update) {
1012 // Map same ID to same ID....
1013 $this->import_mapId[$table][$old_uid] = $id;
1014 } else {
1015 // if $this->import_mapId contains already the right mapping, skip the error msg.
1016 // See special handling of sys_file_metadata in addSingle() => nothing to do
1017 if (!($table === 'sys_file_metadata' && isset($this->import_mapId[$table][$old_uid]) && $this->import_mapId[$table][$old_uid] == $id)) {
1018 $this->error('Possible error: ' . $table . ':' . $old_uid . ' had no new id assigned to it. This indicates that the record was not added to database during import. Please check changelog!');
1019 }
1020 }
1021 }
1022 }
1023 }
1024
1025 /**
1026 * Returns a new $TCE object
1027 *
1028 * @return DataHandler $TCE object
1029 */
1030 public function getNewTCE()
1031 {
1032 $tce = GeneralUtility::makeInstance(DataHandler::class);
1033 $tce->dontProcessTransformations = 1;
1034 $tce->enableLogging = $this->enableLogging;
1035 $tce->alternativeFileName = $this->alternativeFileName;
1036 $tce->alternativeFilePath = $this->alternativeFilePath;
1037 return $tce;
1038 }
1039
1040 /**
1041 * Cleaning up all the temporary files stored in typo3temp/ folder
1042 *
1043 * @return void
1044 */
1045 public function unlinkTempFiles()
1046 {
1047 foreach ($this->unlinkFiles as $fileName) {
1048 if (GeneralUtility::isFirstPartOfStr($fileName, PATH_site . 'typo3temp/')) {
1049 GeneralUtility::unlink_tempfile($fileName);
1050 clearstatcache();
1051 if (is_file($fileName)) {
1052 $this->error('Error: ' . $fileName . ' was NOT unlinked as it should have been!');
1053 }
1054 } else {
1055 $this->error('Error: ' . $fileName . ' was not in temp-path. Not removed!');
1056 }
1057 }
1058 $this->unlinkFiles = array();
1059 }
1060
1061 /***************************
1062 * Import / Relations setting
1063 ***************************/
1064
1065 /**
1066 * At the end of the import process all file and DB relations should be set properly (that is relations
1067 * to imported records are all re-created so imported records are correctly related again)
1068 * Relations in flexform fields are processed in setFlexFormRelations() after this function
1069 *
1070 * @return void
1071 * @see setFlexFormRelations()
1072 */
1073 public function setRelations()
1074 {
1075 $updateData = array();
1076 // import_newId contains a register of all records that was in the import memorys "records" key
1077 foreach ($this->import_newId as $nId => $dat) {
1078 $table = $dat['table'];
1079 $uid = $dat['uid'];
1080 // original UID - NOT the new one!
1081 // If the record has been written and received a new id, then proceed:
1082 if (is_array($this->import_mapId[$table]) && isset($this->import_mapId[$table][$uid])) {
1083 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
1084 if (is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {
1085 $thisNewPageUid = 0;
1086 if ($this->legacyImport) {
1087 if ($table != 'pages') {
1088 $oldPid = $this->dat['records'][$table . ':' . $uid]['data']['pid'];
1089 $thisNewPageUid = BackendUtility::wsMapId($table, $this->import_mapId['pages'][$oldPid]);
1090 } else {
1091 $thisNewPageUid = $thisNewUid;
1092 }
1093 }
1094 // Traverse relation fields of each record
1095 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
1096 // uid_local of sys_file_reference needs no update because the correct reference uid was already written
1097 // @see ImportExport::fixUidLocalInSysFileReferenceRecords()
1098 if ($table === 'sys_file_reference' && $field === 'uid_local') {
1099 continue;
1100 }
1101 switch ((string)$config['type']) {
1102 case 'db':
1103 if (is_array($config['itemArray']) && !empty($config['itemArray'])) {
1104 $itemConfig = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
1105 $valArray = $this->setRelations_db($config['itemArray'], $itemConfig);
1106 $updateData[$table][$thisNewUid][$field] = implode(',', $valArray);
1107 }
1108 break;
1109 case 'file':
1110 if (is_array($config['newValueFiles']) && !empty($config['newValueFiles'])) {
1111 $valArr = array();
1112 foreach ($config['newValueFiles'] as $fI) {
1113 $valArr[] = $this->import_addFileNameToBeCopied($fI);
1114 }
1115 if ($this->legacyImport && $this->legacyImportFolder === null && isset($this->legacyImportMigrationTables[$table][$field])) {
1116 // Do nothing - the legacy import folder is missing
1117 } elseif ($this->legacyImport && $this->legacyImportFolder !== null && isset($this->legacyImportMigrationTables[$table][$field])) {
1118 $refIds = array();
1119 foreach ($valArr as $tempFile) {
1120 $fileName = $this->alternativeFileName[$tempFile];
1121 $fileObject = null;
1122
1123 try {
1124 // check, if there is alreay the same file in the folder
1125 if ($this->legacyImportFolder->hasFile($fileName)) {
1126 $fileStorage = $this->legacyImportFolder->getStorage();
1127 $file = $fileStorage->getFile($this->legacyImportFolder->getIdentifier() . $fileName);
1128 if ($file->getSha1() === sha1_file($tempFile)) {
1129 $fileObject = $file;
1130 }
1131 }
1132 } catch (Exception $e) {
1133 }
1134
1135 if ($fileObject === null) {
1136 try {
1137 $fileObject = $this->legacyImportFolder->addFile($tempFile, $fileName, DuplicationBehavior::RENAME);
1138 } catch (Exception $e) {
1139 $this->error('Error: no file could be added to the storage for file name' . $this->alternativeFileName[$tempFile]);
1140 }
1141 }
1142 if ($fileObject !== null) {
1143 $refId = StringUtility::getUniqueId('NEW');
1144 $refIds[] = $refId;
1145 $updateData['sys_file_reference'][$refId] = array(
1146 'uid_local' => $fileObject->getUid(),
1147 'uid_foreign' => $thisNewUid, // uid of your content record
1148 'tablenames' => $table,
1149 'fieldname' => $field,
1150 'pid' => $thisNewPageUid, // parent id of the parent page
1151 'table_local' => 'sys_file',
1152 );
1153 }
1154 }
1155 $updateData[$table][$thisNewUid][$field] = implode(',', $refIds);
1156 if (!empty($this->legacyImportMigrationTables[$table][$field])) {
1157 $this->legacyImportMigrationRecords[$table][$thisNewUid][$field] = $refIds;
1158 }
1159 } else {
1160 $updateData[$table][$thisNewUid][$field] = implode(',', $valArr);
1161 }
1162 }
1163 break;
1164 }
1165 }
1166 } else {
1167 $this->error('Error: no record was found in data array!');
1168 }
1169 } else {
1170 $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')');
1171 }
1172 }
1173 if (!empty($updateData)) {
1174 $tce = $this->getNewTCE();
1175 $tce->isImporting = true;
1176 $this->callHook('before_setRelation', array(
1177 'tce' => &$tce,
1178 'data' => &$updateData
1179 ));
1180 $tce->start($updateData, array());
1181 $tce->process_datamap();
1182 // Replace the temporary "NEW" ids with the final ones.
1183 foreach ($this->legacyImportMigrationRecords as $table => $records) {
1184 foreach ($records as $uid => $fields) {
1185 foreach ($fields as $field => $referenceIds) {
1186 foreach ($referenceIds as $key => $referenceId) {
1187 $this->legacyImportMigrationRecords[$table][$uid][$field][$key] = $tce->substNEWwithIDs[$referenceId];
1188 }
1189 }
1190 }
1191 }
1192 $this->callHook('after_setRelations', array(
1193 'tce' => &$tce
1194 ));
1195 }
1196 }
1197
1198 /**
1199 * Maps relations for database
1200 *
1201 * @param array $itemArray Array of item sets (table/uid) from a dbAnalysis object
1202 * @param array $itemConfig Array of TCA config of the field the relation to be set on
1203 * @return array Array with values [table]_[uid] or [uid] for field of type group / internal_type file_reference. These values have the regular tcemain-input group/select type which means they will automatically be processed into a uid-list or MM relations.
1204 */
1205 public function setRelations_db($itemArray, $itemConfig)
1206 {
1207 $valArray = array();
1208 foreach ($itemArray as $relDat) {
1209 if (is_array($this->import_mapId[$relDat['table']]) && isset($this->import_mapId[$relDat['table']][$relDat['id']])) {
1210 // Since non FAL file relation type group internal_type file_reference are handled as reference to
1211 // sys_file records Datahandler requires the value as uid of the the related sys_file record only
1212 if ($itemConfig['type'] === 'group' && $itemConfig['internal_type'] === 'file_reference') {
1213 $value = $this->import_mapId[$relDat['table']][$relDat['id']];
1214 } elseif ($itemConfig['type'] === 'input' && isset($itemConfig['wizards']['link'])) {
1215 // If an input field has a relation to a sys_file record this need to be converted back to
1216 // the public path. But use getPublicUrl here, because could normally only be a local file path.
1217 $fileUid = $this->import_mapId[$relDat['table']][$relDat['id']];
1218 // Fallback value
1219 $value = 'file:' . $fileUid;
1220 try {
1221 $file = ResourceFactory::getInstance()->retrieveFileOrFolderObject($fileUid);
1222 } catch (\Exception $e) {
1223 $file = null;
1224 }
1225 if ($file instanceof FileInterface) {
1226 $value = $file->getPublicUrl();
1227 }
1228 } else {
1229 $value = $relDat['table'] . '_' . $this->import_mapId[$relDat['table']][$relDat['id']];
1230 }
1231 $valArray[] = $value;
1232 } elseif ($this->isTableStatic($relDat['table']) || $this->isExcluded($relDat['table'], $relDat['id']) || $relDat['id'] < 0) {
1233 // Checking for less than zero because some select types could contain negative values,
1234 // eg. fe_groups (-1, -2) and sys_language (-1 = ALL languages). This must be handled on both export and import.
1235 $valArray[] = $relDat['table'] . '_' . $relDat['id'];
1236 } else {
1237 $this->error('Lost relation: ' . $relDat['table'] . ':' . $relDat['id']);
1238 }
1239 }
1240 return $valArray;
1241 }
1242
1243 /**
1244 * Writes the file from import array to temp dir and returns the filename of it.
1245 *
1246 * @param array $fI File information with three keys: "filename" = filename without path, "ID_absFile" = absolute filepath to the file (including the filename), "ID" = md5 hash of "ID_absFile
1247 * @return string|NULL Absolute filename of the temporary filename of the file. In ->alternativeFileName the original name is set.
1248 */
1249 public function import_addFileNameToBeCopied($fI)
1250 {
1251 if (is_array($this->dat['files'][$fI['ID']])) {
1252 $tmpFile = null;
1253 // check if there is the right file already in the local folder
1254 if ($this->filesPathForImport !== null) {
1255 if (is_file($this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5']) &&
1256 md5_file($this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5']) === $this->dat['files'][$fI['ID']]['content_md5']) {
1257 $tmpFile = $this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5'];
1258 }
1259 }
1260 if ($tmpFile === null) {
1261 $tmpFile = GeneralUtility::tempnam('import_temp_');
1262 GeneralUtility::writeFile($tmpFile, $this->dat['files'][$fI['ID']]['content']);
1263 }
1264 clearstatcache();
1265 if (@is_file($tmpFile)) {
1266 $this->unlinkFiles[] = $tmpFile;
1267 if (filesize($tmpFile) == $this->dat['files'][$fI['ID']]['filesize']) {
1268 $this->alternativeFileName[$tmpFile] = $fI['filename'];
1269 $this->alternativeFilePath[$tmpFile] = $this->dat['files'][$fI['ID']]['relFileRef'];
1270 return $tmpFile;
1271 } else {
1272 $this->error('Error: temporary file ' . $tmpFile . ' had a size (' . filesize($tmpFile) . ') different from the original (' . $this->dat['files'][$fI['ID']]['filesize'] . ')');
1273 }
1274 } else {
1275 $this->error('Error: temporary file ' . $tmpFile . ' was not written as it should have been!');
1276 }
1277 } else {
1278 $this->error('Error: No file found for ID ' . $fI['ID']);
1279 }
1280 return null;
1281 }
1282
1283 /**
1284 * After all DB relations has been set in the end of the import (see setRelations()) then it is time to correct all relations inside of FlexForm fields.
1285 * The reason for doing this after is that the setting of relations may affect (quite often!) which data structure is used for the flexforms field!
1286 *
1287 * @return void
1288 * @see setRelations()
1289 */
1290 public function setFlexFormRelations()
1291 {
1292 $updateData = array();
1293 // import_newId contains a register of all records that was in the import memorys "records" key
1294 foreach ($this->import_newId as $nId => $dat) {
1295 $table = $dat['table'];
1296 $uid = $dat['uid'];
1297 // original UID - NOT the new one!
1298 // If the record has been written and received a new id, then proceed:
1299 if (!isset($this->import_mapId[$table][$uid])) {
1300 $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')');
1301 continue;
1302 }
1303
1304 if (!is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {
1305 $this->error('Error: no record was found in data array!');
1306 continue;
1307 }
1308 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
1309 // Traverse relation fields of each record
1310 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
1311 switch ((string)$config['type']) {
1312 case 'flex':
1313 // Get XML content and set as default value (string, non-processed):
1314 $updateData[$table][$thisNewUid][$field] = $this->dat['records'][$table . ':' . $uid]['data'][$field];
1315 // If there has been registered relations inside the flex form field, run processing on the content:
1316 if (!empty($config['flexFormRels']['db']) || !empty($config['flexFormRels']['file'])) {
1317 $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');
1318 // This will fetch the new row for the element (which should be updated with any references to data structures etc.)
1319 $conf = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
1320 if (is_array($origRecordRow) && is_array($conf) && $conf['type'] === 'flex') {
1321 // Get current data structure and value array:
1322 $dataStructArray = BackendUtility::getFlexFormDS($conf, $origRecordRow, $table, $field);
1323 $currentValueArray = GeneralUtility::xml2array($updateData[$table][$thisNewUid][$field]);
1324 // Do recursive processing of the XML data:
1325 $iteratorObj = GeneralUtility::makeInstance(DataHandler::class);
1326 $iteratorObj->callBackObj = $this;
1327 $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData(
1328 $currentValueArray['data'],
1329 array(),
1330 array(),
1331 $dataStructArray,
1332 array($table, $thisNewUid, $field, $config),
1333 'remapListedDBRecords_flexFormCallBack'
1334 );
1335 // The return value is set as an array which means it will be processed by tcemain for file and DB references!
1336 if (is_array($currentValueArray['data'])) {
1337 $updateData[$table][$thisNewUid][$field] = $currentValueArray;
1338 }
1339 }
1340 }
1341 break;
1342 }
1343 }
1344 }
1345 if (!empty($updateData)) {
1346 $tce = $this->getNewTCE();
1347 $tce->isImporting = true;
1348 $this->callHook('before_setFlexFormRelations', array(
1349 'tce' => &$tce,
1350 'data' => &$updateData
1351 ));
1352 $tce->start($updateData, array());
1353 $tce->process_datamap();
1354 $this->callHook('after_setFlexFormRelations', array(
1355 'tce' => &$tce
1356 ));
1357 }
1358 }
1359
1360 /**
1361 * Callback function for traversing the FlexForm structure in relation to remapping database relations
1362 *
1363 * @param array $pParams Set of parameters in numeric array: table, uid, field
1364 * @param array $dsConf TCA config for field (from Data Structure of course)
1365 * @param string $dataValue Field value (from FlexForm XML)
1366 * @param string $dataValue_ext1 Not used
1367 * @param string $dataValue_ext2 Not used
1368 * @param string $path Path of where the data structure of the element is found
1369 * @return array Array where the "value" key carries the value.
1370 * @see setFlexFormRelations()
1371 */
1372 public function remapListedDBRecords_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path)
1373 {
1374 // Extract parameters:
1375 list(, , , $config) = $pParams;
1376 // In case the $path is used as index without a trailing slash we will remove that
1377 if (!is_array($config['flexFormRels']['db'][$path]) && is_array($config['flexFormRels']['db'][rtrim($path, '/')])) {
1378 $path = rtrim($path, '/');
1379 }
1380 if (is_array($config['flexFormRels']['db'][$path])) {
1381 $valArray = $this->setRelations_db($config['flexFormRels']['db'][$path], $dsConf);
1382 $dataValue = implode(',', $valArray);
1383 }
1384 if (is_array($config['flexFormRels']['file'][$path])) {
1385 $valArr = array();
1386 foreach ($config['flexFormRels']['file'][$path] as $fI) {
1387 $valArr[] = $this->import_addFileNameToBeCopied($fI);
1388 }
1389 $dataValue = implode(',', $valArr);
1390 }
1391 return array('value' => $dataValue);
1392 }
1393
1394 /**************************
1395 * Import / Soft References
1396 *************************/
1397
1398 /**
1399 * Processing of soft references
1400 *
1401 * @return void
1402 */
1403 public function processSoftReferences()
1404 {
1405 // Initialize:
1406 $inData = array();
1407 // Traverse records:
1408 if (is_array($this->dat['header']['records'])) {
1409 foreach ($this->dat['header']['records'] as $table => $recs) {
1410 foreach ($recs as $uid => $thisRec) {
1411 // If there are soft references defined, traverse those:
1412 if (isset($GLOBALS['TCA'][$table]) && is_array($thisRec['softrefs'])) {
1413 // First traversal is to collect softref configuration and split them up based on fields.
1414 // This could probably also have been done with the "records" key instead of the header.
1415 $fieldsIndex = array();
1416 foreach ($thisRec['softrefs'] as $softrefDef) {
1417 // If a substitution token is set:
1418 if ($softrefDef['field'] && is_array($softrefDef['subst']) && $softrefDef['subst']['tokenID']) {
1419 $fieldsIndex[$softrefDef['field']][$softrefDef['subst']['tokenID']] = $softrefDef;
1420 }
1421 }
1422 // The new id:
1423 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
1424 // Now, if there are any fields that require substitution to be done, lets go for that:
1425 foreach ($fieldsIndex as $field => $softRefCfgs) {
1426 if (is_array($GLOBALS['TCA'][$table]['columns'][$field])) {
1427 $conf = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
1428 if ($conf['type'] === 'flex') {
1429 // This will fetch the new row for the element (which should be updated with any references to data structures etc.)
1430 $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');
1431 if (is_array($origRecordRow)) {
1432 // Get current data structure and value array:
1433 $dataStructArray = BackendUtility::getFlexFormDS($conf, $origRecordRow, $table, $field);
1434 $currentValueArray = GeneralUtility::xml2array($origRecordRow[$field]);
1435 // Do recursive processing of the XML data:
1436 /** @var $iteratorObj DataHandler */
1437 $iteratorObj = GeneralUtility::makeInstance(DataHandler::class);
1438 $iteratorObj->callBackObj = $this;
1439 $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData($currentValueArray['data'], array(), array(), $dataStructArray, array($table, $uid, $field, $softRefCfgs), 'processSoftReferences_flexFormCallBack');
1440 // The return value is set as an array which means it will be processed by tcemain for file and DB references!
1441 if (is_array($currentValueArray['data'])) {
1442 $inData[$table][$thisNewUid][$field] = $currentValueArray;
1443 }
1444 }
1445 } else {
1446 // Get tokenizedContent string and proceed only if that is not blank:
1447 $tokenizedContent = $this->dat['records'][$table . ':' . $uid]['rels'][$field]['softrefs']['tokenizedContent'];
1448 if (strlen($tokenizedContent) && is_array($softRefCfgs)) {
1449 $inData[$table][$thisNewUid][$field] = $this->processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid);
1450 }
1451 }
1452 }
1453 }
1454 }
1455 }
1456 }
1457 }
1458 // Now write to database:
1459 $tce = $this->getNewTCE();
1460 $tce->isImporting = true;
1461 $this->callHook('before_processSoftReferences', array(
1462 'tce' => $tce,
1463 'data' => &$inData
1464 ));
1465 $tce->enableLogging = true;
1466 $tce->start($inData, array());
1467 $tce->process_datamap();
1468 $this->callHook('after_processSoftReferences', array(
1469 'tce' => $tce
1470 ));
1471 }
1472
1473 /**
1474 * Callback function for traversing the FlexForm structure in relation to remapping softreference relations
1475 *
1476 * @param array $pParams Set of parameters in numeric array: table, uid, field
1477 * @param array $dsConf TCA config for field (from Data Structure of course)
1478 * @param string $dataValue Field value (from FlexForm XML)
1479 * @param string $dataValue_ext1 Not used
1480 * @param string $dataValue_ext2 Not used
1481 * @param string $path Path of where the data structure where the element is found
1482 * @return array Array where the "value" key carries the value.
1483 * @see setFlexFormRelations()
1484 */
1485 public function processSoftReferences_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path)
1486 {
1487 // Extract parameters:
1488 list($table, $origUid, $field, $softRefCfgs) = $pParams;
1489 if (is_array($softRefCfgs)) {
1490 // First, find all soft reference configurations for this structure path (they are listed flat in the header):
1491 $thisSoftRefCfgList = array();
1492 foreach ($softRefCfgs as $sK => $sV) {
1493 if ($sV['structurePath'] === $path) {
1494 $thisSoftRefCfgList[$sK] = $sV;
1495 }
1496 }
1497 // If any was found, do processing:
1498 if (!empty($thisSoftRefCfgList)) {
1499 // Get tokenizedContent string and proceed only if that is not blank:
1500 $tokenizedContent = $this->dat['records'][$table . ':' . $origUid]['rels'][$field]['flexFormRels']['softrefs'][$path]['tokenizedContent'];
1501 if (strlen($tokenizedContent)) {
1502 $dataValue = $this->processSoftReferences_substTokens($tokenizedContent, $thisSoftRefCfgList, $table, $origUid);
1503 }
1504 }
1505 }
1506 // Return
1507 return array('value' => $dataValue);
1508 }
1509
1510 /**
1511 * Substition of softreference tokens
1512 *
1513 * @param string $tokenizedContent Content of field with soft reference tokens in.
1514 * @param array $softRefCfgs Soft reference configurations
1515 * @param string $table Table for which the processing occurs
1516 * @param string $uid UID of record from table
1517 * @return string The input content with tokens substituted according to entries in softRefCfgs
1518 */
1519 public function processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid)
1520 {
1521 // traverse each softref type for this field:
1522 foreach ($softRefCfgs as $cfg) {
1523 // Get token ID:
1524 $tokenID = $cfg['subst']['tokenID'];
1525 // Default is current token value:
1526 $insertValue = $cfg['subst']['tokenValue'];
1527 // Based on mode:
1528 switch ((string)$this->softrefCfg[$tokenID]['mode']) {
1529 case 'exclude':
1530 // Exclude is a simple passthrough of the value
1531 break;
1532 case 'editable':
1533 // Editable always picks up the value from this input array:
1534 $insertValue = $this->softrefInputValues[$tokenID];
1535 break;
1536 default:
1537 // Mapping IDs/creating files: Based on type, look up new value:
1538 switch ((string)$cfg['subst']['type']) {
1539 case 'file':
1540 // Create / Overwrite file:
1541 $insertValue = $this->processSoftReferences_saveFile($cfg['subst']['relFileName'], $cfg, $table, $uid);
1542 break;
1543 case 'db':
1544 default:
1545 // Trying to map database element if found in the mapID array:
1546 list($tempTable, $tempUid) = explode(':', $cfg['subst']['recordRef']);
1547 if (isset($this->import_mapId[$tempTable][$tempUid])) {
1548 $insertValue = BackendUtility::wsMapId($tempTable, $this->import_mapId[$tempTable][$tempUid]);
1549 // Look if reference is to a page and the original token value was NOT an integer - then we assume is was an alias and try to look up the new one!
1550 if ($tempTable === 'pages' && !MathUtility::canBeInterpretedAsInteger($cfg['subst']['tokenValue'])) {
1551 $recWithUniqueValue = BackendUtility::getRecord($tempTable, $insertValue, 'alias');
1552 if ($recWithUniqueValue['alias']) {
1553 $insertValue = $recWithUniqueValue['alias'];
1554 }
1555 } elseif (strpos($cfg['subst']['tokenValue'], ':') !== false) {
1556 list($tokenKey) = explode(':', $cfg['subst']['tokenValue']);
1557 $insertValue = $tokenKey . ':' . $insertValue;
1558 }
1559 }
1560 }
1561 }
1562 // Finally, swap the soft reference token in tokenized content with the insert value:
1563 $tokenizedContent = str_replace('{softref:' . $tokenID . '}', $insertValue, $tokenizedContent);
1564 }
1565 return $tokenizedContent;
1566 }
1567
1568 /**
1569 * Process a soft reference file
1570 *
1571 * @param string $relFileName Old Relative filename
1572 * @param array $cfg soft reference configuration array
1573 * @param string $table Table for which the processing occurs
1574 * @param string $uid UID of record from table
1575 * @return string New relative filename (value to insert instead of the softref token)
1576 */
1577 public function processSoftReferences_saveFile($relFileName, $cfg, $table, $uid)
1578 {
1579 if ($fileHeaderInfo = $this->dat['header']['files'][$cfg['file_ID']]) {
1580 // Initialize; Get directory prefix for file and find possible RTE filename
1581 $dirPrefix = PathUtility::dirname($relFileName) . '/';
1582 $rteOrigName = $this->getRTEoriginalFilename(PathUtility::basename($relFileName));
1583 // If filename looks like an RTE file, and the directory is in "uploads/", then process as a RTE file!
1584 if ($rteOrigName && GeneralUtility::isFirstPartOfStr($dirPrefix, 'uploads/')) {
1585 // RTE:
1586 // First, find unique RTE file name:
1587 if (@is_dir((PATH_site . $dirPrefix))) {
1588 // From the "original" RTE filename, produce a new "original" destination filename which is unused.
1589 // Even if updated, the image should be unique. Currently the problem with this is that it leaves a lot of unused RTE images...
1590 $fileProcObj = $this->getFileProcObj();
1591 $origDestName = $fileProcObj->getUniqueName($rteOrigName, PATH_site . $dirPrefix);
1592 // Create copy file name:
1593 $pI = pathinfo($relFileName);
1594 $copyDestName = PathUtility::dirname($origDestName) . '/RTEmagicC_' . substr(PathUtility::basename($origDestName), 10) . '.' . $pI['extension'];
1595 if (
1596 !@is_file($copyDestName) && !@is_file($origDestName)
1597 && $origDestName === GeneralUtility::getFileAbsFileName($origDestName)
1598 && $copyDestName === GeneralUtility::getFileAbsFileName($copyDestName)
1599 ) {
1600 if ($this->dat['header']['files'][$fileHeaderInfo['RTE_ORIG_ID']]) {
1601 if ($this->legacyImport) {
1602 $fileName = PathUtility::basename($copyDestName);
1603 $this->writeSysFileResourceForLegacyImport($fileName, $cfg['file_ID']);
1604 $relFileName = $this->filePathMap[$cfg['file_ID']] . '" data-htmlarea-file-uid="' . $fileName . '" data-htmlarea-file-table="sys_file';
1605 // Also save the original file
1606 $originalFileName = PathUtility::basename($origDestName);
1607 $this->writeSysFileResourceForLegacyImport($originalFileName, $fileHeaderInfo['RTE_ORIG_ID']);
1608 } else {
1609 // Write the copy and original RTE file to the respective filenames:
1610 $this->writeFileVerify($copyDestName, $cfg['file_ID'], true);
1611 $this->writeFileVerify($origDestName, $fileHeaderInfo['RTE_ORIG_ID'], true);
1612 // Return the relative path of the copy file name:
1613 return PathUtility::stripPathSitePrefix($copyDestName);
1614 }
1615 } else {
1616 $this->error('ERROR: Could not find original file ID');
1617 }
1618 } else {
1619 $this->error('ERROR: The destination filenames "' . $copyDestName . '" and "' . $origDestName . '" either existed or have non-valid names');
1620 }
1621 } else {
1622 $this->error('ERROR: "' . PATH_site . $dirPrefix . '" was not a directory, so could not process file "' . $relFileName . '"');
1623 }
1624 } elseif (GeneralUtility::isFirstPartOfStr($dirPrefix, $this->fileadminFolderName . '/')) {
1625 // File in fileadmin/ folder:
1626 // Create file (and possible resources)
1627 $newFileName = $this->processSoftReferences_saveFile_createRelFile($dirPrefix, PathUtility::basename($relFileName), $cfg['file_ID'], $table, $uid);
1628 if (strlen($newFileName)) {
1629 $relFileName = $newFileName;
1630 } else {
1631 $this->error('ERROR: No new file created for "' . $relFileName . '"');
1632 }
1633 } else {
1634 $this->error('ERROR: Sorry, cannot operate on non-RTE files which are outside the fileadmin folder.');
1635 }
1636 } else {
1637 $this->error('ERROR: Could not find file ID in header.');
1638 }
1639 // Return (new) filename relative to PATH_site:
1640 return $relFileName;
1641 }
1642
1643 /**
1644 * Create file in directory and return the new (unique) filename
1645 *
1646 * @param string $origDirPrefix Directory prefix, relative, with trailing slash
1647 * @param string $fileName Filename (without path)
1648 * @param string $fileID File ID from import memory
1649 * @param string $table Table for which the processing occurs
1650 * @param string $uid UID of record from table
1651 * @return string|NULL New relative filename, if any
1652 */
1653 public function processSoftReferences_saveFile_createRelFile($origDirPrefix, $fileName, $fileID, $table, $uid)
1654 {
1655 // If the fileID map contains an entry for this fileID then just return the relative filename of that entry;
1656 // we don't want to write another unique filename for this one!
1657 if (isset($this->fileIDMap[$fileID])) {
1658 return PathUtility::stripPathSitePrefix($this->fileIDMap[$fileID]);
1659 }
1660 if ($this->legacyImport) {
1661 // set dirPrefix to fileadmin because the right target folder is set and checked for permissions later
1662 $dirPrefix = $this->fileadminFolderName . '/';
1663 } else {
1664 // Verify FileMount access to dir-prefix. Returns the best alternative relative path if any
1665 $dirPrefix = $this->verifyFolderAccess($origDirPrefix);
1666 }
1667 if ($dirPrefix && (!$this->update || $origDirPrefix === $dirPrefix) && $this->checkOrCreateDir($dirPrefix)) {
1668 $fileHeaderInfo = $this->dat['header']['files'][$fileID];
1669 $updMode = $this->update && $this->import_mapId[$table][$uid] === $uid && $this->import_mode[$table . ':' . $uid] !== 'as_new';
1670 // Create new name for file:
1671 // Must have same ID in map array (just for security, is not really needed) and NOT be set "as_new".
1672
1673 // Write main file:
1674 if ($this->legacyImport) {
1675 $fileWritten = $this->writeSysFileResourceForLegacyImport($fileName, $fileID);
1676 if ($fileWritten) {
1677 $newName = 'file:' . $fileName;
1678 return $newName;
1679 // no support for HTML/CSS file resources attached ATM - see below
1680 }
1681 } else {
1682 if ($updMode) {
1683 $newName = PATH_site . $dirPrefix . $fileName;
1684 } else {
1685 // Create unique filename:
1686 $fileProcObj = $this->getFileProcObj();
1687 $newName = $fileProcObj->getUniqueName($fileName, PATH_site . $dirPrefix);
1688 }
1689 if ($this->writeFileVerify($newName, $fileID)) {
1690 // If the resource was an HTML/CSS file with resources attached, we will write those as well!
1691 if (is_array($fileHeaderInfo['EXT_RES_ID'])) {
1692 $tokenizedContent = $this->dat['files'][$fileID]['tokenizedContent'];
1693 $tokenSubstituted = false;
1694 $fileProcObj = $this->getFileProcObj();
1695 if ($updMode) {
1696 foreach ($fileHeaderInfo['EXT_RES_ID'] as $res_fileID) {
1697 if ($this->dat['files'][$res_fileID]['filename']) {
1698 // Resolve original filename:
1699 $relResourceFileName = $this->dat['files'][$res_fileID]['parentRelFileName'];
1700 $absResourceFileName = GeneralUtility::resolveBackPath(PATH_site . $origDirPrefix . $relResourceFileName);
1701 $absResourceFileName = GeneralUtility::getFileAbsFileName($absResourceFileName);
1702 if ($absResourceFileName && GeneralUtility::isFirstPartOfStr($absResourceFileName, PATH_site . $this->fileadminFolderName . '/')) {
1703 $destDir = PathUtility::stripPathSitePrefix(PathUtility::dirname($absResourceFileName) . '/');
1704 if ($this->verifyFolderAccess($destDir, true) && $this->checkOrCreateDir($destDir)) {
1705 $this->writeFileVerify($absResourceFileName, $res_fileID);
1706 } else {
1707 $this->error('ERROR: Could not create file in directory "' . $destDir . '"');
1708 }
1709 } else {
1710 $this->error('ERROR: Could not resolve path for "' . $relResourceFileName . '"');
1711 }
1712 $tokenizedContent = str_replace('{EXT_RES_ID:' . $res_fileID . '}', $relResourceFileName, $tokenizedContent);
1713 $tokenSubstituted = true;
1714 }
1715 }
1716 } else {
1717 // Create the resouces directory name (filename without extension, suffixed "_FILES")
1718 $resourceDir = PathUtility::dirname($newName) . '/' . preg_replace('/\\.[^.]*$/', '', PathUtility::basename($newName)) . '_FILES';
1719 if (GeneralUtility::mkdir($resourceDir)) {
1720 foreach ($fileHeaderInfo['EXT_RES_ID'] as $res_fileID) {
1721 if ($this->dat['files'][$res_fileID]['filename']) {
1722 $absResourceFileName = $fileProcObj->getUniqueName($this->dat['files'][$res_fileID]['filename'], $resourceDir);
1723 $relResourceFileName = substr($absResourceFileName, strlen(PathUtility::dirname($resourceDir)) + 1);
1724 $this->writeFileVerify($absResourceFileName, $res_fileID);
1725 $tokenizedContent = str_replace('{EXT_RES_ID:' . $res_fileID . '}', $relResourceFileName, $tokenizedContent);
1726 $tokenSubstituted = true;
1727 }
1728 }
1729 }
1730 }
1731 // If substitutions has been made, write the content to the file again:
1732 if ($tokenSubstituted) {
1733 GeneralUtility::writeFile($newName, $tokenizedContent);
1734 }
1735 }
1736 return PathUtility::stripPathSitePrefix($newName);
1737 }
1738 }
1739 }
1740 return null;
1741 }
1742
1743 /**
1744 * Writes a file from the import memory having $fileID to file name $fileName which must be an absolute path inside PATH_site
1745 *
1746 * @param string $fileName Absolute filename inside PATH_site to write to
1747 * @param string $fileID File ID from import memory
1748 * @param bool $bypassMountCheck Bypasses the checking against filemounts - only for RTE files!
1749 * @return bool Returns TRUE if it went well. Notice that the content of the file is read again, and md5 from import memory is validated.
1750 */
1751 public function writeFileVerify($fileName, $fileID, $bypassMountCheck = false)
1752 {
1753 $fileProcObj = $this->getFileProcObj();
1754 if (!$fileProcObj->actionPerms['addFile']) {
1755 $this->error('ERROR: You did not have sufficient permissions to write the file "' . $fileName . '"');
1756 return false;
1757 }
1758 // Just for security, check again. Should actually not be necessary.
1759 if (!$fileProcObj->checkPathAgainstMounts($fileName) && !$bypassMountCheck) {
1760 $this->error('ERROR: Filename "' . $fileName . '" was not allowed in destination path!');
1761 return false;
1762 }
1763 $fI = GeneralUtility::split_fileref($fileName);
1764 if (!$fileProcObj->checkIfAllowed($fI['fileext'], $fI['path'], $fI['file']) && (!$this->allowPHPScripts || !$this->getBackendUser()->isAdmin())) {
1765 $this->error('ERROR: Filename "' . $fileName . '" failed against extension check or deny-pattern!');
1766 return false;
1767 }
1768 if (!GeneralUtility::getFileAbsFileName($fileName)) {
1769 $this->error('ERROR: Filename "' . $fileName . '" was not a valid relative file path!');
1770 return false;
1771 }
1772 if (!$this->dat['files'][$fileID]) {
1773 $this->error('ERROR: File ID "' . $fileID . '" could not be found');
1774 return false;
1775 }
1776 GeneralUtility::writeFile($fileName, $this->dat['files'][$fileID]['content']);
1777 $this->fileIDMap[$fileID] = $fileName;
1778 if (md5(GeneralUtility::getUrl($fileName)) == $this->dat['files'][$fileID]['content_md5']) {
1779 return true;
1780 } else {
1781 $this->error('ERROR: File content "' . $fileName . '" was corrupted');
1782 return false;
1783 }
1784 }
1785
1786 /**
1787 * Writes the file with the is $fileId to the legacy import folder. The file name will used from
1788 * argument $fileName and the file was successfully created or an identical file was already found,
1789 * $fileName will held the uid of the new created file record.
1790 *
1791 * @param string $fileName The file name for the new file. Value would be changed to the uid of the new created file record.
1792 * @param int $fileId The id of the file in data array
1793 * @return bool
1794 */
1795 protected function writeSysFileResourceForLegacyImport(&$fileName, $fileId)
1796 {
1797 if ($this->legacyImportFolder === null) {
1798 return false;
1799 }
1800
1801 if (!isset($this->dat['files'][$fileId])) {
1802 $this->error('ERROR: File ID "' . $fileId . '" could not be found');
1803 return false;
1804 }
1805
1806 $temporaryFile = $this->writeTemporaryFileFromData($fileId, 'files');
1807 if ($temporaryFile === null) {
1808 // error on writing the file. Error message was already added
1809 return false;
1810 }
1811
1812 $importFolder = $this->legacyImportFolder;
1813
1814 if (isset($this->dat['files'][$fileId]['relFileName'])) {
1815 $relativeFilePath = PathUtility::dirname($this->dat['files'][$fileId]['relFileName']);
1816
1817 if (!$this->legacyImportFolder->hasFolder($relativeFilePath)) {
1818 $this->legacyImportFolder->createFolder($relativeFilePath);
1819 }
1820 $importFolder = $this->legacyImportFolder->getSubfolder($relativeFilePath);
1821 }
1822
1823 $fileObject = null;
1824
1825 try {
1826 // check, if there is alreay the same file in the folder
1827 if ($importFolder->hasFile($fileName)) {
1828 $fileStorage = $importFolder->getStorage();
1829 $file = $fileStorage->getFile($importFolder->getIdentifier() . $fileName);
1830 if ($file->getSha1() === sha1_file($temporaryFile)) {
1831 $fileObject = $file;
1832 }
1833 }
1834 } catch (Exception $e) {
1835 }
1836
1837 if ($fileObject === null) {
1838 try {
1839 $fileObject = $importFolder->addFile($temporaryFile, $fileName, DuplicationBehavior::RENAME);
1840 } catch (Exception $e) {
1841 $this->error('Error: no file could be added to the storage for file name ' . $this->alternativeFileName[$temporaryFile]);
1842 }
1843 }
1844
1845 if (md5_file(PATH_site . $fileObject->getPublicUrl()) == $this->dat['files'][$fileId]['content_md5']) {
1846 $fileName = $fileObject->getUid();
1847 $this->fileIDMap[$fileId] = $fileName;
1848 $this->filePathMap[$fileId] = $fileObject->getPublicUrl();
1849 return true;
1850 } else {
1851 $this->error('ERROR: File content "' . $this->dat['files'][$fileId]['relFileName'] . '" was corrupted');
1852 }
1853
1854 return false;
1855 }
1856
1857 /**
1858 * Migrate legacy import records
1859 *
1860 * @return void
1861 */
1862 protected function migrateLegacyImportRecords()
1863 {
1864 $updateData= array();
1865
1866 foreach ($this->legacyImportMigrationRecords as $table => $records) {
1867 foreach ($records as $uid => $fields) {
1868 $row = BackendUtility::getRecord($table, $uid);
1869 if (empty($row)) {
1870 continue;
1871 }
1872
1873 foreach ($fields as $field => $referenceIds) {
1874 $fieldConfiguration = $this->legacyImportMigrationTables[$table][$field];
1875
1876 if (isset($fieldConfiguration['titleTexts'])) {
1877 $titleTextField = $fieldConfiguration['titleTexts'];
1878 if (isset($row[$titleTextField]) && $row[$titleTextField] !== '') {
1879 $titleTextContents = explode(LF, $row[$titleTextField]);
1880 $updateData[$table][$uid][$titleTextField] = '';
1881 }
1882 }
1883
1884 if (isset($fieldConfiguration['alternativeTexts'])) {
1885 $alternativeTextField = $fieldConfiguration['alternativeTexts'];
1886 if (isset($row[$alternativeTextField]) && $row[$alternativeTextField] !== '') {
1887 $alternativeTextContents = explode(LF, $row[$alternativeTextField]);
1888 $updateData[$table][$uid][$alternativeTextField] = '';
1889 }
1890 }
1891 if (isset($fieldConfiguration['description'])) {
1892 $descriptionField = $fieldConfiguration['description'];
1893 if ($row[$descriptionField] !== '') {
1894 $descriptionContents = explode(LF, $row[$descriptionField]);
1895 $updateData[$table][$uid][$descriptionField] = '';
1896 }
1897 }
1898 if (isset($fieldConfiguration['links'])) {
1899 $linkField = $fieldConfiguration['links'];
1900 if ($row[$linkField] !== '') {
1901 $linkContents = explode(LF, $row[$linkField]);
1902 $updateData[$table][$uid][$linkField] = '';
1903 }
1904 }
1905
1906 foreach ($referenceIds as $key => $referenceId) {
1907 if (isset($titleTextContents[$key])) {
1908 $updateData['sys_file_reference'][$referenceId]['title'] = trim($titleTextContents[$key]);
1909 }
1910 if (isset($alternativeTextContents[$key])) {
1911 $updateData['sys_file_reference'][$referenceId]['alternative'] = trim($alternativeTextContents[$key]);
1912 }
1913 if (isset($descriptionContents[$key])) {
1914 $updateData['sys_file_reference'][$referenceId]['description'] = trim($descriptionContents[$key]);
1915 }
1916 if (isset($linkContents[$key])) {
1917 $updateData['sys_file_reference'][$referenceId]['link'] = trim($linkContents[$key]);
1918 }
1919 }
1920 }
1921 }
1922 }
1923
1924 // update
1925 $tce = $this->getNewTCE();
1926 $tce->isImporting = true;
1927 $tce->start($updateData, array());
1928 $tce->process_datamap();
1929 }
1930
1931 /**
1932 * Returns TRUE if directory exists and if it doesn't it will create directory and return TRUE if that succeeded.
1933 *
1934 * @param string $dirPrefix Directory to create. Having a trailing slash. Must be in fileadmin/. Relative to PATH_site
1935 * @return bool TRUE, if directory exists (was created)
1936 */
1937 public function checkOrCreateDir($dirPrefix)
1938 {
1939 // Split dir path and remove first directory (which should be "fileadmin")
1940 $filePathParts = explode('/', $dirPrefix);
1941 $firstDir = array_shift($filePathParts);
1942 if ($firstDir === $this->fileadminFolderName && GeneralUtility::getFileAbsFileName($dirPrefix)) {
1943 $pathAcc = '';
1944 foreach ($filePathParts as $dirname) {
1945 $pathAcc .= '/' . $dirname;
1946 if (strlen($dirname)) {
1947 if (!@is_dir((PATH_site . $this->fileadminFolderName . $pathAcc))) {
1948 if (!GeneralUtility::mkdir((PATH_site . $this->fileadminFolderName . $pathAcc))) {
1949 $this->error('ERROR: Directory could not be created....B');
1950 return false;
1951 }
1952 }
1953 } elseif ($dirPrefix === $this->fileadminFolderName . $pathAcc) {
1954 return true;
1955 } else {
1956 $this->error('ERROR: Directory could not be created....A');
1957 }
1958 }
1959 }
1960 return false;
1961 }
1962
1963 /**************************
1964 * File Input
1965 *************************/
1966
1967 /**
1968 * Loads the header section/all of the $filename into memory
1969 *
1970 * @param string $filename Filename, absolute
1971 * @param bool $all If set, all information is loaded (header, records and files). Otherwise the default is to read only the header information
1972 * @return bool TRUE if the operation went well
1973 */
1974 public function loadFile($filename, $all = false)
1975 {
1976 if (!@is_file($filename)) {
1977 $this->error('Filename not found: ' . $filename);
1978 return false;
1979 }
1980 $fI = pathinfo($filename);
1981 if (@is_dir($filename . '.files')) {
1982 if (GeneralUtility::isAllowedAbsPath($filename . '.files')) {
1983 // copy the folder lowlevel to typo3temp, because the files would be deleted after import
1984 $temporaryFolderName = $this->getTemporaryFolderName();
1985 GeneralUtility::copyDirectory($filename . '.files', $temporaryFolderName);
1986 $this->filesPathForImport = $temporaryFolderName;
1987 } else {
1988 $this->error('External import files for the given import source is currently not supported.');
1989 }
1990 }
1991 if (strtolower($fI['extension']) == 'xml') {
1992 // XML:
1993 $xmlContent = GeneralUtility::getUrl($filename);
1994 if (strlen($xmlContent)) {
1995 $this->dat = GeneralUtility::xml2array($xmlContent, '', true);
1996 if (is_array($this->dat)) {
1997 if ($this->dat['_DOCUMENT_TAG'] === 'T3RecordDocument' && is_array($this->dat['header']) && is_array($this->dat['records'])) {
1998 $this->loadInit();
1999 return true;
2000 } else {
2001 $this->error('XML file did not contain proper XML for TYPO3 Import');
2002 }
2003 } else {
2004 $this->error('XML could not be parsed: ' . $this->dat);
2005 }
2006 } else {
2007 $this->error('Error opening file: ' . $filename);
2008 }
2009 } else {
2010 // T3D
2011 if ($fd = fopen($filename, 'rb')) {
2012 $this->dat['header'] = $this->getNextFilePart($fd, 1, 'header');
2013 if ($all) {
2014 $this->dat['records'] = $this->getNextFilePart($fd, 1, 'records');
2015 $this->dat['files'] = $this->getNextFilePart($fd, 1, 'files');
2016 $this->dat['files_fal'] = $this->getNextFilePart($fd, 1, 'files_fal');
2017 }
2018 $this->loadInit();
2019 return true;
2020 } else {
2021 $this->error('Error opening file: ' . $filename);
2022 }
2023 fclose($fd);
2024 }
2025 return false;
2026 }
2027
2028 /**
2029 * Returns the next content part form the fileresource (t3d), $fd
2030 *
2031 * @param resource $fd File pointer
2032 * @param bool $unserialize If set, the returned content is unserialized into an array, otherwise you get the raw string
2033 * @param string $name For error messages this indicates the section of the problem.
2034 * @return string|NULL Data string or NULL in case of an error
2035 * @access private
2036 * @see loadFile()
2037 */
2038 public function getNextFilePart($fd, $unserialize = false, $name = '')
2039 {
2040 $initStrLen = 32 + 1 + 1 + 1 + 10 + 1;
2041 // Getting header data
2042 $initStr = fread($fd, $initStrLen);
2043 if (empty($initStr)) {
2044 $this->error('File does not contain data for "' . $name . '"');
2045 return null;
2046 }
2047 $initStrDat = explode(':', $initStr);
2048 if (strstr($initStrDat[0], 'Warning')) {
2049 $this->error('File read error: Warning message in file. (' . $initStr . fgets($fd) . ')');
2050 return null;
2051 }
2052 if ((string)$initStrDat[3] !== '') {
2053 $this->error('File read error: InitString had a wrong length. (' . $name . ')');
2054 return null;
2055 }
2056 $datString = fread($fd, (int)$initStrDat[2]);
2057 fread($fd, 1);
2058 if (md5($datString) === $initStrDat[0]) {
2059 if ($initStrDat[1]) {
2060 if ($this->compress) {
2061 $datString = gzuncompress($datString);
2062 } else {
2063 $this->error('Content read error: This file requires decompression, but this server does not offer gzcompress()/gzuncompress() functions.');
2064 return null;
2065 }
2066 }
2067 return $unserialize ? unserialize($datString) : $datString;
2068 } else {
2069 $this->error('MD5 check failed (' . $name . ')');
2070 }
2071 return null;
2072 }
2073
2074 /**
2075 * Loads T3D file content into the $this->dat array
2076 * (This function can be used to test the output strings from ->compileMemoryToFileContent())
2077 *
2078 * @param string $filecontent File content
2079 * @return void
2080 */
2081 public function loadContent($filecontent)
2082 {
2083 $pointer = 0;
2084 $this->dat['header'] = $this->getNextContentPart($filecontent, $pointer, 1, 'header');
2085 $this->dat['records'] = $this->getNextContentPart($filecontent, $pointer, 1, 'records');
2086 $this->dat['files'] = $this->getNextContentPart($filecontent, $pointer, 1, 'files');
2087 $this->loadInit();
2088 }
2089
2090 /**
2091 * Returns the next content part from the $filecontent
2092 *
2093 * @param string $filecontent File content string
2094 * @param int $pointer File pointer (where to read from)
2095 * @param bool $unserialize If set, the returned content is unserialized into an array, otherwise you get the raw string
2096 * @param string $name For error messages this indicates the section of the problem.
2097 * @return string|NULL Data string
2098 */
2099 public function getNextContentPart($filecontent, &$pointer, $unserialize = false, $name = '')
2100 {
2101 $initStrLen = 32 + 1 + 1 + 1 + 10 + 1;
2102 // getting header data
2103 $initStr = substr($filecontent, $pointer, $initStrLen);
2104 $pointer += $initStrLen;
2105 $initStrDat = explode(':', $initStr);
2106 if ((string)$initStrDat[3] !== '') {
2107 $this->error('Content read error: InitString had a wrong length. (' . $name . ')');
2108 return null;
2109 }
2110 $datString = substr($filecontent, $pointer, (int)$initStrDat[2]);
2111 $pointer += (int)$initStrDat[2] + 1;
2112 if (md5($datString) === $initStrDat[0]) {
2113 if ($initStrDat[1]) {
2114 if ($this->compress) {
2115 $datString = gzuncompress($datString);
2116 return $unserialize ? unserialize($datString) : $datString;
2117 } else {
2118 $this->error('Content read error: This file requires decompression, but this server does not offer gzcompress()/gzuncompress() functions.');
2119 }
2120 }
2121 } else {
2122 $this->error('MD5 check failed (' . $name . ')');
2123 }
2124 return null;
2125 }
2126
2127 /**
2128 * Setting up the object based on the recently loaded ->dat array
2129 *
2130 * @return void
2131 */
2132 public function loadInit()
2133 {
2134 $this->relStaticTables = (array)$this->dat['header']['relStaticTables'];
2135 $this->excludeMap = (array)$this->dat['header']['excludeMap'];
2136 $this->softrefCfg = (array)$this->dat['header']['softrefCfg'];
2137 $this->fixCharsets();
2138 if (
2139 isset($this->dat['header']['meta']['TYPO3_version'])
2140 && VersionNumberUtility::convertVersionNumberToInteger($this->dat['header']['meta']['TYPO3_version']) < 6000000
2141 ) {
2142 $this->legacyImport = true;
2143 $this->initializeLegacyImportFolder();
2144 }
2145 }
2146
2147 /**
2148 * Fix charset of import memory if different from system charset
2149 *
2150 * @return void
2151 * @see loadInit()
2152 */
2153 public function fixCharsets()
2154 {
2155 $importCharset = $this->dat['header']['charset'];
2156 if ($importCharset) {
2157 if ($importCharset !== 'utf-8') {
2158 /** @var CharsetConverter $charsetConverter */
2159 $charsetConverter = GeneralUtility::makeInstance(CharsetConverter::class);
2160 $this->error('CHARSET: Converting charset of input file (' . $importCharset . ') to the system charset (utf-8)');
2161 // Convert meta data:
2162 if (is_array($this->dat['header']['meta'])) {
2163 $charsetConverter->convArray($this->dat['header']['meta'], $importCharset, 'utf-8');
2164 }
2165 // Convert record headers:
2166 if (is_array($this->dat['header']['records'])) {
2167 $charsetConverter->convArray($this->dat['header']['records'], $importCharset, 'utf-8');
2168 }
2169 // Convert records themselves:
2170 if (is_array($this->dat['records'])) {
2171 $charsetConverter->convArray($this->dat['records'], $importCharset, 'utf-8');
2172 }
2173 }
2174 } else {
2175 $this->error('CHARSET: No charset found in import file!');
2176 }
2177 }
2178 }