[BUGFIX] impexp ignore file reference records with missing related files
[Packages/TYPO3.CMS.git] / typo3 / sysext / impexp / Classes / Import.php
1 <?php
2 namespace TYPO3\CMS\Impexp;
3
4 /*
5 * This file is part of the TYPO3 CMS project.
6 *
7 * It is free software; you can redistribute it and/or modify it under
8 * the terms of the GNU General Public License, either version 2
9 * of the License, or any later version.
10 *
11 * For the full copyright and license information, please read the
12 * LICENSE.txt file that was distributed with this source code.
13 *
14 * The TYPO3 project - inspiring people to share!
15 */
16
17 use TYPO3\CMS\Backend\Utility\BackendUtility;
18 use TYPO3\CMS\Core\DataHandling\DataHandler;
19 use TYPO3\CMS\Core\Exception;
20 use TYPO3\CMS\Core\Resource\DuplicationBehavior;
21 use TYPO3\CMS\Core\Resource\File;
22 use TYPO3\CMS\Core\Resource\FileInterface;
23 use TYPO3\CMS\Core\Resource\ResourceFactory;
24 use TYPO3\CMS\Core\Resource\ResourceStorage;
25 use TYPO3\CMS\Core\Resource\StorageRepository;
26 use TYPO3\CMS\Core\Utility\ExtensionManagementUtility;
27 use TYPO3\CMS\Core\Utility\GeneralUtility;
28 use TYPO3\CMS\Core\Utility\MathUtility;
29 use TYPO3\CMS\Core\Utility\PathUtility;
30 use TYPO3\CMS\Core\Utility\StringUtility;
31 use TYPO3\CMS\Core\Utility\VersionNumberUtility;
32
33 /**
34 * T3D file Import library (TYPO3 Record Document)
35 */
36 class Import extends ImportExport
37 {
38 /**
39 * Used to register the forged UID values for imported records that we want
40 * to create with the same UIDs as in the import file. Admin-only feature.
41 *
42 * @var array
43 */
44 public $suggestedInsertUids = array();
45
46 /**
47 * Disable logging when importing
48 *
49 * @var bool
50 */
51 public $enableLogging = false;
52
53 /**
54 * Keys are [tablename]:[new NEWxxx ids (or when updating it is uids)]
55 * while values are arrays with table/uid of the original record it is based on.
56 * With the array keys the new ids can be looked up inside tcemain
57 *
58 * @var array
59 */
60 public $import_newId = array();
61
62 /**
63 * Page id map for page tree (import)
64 *
65 * @var array
66 */
67 public $import_newId_pids = array();
68
69 /**
70 * Internal data accumulation for writing records during import
71 *
72 * @var array
73 */
74 public $import_data = array();
75
76 /**
77 * Array of current registered storage objects
78 *
79 * @var ResourceStorage[]
80 */
81 protected $storageObjects = array();
82
83 /**
84 * Is set, if the import file has a TYPO3 version below 6.0
85 *
86 * @var bool
87 */
88 protected $legacyImport = false;
89
90 /**
91 * @var \TYPO3\CMS\Core\Resource\Folder
92 */
93 protected $legacyImportFolder = null;
94
95 /**
96 * Related to the default storage root
97 *
98 * @var string
99 */
100 protected $legacyImportTargetPath = '_imported/';
101
102 /**
103 * Table fields to migrate
104 *
105 * @var array
106 */
107 protected $legacyImportMigrationTables = array(
108 'tt_content' => array(
109 'image' => array(
110 'titleTexts' => 'titleText',
111 'description' => 'imagecaption',
112 'links' => 'image_link',
113 'alternativeTexts' => 'altText'
114 ),
115 'media' => array(
116 'description' => 'imagecaption',
117 )
118 ),
119 'pages' => array(
120 'media' => array()
121 ),
122 'pages_language_overlay' => array(
123 'media' => array()
124 )
125 );
126
127 /**
128 * Records to be migrated after all
129 * Multidimensional array [table][uid][field] = array([related sys_file_reference uids])
130 *
131 * @var array
132 */
133 protected $legacyImportMigrationRecords = array();
134
135 /**
136 * @var NULL|string
137 */
138 protected $filesPathForImport = null;
139
140 /**
141 * @var array
142 */
143 protected $unlinkFiles = array();
144
145 /**
146 * @var array
147 */
148 protected $alternativeFileName = array();
149
150 /**
151 * @var array
152 */
153 protected $alternativeFilePath = array();
154
155 /**
156 * @var array
157 */
158 protected $filePathMap = array();
159
160 /**************************
161 * Initialize
162 *************************/
163
164 /**
165 * Init the object
166 *
167 * @return void
168 */
169 public function init()
170 {
171 parent::init();
172 $this->mode = 'import';
173 }
174
175 /***********************
176 * Import
177 ***********************/
178
179 /**
180 * Initialize all settings for the import
181 *
182 * @return void
183 */
184 protected function initializeImport()
185 {
186 // Set this flag to indicate that an import is being/has been done.
187 $this->doesImport = 1;
188 // Initialize:
189 // These vars MUST last for the whole section not being cleared. They are used by the method setRelations() which are called at the end of the import session.
190 $this->import_mapId = array();
191 $this->import_newId = array();
192 $this->import_newId_pids = array();
193 // Temporary files stack initialized:
194 $this->unlinkFiles = array();
195 $this->alternativeFileName = array();
196 $this->alternativeFilePath = array();
197
198 $this->initializeStorageObjects();
199 }
200
201 /**
202 * Initialize the all present storage objects
203 *
204 * @return void
205 */
206 protected function initializeStorageObjects()
207 {
208 /** @var $storageRepository StorageRepository */
209 $storageRepository = GeneralUtility::makeInstance(StorageRepository::class);
210 $this->storageObjects = $storageRepository->findAll();
211 }
212
213 /**
214 * Imports the internal data array to $pid.
215 *
216 * @param int $pid Page ID in which to import the content
217 * @return void
218 */
219 public function importData($pid)
220 {
221 $this->initializeImport();
222
223 // Write sys_file_storages first
224 $this->writeSysFileStorageRecords();
225 // Write sys_file records and write the binary file data
226 $this->writeSysFileRecords();
227 // Write records, first pages, then the rest
228 // Fields with "hard" relations to database, files and flexform fields are kept empty during this run
229 $this->writeRecords_pages($pid);
230 $this->writeRecords_records($pid);
231 // Finally all the file and DB record references must be fixed. This is done after all records have supposedly been written to database:
232 // $this->import_mapId will indicate two things: 1) that a record WAS written to db and 2) that it has got a new id-number.
233 $this->setRelations();
234 // And when all DB relations are in place, we can fix file and DB relations in flexform fields (since data structures often depends on relations to a DS record):
235 $this->setFlexFormRelations();
236 // Unlink temporary files:
237 $this->unlinkTempFiles();
238 // Finally, traverse all records and process softreferences with substitution attributes.
239 $this->processSoftReferences();
240 // After all migrate records using sys_file_reference now
241 if ($this->legacyImport) {
242 $this->migrateLegacyImportRecords();
243 }
244 }
245
246 /**
247 * Imports the sys_file_storage records from internal data array.
248 *
249 * @return void
250 */
251 protected function writeSysFileStorageRecords()
252 {
253 if (!isset($this->dat['header']['records']['sys_file_storage'])) {
254 return;
255 }
256 $sysFileStorageUidsToBeResetToDefaultStorage = array();
257 foreach ($this->dat['header']['records']['sys_file_storage'] as $sysFileStorageUid => $_) {
258 $storageRecord = $this->dat['records']['sys_file_storage:' . $sysFileStorageUid]['data'];
259 // continue with Local, writable and online storage only
260 if ($storageRecord['driver'] === 'Local' && $storageRecord['is_writable'] && $storageRecord['is_online']) {
261 foreach ($this->storageObjects as $localStorage) {
262 if ($this->isEquivalentObjectStorage($localStorage, $storageRecord)) {
263 $this->import_mapId['sys_file_storage'][$sysFileStorageUid] = $localStorage->getUid();
264 break;
265 }
266 }
267
268 if (!isset($this->import_mapId['sys_file_storage'][$sysFileStorageUid])) {
269 // Local, writable and online storage. Is allowed to be used to later write files in.
270 // Does currently not exist so add the record.
271 $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);
272 }
273 } else {
274 // Storage with non Local drivers could be imported but must not be used to saves files in, because you
275 // could not be sure, that this is supported. The default storage will be used in this case.
276 // It could happen that non writable and non online storage will be created as dupes because you could not
277 // check the detailed configuration options at this point
278 $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);
279 $sysFileStorageUidsToBeResetToDefaultStorage[] = $sysFileStorageUid;
280 }
281 }
282
283 // Importing the added ones
284 $tce = $this->getNewTCE();
285 // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.
286 $tce->reverseOrder = 1;
287 $tce->isImporting = true;
288 $tce->start($this->import_data, array());
289 $tce->process_datamap();
290 $this->addToMapId($tce->substNEWwithIDs);
291
292 $defaultStorageUid = null;
293 // get default storage
294 $defaultStorage = ResourceFactory::getInstance()->getDefaultStorage();
295 if ($defaultStorage !== null) {
296 $defaultStorageUid = $defaultStorage->getUid();
297 }
298 foreach ($sysFileStorageUidsToBeResetToDefaultStorage as $sysFileStorageUidToBeResetToDefaultStorage) {
299 $this->import_mapId['sys_file_storage'][$sysFileStorageUidToBeResetToDefaultStorage] = $defaultStorageUid;
300 }
301
302 // unset the sys_file_storage records to prevent an import in writeRecords_records
303 unset($this->dat['header']['records']['sys_file_storage']);
304 }
305
306 /**
307 * Determines whether the passed storage object and record (sys_file_storage) can be
308 * seen as equivalent during import.
309 *
310 * @param ResourceStorage $storageObject The storage object which should get compared
311 * @param array $storageRecord The storage record which should get compared
312 * @return bool Returns TRUE when both object storages can be seen as equivalent
313 */
314 protected function isEquivalentObjectStorage(ResourceStorage $storageObject, array $storageRecord)
315 {
316 // compare the properties: driver, writable and online
317 if ($storageObject->getDriverType() === $storageRecord['driver']
318 && (bool)$storageObject->isWritable() === (bool)$storageRecord['is_writable']
319 && (bool)$storageObject->isOnline() === (bool)$storageRecord['is_online']
320 ) {
321 $storageRecordConfiguration = ResourceFactory::getInstance()->convertFlexFormDataToConfigurationArray($storageRecord['configuration']);
322 $storageObjectConfiguration = $storageObject->getConfiguration();
323 // compare the properties: pathType and basePath
324 if ($storageRecordConfiguration['pathType'] === $storageObjectConfiguration['pathType']
325 && $storageRecordConfiguration['basePath'] === $storageObjectConfiguration['basePath']
326 ) {
327 return true;
328 }
329 }
330 return false;
331 }
332
333 /**
334 * Checks any prerequisites necessary to get fullfilled before import
335 *
336 * @return array Messages explaining issues which need to get resolved before import
337 */
338 public function checkImportPrerequisites()
339 {
340 $messages = array();
341
342 // Check #1: Extension dependencies
343 $extKeysToInstall = array();
344 foreach ($this->dat['header']['extensionDependencies'] as $extKey) {
345 if (!ExtensionManagementUtility::isLoaded($extKey)) {
346 $extKeysToInstall[] = $extKey;
347 }
348 }
349 if (!empty($extKeysToInstall)) {
350 $messages['missingExtensions'] = 'Before you can install this T3D file you need to install the extensions "'
351 . implode('", "', $extKeysToInstall) . '".';
352 }
353
354 // Check #2: If the path for every local storage object exists.
355 // Else files can't get moved into a newly imported storage.
356 if (!empty($this->dat['header']['records']['sys_file_storage'])) {
357 foreach ($this->dat['header']['records']['sys_file_storage'] as $sysFileStorageUid => $_) {
358 $storageRecord = $this->dat['records']['sys_file_storage:' . $sysFileStorageUid]['data'];
359 // continue with Local, writable and online storage only
360 if ($storageRecord['driver'] === 'Local'
361 && $storageRecord['is_writable']
362 && $storageRecord['is_online']
363 ) {
364 foreach ($this->storageObjects as $localStorage) {
365 if ($this->isEquivalentObjectStorage($localStorage, $storageRecord)) {
366 // There is already an existing storage
367 break;
368 }
369
370 // The storage from the import does not have an equivalent storage
371 // in the current instance (same driver, same path, etc.). Before
372 // the storage record can get inserted later on take care the path
373 // it points to really exists and is accessible.
374 $storageRecordUid = $storageRecord['uid'];
375 // Unset the storage record UID when trying to create the storage object
376 // as the record does not already exist in DB. The constructor of the
377 // storage object will check whether the target folder exists and set the
378 // isOnline flag depending on the outcome.
379 $storageRecord['uid'] = 0;
380 $resourceStorage = ResourceFactory::getInstance()->createStorageObject($storageRecord);
381 if (!$resourceStorage->isOnline()) {
382 $configuration = $resourceStorage->getConfiguration();
383 $messages['resourceStorageFolderMissing_' . $storageRecordUid] =
384 'The resource storage "'
385 . $resourceStorage->getName()
386 . $configuration['basePath']
387 . '" does not exist. Please create the directory prior to starting the import!';
388 }
389 }
390 }
391 }
392 }
393
394 return $messages;
395 }
396
397 /**
398 * Imports the sys_file records and the binary files data from internal data array.
399 *
400 * @return void
401 */
402 protected function writeSysFileRecords()
403 {
404 if (!isset($this->dat['header']['records']['sys_file'])) {
405 return;
406 }
407 $this->addGeneralErrorsByTable('sys_file');
408
409 // fetch fresh storage records from database
410 $storageRecords = $this->fetchStorageRecords();
411
412 $defaultStorage = ResourceFactory::getInstance()->getDefaultStorage();
413
414 $sanitizedFolderMappings = array();
415
416 foreach ($this->dat['header']['records']['sys_file'] as $sysFileUid => $_) {
417 $fileRecord = $this->dat['records']['sys_file:' . $sysFileUid]['data'];
418
419 $temporaryFile = null;
420 // check if there is the right file already in the local folder
421 if ($this->filesPathForImport !== null) {
422 if (is_file($this->filesPathForImport . '/' . $fileRecord['sha1']) && sha1_file($this->filesPathForImport . '/' . $fileRecord['sha1']) === $fileRecord['sha1']) {
423 $temporaryFile = $this->filesPathForImport . '/' . $fileRecord['sha1'];
424 }
425 }
426
427 // save file to disk
428 if ($temporaryFile === null) {
429 $fileId = md5($fileRecord['storage'] . ':' . $fileRecord['identifier_hash']);
430 $temporaryFile = $this->writeTemporaryFileFromData($fileId);
431 if ($temporaryFile === null) {
432 // error on writing the file. Error message was already added
433 continue;
434 }
435 }
436
437 $originalStorageUid = $fileRecord['storage'];
438 $useStorageFromStorageRecords = false;
439
440 // replace storage id, if an alternative one was registered
441 if (isset($this->import_mapId['sys_file_storage'][$fileRecord['storage']])) {
442 $fileRecord['storage'] = $this->import_mapId['sys_file_storage'][$fileRecord['storage']];
443 $useStorageFromStorageRecords = true;
444 }
445
446 if (empty($fileRecord['storage']) && !$this->isFallbackStorage($fileRecord['storage'])) {
447 // no storage for the file is defined, mostly because of a missing default storage.
448 $this->error('Error: No storage for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $originalStorageUid . '"');
449 continue;
450 }
451
452 // using a storage from the local storage is only allowed, if the uid is present in the
453 // mapping. Only in this case we could be sure, that it's a local, online and writable storage.
454 if ($useStorageFromStorageRecords && isset($storageRecords[$fileRecord['storage']])) {
455 /** @var $storage \TYPO3\CMS\Core\Resource\ResourceStorage */
456 $storage = ResourceFactory::getInstance()->getStorageObject($fileRecord['storage'], $storageRecords[$fileRecord['storage']]);
457 } elseif ($this->isFallbackStorage($fileRecord['storage'])) {
458 $storage = ResourceFactory::getInstance()->getStorageObject(0);
459 } elseif ($defaultStorage !== null) {
460 $storage = $defaultStorage;
461 } else {
462 $this->error('Error: No storage available for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
463 continue;
464 }
465
466 $newFile = null;
467
468 // check, if there is an identical file
469 try {
470 if ($storage->hasFile($fileRecord['identifier'])) {
471 $file = $storage->getFile($fileRecord['identifier']);
472 if ($file->getSha1() === $fileRecord['sha1']) {
473 $newFile = $file;
474 }
475 }
476 } catch (Exception $e) {
477 }
478
479 if ($newFile === null) {
480 $folderName = PathUtility::dirname(ltrim($fileRecord['identifier'], '/'));
481 if (in_array($folderName, $sanitizedFolderMappings)) {
482 $folderName = $sanitizedFolderMappings[$folderName];
483 }
484 if (!$storage->hasFolder($folderName)) {
485 try {
486 $importFolder = $storage->createFolder($folderName);
487 if ($importFolder->getIdentifier() !== $folderName && !in_array($folderName, $sanitizedFolderMappings)) {
488 $sanitizedFolderMappings[$folderName] = $importFolder->getIdentifier();
489 }
490 } catch (Exception $e) {
491 $this->error('Error: Folder "' . $folderName . '" could not be created for file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
492 continue;
493 }
494 } else {
495 $importFolder = $storage->getFolder($folderName);
496 }
497
498 try {
499 /** @var $newFile File */
500 $newFile = $storage->addFile($temporaryFile, $importFolder, $fileRecord['name']);
501 } catch (Exception $e) {
502 $this->error('Error: File could not be added to the storage: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
503 continue;
504 }
505
506 if ($newFile->getSha1() !== $fileRecord['sha1']) {
507 $this->error('Error: The hash of the written file is not identical to the import data! File could be corrupted! File: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
508 }
509 }
510
511 // save the new uid in the import id map
512 $this->import_mapId['sys_file'][$fileRecord['uid']] = $newFile->getUid();
513 $this->fixUidLocalInSysFileReferenceRecords($fileRecord['uid'], $newFile->getUid());
514 }
515
516 // unset the sys_file records to prevent an import in writeRecords_records
517 unset($this->dat['header']['records']['sys_file']);
518 // remove all sys_file_reference records that point to file records which are unknown
519 // in the system to prevent exceptions
520 $this->removeSysFileReferenceRecordsFromImportDataWithRelationToMissingFile();
521 }
522
523 /**
524 * Removes all sys_file_reference records from the import data array that are pointing to sys_file records which
525 * are missing not in the import data to prevent exceptions on checking the related file started by the Datahandler.
526 *
527 * @return void
528 */
529 protected function removeSysFileReferenceRecordsFromImportDataWithRelationToMissingFile()
530 {
531 if (!isset($this->dat['header']['records']['sys_file_reference'])) {
532 return;
533 }
534
535 foreach ($this->dat['header']['records']['sys_file_reference'] as $sysFileReferenceUid => $_) {
536 $fileReferenceRecord = $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'];
537 if (!in_array($fileReferenceRecord['uid_local'], $this->import_mapId['sys_file'])) {
538 unset($this->dat['header']['records']['sys_file_reference'][$sysFileReferenceUid]);
539 unset($this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]);
540 $this->error('Error: sys_file_reference record ' . (int)$sysFileReferenceUid
541 . ' with relation to sys_file record ' . (int)$fileReferenceRecord['uid_local']
542 . ', which is not part of the import data, was not imported.'
543 );
544 }
545 }
546 }
547
548 /**
549 * Checks if the $storageId is the id of the fallback storage
550 *
551 * @param int|string $storageId
552 * @return bool
553 */
554 protected function isFallbackStorage($storageId)
555 {
556 return $storageId === 0 || $storageId === '0';
557 }
558
559 /**
560 * Normally the importer works like the following:
561 * Step 1: import the records with cleared field values of relation fields (see addSingle())
562 * Step 2: update the records with the right relation ids (see setRelations())
563 *
564 * In step 2 the saving fields of type "relation to sys_file_reference" checks the related sys_file_reference
565 * record (created in step 1) with the FileExtensionFilter for matching file extensions of the related file.
566 * To make this work correct, the uid_local of sys_file_reference records has to be not empty AND has to
567 * relate to the correct (imported) sys_file record uid!!!
568 *
569 * This is fixed here.
570 *
571 * @param int $oldFileUid
572 * @param int $newFileUid
573 * @return void
574 */
575 protected function fixUidLocalInSysFileReferenceRecords($oldFileUid, $newFileUid)
576 {
577 if (!isset($this->dat['header']['records']['sys_file_reference'])) {
578 return;
579 }
580
581 foreach ($this->dat['header']['records']['sys_file_reference'] as $sysFileReferenceUid => $_) {
582 $fileReferenceRecord = $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'];
583 if ($fileReferenceRecord['uid_local'] == $oldFileUid) {
584 $fileReferenceRecord['uid_local'] = $newFileUid;
585 $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'] = $fileReferenceRecord;
586 }
587 }
588 }
589
590 /**
591 * Initializes the folder for legacy imports as subfolder of backend users default upload folder
592 *
593 * @return void
594 */
595 protected function initializeLegacyImportFolder()
596 {
597 /** @var \TYPO3\CMS\Core\Resource\Folder $folder */
598 $folder = $this->getBackendUser()->getDefaultUploadFolder();
599 if ($folder === false) {
600 $this->error('Error: the backend users default upload folder is missing! No files will be imported!');
601 }
602 if (!$folder->hasFolder($this->legacyImportTargetPath)) {
603 try {
604 $this->legacyImportFolder = $folder->createFolder($this->legacyImportTargetPath);
605 } catch (Exception $e) {
606 $this->error('Error: the import folder in the default upload folder could not be created! No files will be imported!');
607 }
608 } else {
609 $this->legacyImportFolder = $folder->getSubfolder($this->legacyImportTargetPath);
610 }
611 }
612
613 /**
614 * Fetched fresh storage records from database because the new imported
615 * ones are not in cached data of the StorageRepository
616 *
617 * @return bool|array
618 */
619 protected function fetchStorageRecords()
620 {
621 $whereClause = BackendUtility::BEenableFields('sys_file_storage');
622 $whereClause .= BackendUtility::deleteClause('sys_file_storage');
623
624 $rows = $this->getDatabaseConnection()->exec_SELECTgetRows(
625 '*',
626 'sys_file_storage',
627 '1=1' . $whereClause,
628 '',
629 '',
630 '',
631 'uid'
632 );
633
634 return $rows;
635 }
636
637 /**
638 * Writes the file from import array to temp dir and returns the filename of it.
639 *
640 * @param string $fileId
641 * @param string $dataKey
642 * @return string Absolute filename of the temporary filename of the file
643 */
644 protected function writeTemporaryFileFromData($fileId, $dataKey = 'files_fal')
645 {
646 $temporaryFilePath = null;
647 if (is_array($this->dat[$dataKey][$fileId])) {
648 $temporaryFilePathInternal = GeneralUtility::tempnam('import_temp_');
649 GeneralUtility::writeFile($temporaryFilePathInternal, $this->dat[$dataKey][$fileId]['content']);
650 clearstatcache();
651 if (@is_file($temporaryFilePathInternal)) {
652 $this->unlinkFiles[] = $temporaryFilePathInternal;
653 if (filesize($temporaryFilePathInternal) == $this->dat[$dataKey][$fileId]['filesize']) {
654 $temporaryFilePath = $temporaryFilePathInternal;
655 } else {
656 $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' had a size (' . filesize($temporaryFilePathInternal) . ') different from the original (' . $this->dat[$dataKey][$fileId]['filesize'] . ')');
657 }
658 } else {
659 $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' was not written as it should have been!');
660 }
661 } else {
662 $this->error('Error: No file found for ID ' . $fileId);
663 }
664 return $temporaryFilePath;
665 }
666
667 /**
668 * Writing pagetree/pages to database:
669 *
670 * @param int $pid PID in which to import. If the operation is an update operation, the root of the page tree inside will be moved to this PID unless it is the same as the root page from the import
671 * @return void
672 * @see writeRecords_records()
673 */
674 public function writeRecords_pages($pid)
675 {
676 // First, write page structure if any:
677 if (is_array($this->dat['header']['records']['pages'])) {
678 $this->addGeneralErrorsByTable('pages');
679 // $pageRecords is a copy of the pages array in the imported file. Records here are unset one by one when the addSingle function is called.
680 $pageRecords = $this->dat['header']['records']['pages'];
681 $this->import_data = array();
682 // First add page tree if any
683 if (is_array($this->dat['header']['pagetree'])) {
684 $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);
685 foreach ($pagesFromTree as $uid) {
686 $thisRec = $this->dat['header']['records']['pages'][$uid];
687 // PID: Set the main $pid, unless a NEW-id is found
688 $setPid = isset($this->import_newId_pids[$thisRec['pid']]) ? $this->import_newId_pids[$thisRec['pid']] : $pid;
689 $this->addSingle('pages', $uid, $setPid);
690 unset($pageRecords[$uid]);
691 }
692 }
693 // Then add all remaining pages not in tree on root level:
694 if (!empty($pageRecords)) {
695 $remainingPageUids = array_keys($pageRecords);
696 foreach ($remainingPageUids as $pUid) {
697 $this->addSingle('pages', $pUid, $pid);
698 }
699 }
700 // Now write to database:
701 $tce = $this->getNewTCE();
702 $tce->isImporting = true;
703 $this->callHook('before_writeRecordsPages', array(
704 'tce' => &$tce,
705 'data' => &$this->import_data
706 ));
707 $tce->suggestedInsertUids = $this->suggestedInsertUids;
708 $tce->start($this->import_data, array());
709 $tce->process_datamap();
710 $this->callHook('after_writeRecordsPages', array(
711 'tce' => &$tce
712 ));
713 // post-processing: Registering new ids (end all tcemain sessions with this)
714 $this->addToMapId($tce->substNEWwithIDs);
715 // In case of an update, order pages from the page tree correctly:
716 if ($this->update && is_array($this->dat['header']['pagetree'])) {
717 $this->writeRecords_pages_order();
718 }
719 }
720 }
721
722 /**
723 * Organize all updated pages in page tree so they are related like in the import file
724 * Only used for updates and when $this->dat['header']['pagetree'] is an array.
725 *
726 * @return void
727 * @access private
728 * @see writeRecords_pages(), writeRecords_records_order()
729 */
730 public function writeRecords_pages_order()
731 {
732 $cmd_data = array();
733 // Get uid-pid relations and traverse them in order to map to possible new IDs
734 $pidsFromTree = $this->flatInversePageTree_pid($this->dat['header']['pagetree']);
735 foreach ($pidsFromTree as $origPid => $newPid) {
736 if ($newPid >= 0 && $this->dontIgnorePid('pages', $origPid)) {
737 // If the page had a new id (because it was created) use that instead!
738 if (substr($this->import_newId_pids[$origPid], 0, 3) === 'NEW') {
739 if ($this->import_mapId['pages'][$origPid]) {
740 $mappedPid = $this->import_mapId['pages'][$origPid];
741 $cmd_data['pages'][$mappedPid]['move'] = $newPid;
742 }
743 } else {
744 $cmd_data['pages'][$origPid]['move'] = $newPid;
745 }
746 }
747 }
748 // Execute the move commands if any:
749 if (!empty($cmd_data)) {
750 $tce = $this->getNewTCE();
751 $this->callHook('before_writeRecordsPagesOrder', array(
752 'tce' => &$tce,
753 'data' => &$cmd_data
754 ));
755 $tce->start(array(), $cmd_data);
756 $tce->process_cmdmap();
757 $this->callHook('after_writeRecordsPagesOrder', array(
758 'tce' => &$tce
759 ));
760 }
761 }
762
763 /**
764 * Recursively flattening the idH array, setting PIDs as values
765 *
766 * @param array $idH Page uid hierarchy
767 * @param array $a Accumulation array of pages (internal, don't set from outside)
768 * @param int $pid PID value (internal)
769 * @return array Array with uid-pid pairs for all pages in the page tree.
770 * @see ImportExport::flatInversePageTree()
771 */
772 public function flatInversePageTree_pid($idH, $a = array(), $pid = -1)
773 {
774 if (is_array($idH)) {
775 $idH = array_reverse($idH);
776 foreach ($idH as $v) {
777 $a[$v['uid']] = $pid;
778 if (is_array($v['subrow'])) {
779 $a = $this->flatInversePageTree_pid($v['subrow'], $a, $v['uid']);
780 }
781 }
782 }
783 return $a;
784 }
785
786 /**
787 * Write all database records except pages (writtein in writeRecords_pages())
788 *
789 * @param int $pid Page id in which to import
790 * @return void
791 * @see writeRecords_pages()
792 */
793 public function writeRecords_records($pid)
794 {
795 // Write the rest of the records
796 $this->import_data = array();
797 if (is_array($this->dat['header']['records'])) {
798 foreach ($this->dat['header']['records'] as $table => $recs) {
799 $this->addGeneralErrorsByTable($table);
800 if ($table != 'pages') {
801 foreach ($recs as $uid => $thisRec) {
802 // PID: Set the main $pid, unless a NEW-id is found
803 $setPid = isset($this->import_mapId['pages'][$thisRec['pid']])
804 ? (int)$this->import_mapId['pages'][$thisRec['pid']]
805 : (int)$pid;
806 if (is_array($GLOBALS['TCA'][$table]) && isset($GLOBALS['TCA'][$table]['ctrl']['rootLevel'])) {
807 $rootLevelSetting = (int)$GLOBALS['TCA'][$table]['ctrl']['rootLevel'];
808 if ($rootLevelSetting === 1) {
809 $setPid = 0;
810 } elseif ($rootLevelSetting === 0 && $setPid === 0) {
811 $this->error('Error: Record type ' . $table . ' is not allowed on pid 0');
812 continue;
813 }
814 }
815 // Add record:
816 $this->addSingle($table, $uid, $setPid);
817 }
818 }
819 }
820 } else {
821 $this->error('Error: No records defined in internal data array.');
822 }
823 // Now write to database:
824 $tce = $this->getNewTCE();
825 $this->callHook('before_writeRecordsRecords', array(
826 'tce' => &$tce,
827 'data' => &$this->import_data
828 ));
829 $tce->suggestedInsertUids = $this->suggestedInsertUids;
830 // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.
831 $tce->reverseOrder = 1;
832 $tce->isImporting = true;
833 $tce->start($this->import_data, array());
834 $tce->process_datamap();
835 $this->callHook('after_writeRecordsRecords', array(
836 'tce' => &$tce
837 ));
838 // post-processing: Removing files and registering new ids (end all tcemain sessions with this)
839 $this->addToMapId($tce->substNEWwithIDs);
840 // In case of an update, order pages from the page tree correctly:
841 if ($this->update) {
842 $this->writeRecords_records_order($pid);
843 }
844 }
845
846 /**
847 * Organize all updated record to their new positions.
848 * Only used for updates
849 *
850 * @param int $mainPid Main PID into which we import.
851 * @return void
852 * @access private
853 * @see writeRecords_records(), writeRecords_pages_order()
854 */
855 public function writeRecords_records_order($mainPid)
856 {
857 $cmd_data = array();
858 if (is_array($this->dat['header']['pagetree'])) {
859 $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);
860 } else {
861 $pagesFromTree = array();
862 }
863 if (is_array($this->dat['header']['pid_lookup'])) {
864 foreach ($this->dat['header']['pid_lookup'] as $pid => $recList) {
865 $newPid = isset($this->import_mapId['pages'][$pid]) ? $this->import_mapId['pages'][$pid] : $mainPid;
866 if (MathUtility::canBeInterpretedAsInteger($newPid)) {
867 foreach ($recList as $tableName => $uidList) {
868 // If $mainPid===$newPid then we are on root level and we can consider to move pages as well!
869 // (they will not be in the page tree!)
870 if (($tableName != 'pages' || !$pagesFromTree[$pid]) && is_array($uidList)) {
871 $uidList = array_reverse(array_keys($uidList));
872 foreach ($uidList as $uid) {
873 if ($this->dontIgnorePid($tableName, $uid)) {
874 $cmd_data[$tableName][$uid]['move'] = $newPid;
875 } else {
876 }
877 }
878 }
879 }
880 }
881 }
882 }
883 // Execute the move commands if any:
884 if (!empty($cmd_data)) {
885 $tce = $this->getNewTCE();
886 $this->callHook('before_writeRecordsRecordsOrder', array(
887 'tce' => &$tce,
888 'data' => &$cmd_data
889 ));
890 $tce->start(array(), $cmd_data);
891 $tce->process_cmdmap();
892 $this->callHook('after_writeRecordsRecordsOrder', array(
893 'tce' => &$tce
894 ));
895 }
896 }
897
898 /**
899 * Adds a single record to the $importData array. Also copies files to tempfolder.
900 * However all File/DB-references and flexform field contents are set to blank for now!
901 * That is done with setRelations() later
902 *
903 * @param string $table Table name (from import memory)
904 * @param int $uid Record UID (from import memory)
905 * @param int $pid Page id
906 * @return void
907 * @see writeRecords()
908 */
909 public function addSingle($table, $uid, $pid)
910 {
911 if ($this->import_mode[$table . ':' . $uid] === 'exclude') {
912 return;
913 }
914 $record = $this->dat['records'][$table . ':' . $uid]['data'];
915 if (is_array($record)) {
916 if ($this->update && $this->doesRecordExist($table, $uid) && $this->import_mode[$table . ':' . $uid] !== 'as_new') {
917 $ID = $uid;
918 } elseif ($table === 'sys_file_metadata' && $record['sys_language_uid'] == '0' && $this->import_mapId['sys_file'][$record['file']]) {
919 // on adding sys_file records the belonging sys_file_metadata record was also created
920 // if there is one the record need to be overwritten instead of creating a new one.
921 $recordInDatabase = $this->getDatabaseConnection()->exec_SELECTgetSingleRow(
922 'uid',
923 'sys_file_metadata',
924 'file = ' . $this->import_mapId['sys_file'][$record['file']] . ' AND sys_language_uid = 0 AND pid = 0'
925 );
926 // if no record could be found, $this->import_mapId['sys_file'][$record['file']] is pointing
927 // to a file, that was already there, thus a new metadata record should be created
928 if (is_array($recordInDatabase)) {
929 $this->import_mapId['sys_file_metadata'][$record['uid']] = $recordInDatabase['uid'];
930 $ID = $recordInDatabase['uid'];
931 } else {
932 $ID = StringUtility::getUniqueId('NEW');
933 }
934 } else {
935 $ID = StringUtility::getUniqueId('NEW');
936 }
937 $this->import_newId[$table . ':' . $ID] = array('table' => $table, 'uid' => $uid);
938 if ($table == 'pages') {
939 $this->import_newId_pids[$uid] = $ID;
940 }
941 // Set main record data:
942 $this->import_data[$table][$ID] = $record;
943 $this->import_data[$table][$ID]['tx_impexp_origuid'] = $this->import_data[$table][$ID]['uid'];
944 // Reset permission data:
945 if ($table === 'pages') {
946 // Have to reset the user/group IDs so pages are owned by importing user. Otherwise strange things may happen for non-admins!
947 unset($this->import_data[$table][$ID]['perms_userid']);
948 unset($this->import_data[$table][$ID]['perms_groupid']);
949 }
950 // PID and UID:
951 unset($this->import_data[$table][$ID]['uid']);
952 // Updates:
953 if (MathUtility::canBeInterpretedAsInteger($ID)) {
954 unset($this->import_data[$table][$ID]['pid']);
955 } else {
956 // Inserts:
957 $this->import_data[$table][$ID]['pid'] = $pid;
958 if (($this->import_mode[$table . ':' . $uid] === 'force_uid' && $this->update || $this->force_all_UIDS) && $this->getBackendUser()->isAdmin()) {
959 $this->import_data[$table][$ID]['uid'] = $uid;
960 $this->suggestedInsertUids[$table . ':' . $uid] = 'DELETE';
961 }
962 }
963 // Setting db/file blank:
964 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
965 switch ((string)$config['type']) {
966 case 'db':
967
968 case 'file':
969 // Fixed later in ->setRelations() [because we need to know ALL newly created IDs before we can map relations!]
970 // In the meantime we set NO values for relations.
971 //
972 // BUT for field uid_local of table sys_file_reference the relation MUST not be cleared here,
973 // because the value is already the uid of the right imported sys_file record.
974 // @see fixUidLocalInSysFileReferenceRecords()
975 // If it's empty or a uid to another record the FileExtensionFilter will throw an exception or
976 // delete the reference record if the file extension of the related record doesn't match.
977 if ($table !== 'sys_file_reference' && $field !== 'uid_local') {
978 $this->import_data[$table][$ID][$field] = '';
979 }
980 break;
981 case 'flex':
982 // Fixed later in setFlexFormRelations()
983 // In the meantime we set NO value for flexforms - this is mainly because file references
984 // inside will not be processed properly; In fact references will point to no file
985 // or existing files (in which case there will be double-references which is a big problem of course!)
986 $this->import_data[$table][$ID][$field] = '';
987 break;
988 }
989 }
990 } elseif ($table . ':' . $uid != 'pages:0') {
991 // On root level we don't want this error message.
992 $this->error('Error: no record was found in data array!');
993 }
994 }
995
996 /**
997 * Registers the substNEWids in memory.
998 *
999 * @param array $substNEWwithIDs From tcemain to be merged into internal mapping variable in this object
1000 * @return void
1001 * @see writeRecords()
1002 */
1003 public function addToMapId($substNEWwithIDs)
1004 {
1005 foreach ($this->import_data as $table => $recs) {
1006 foreach ($recs as $id => $value) {
1007 $old_uid = $this->import_newId[$table . ':' . $id]['uid'];
1008 if (isset($substNEWwithIDs[$id])) {
1009 $this->import_mapId[$table][$old_uid] = $substNEWwithIDs[$id];
1010 } elseif ($this->update) {
1011 // Map same ID to same ID....
1012 $this->import_mapId[$table][$old_uid] = $id;
1013 } else {
1014 // if $this->import_mapId contains already the right mapping, skip the error msg.
1015 // See special handling of sys_file_metadata in addSingle() => nothing to do
1016 if (!($table === 'sys_file_metadata' && isset($this->import_mapId[$table][$old_uid]) && $this->import_mapId[$table][$old_uid] == $id)) {
1017 $this->error('Possible error: ' . $table . ':' . $old_uid . ' had no new id assigned to it. This indicates that the record was not added to database during import. Please check changelog!');
1018 }
1019 }
1020 }
1021 }
1022 }
1023
1024 /**
1025 * Returns a new $TCE object
1026 *
1027 * @return DataHandler $TCE object
1028 */
1029 public function getNewTCE()
1030 {
1031 $tce = GeneralUtility::makeInstance(DataHandler::class);
1032 $tce->dontProcessTransformations = 1;
1033 $tce->enableLogging = $this->enableLogging;
1034 $tce->alternativeFileName = $this->alternativeFileName;
1035 $tce->alternativeFilePath = $this->alternativeFilePath;
1036 return $tce;
1037 }
1038
1039 /**
1040 * Cleaning up all the temporary files stored in typo3temp/ folder
1041 *
1042 * @return void
1043 */
1044 public function unlinkTempFiles()
1045 {
1046 foreach ($this->unlinkFiles as $fileName) {
1047 if (GeneralUtility::isFirstPartOfStr($fileName, PATH_site . 'typo3temp/')) {
1048 GeneralUtility::unlink_tempfile($fileName);
1049 clearstatcache();
1050 if (is_file($fileName)) {
1051 $this->error('Error: ' . $fileName . ' was NOT unlinked as it should have been!');
1052 }
1053 } else {
1054 $this->error('Error: ' . $fileName . ' was not in temp-path. Not removed!');
1055 }
1056 }
1057 $this->unlinkFiles = array();
1058 }
1059
1060 /***************************
1061 * Import / Relations setting
1062 ***************************/
1063
1064 /**
1065 * At the end of the import process all file and DB relations should be set properly (that is relations
1066 * to imported records are all re-created so imported records are correctly related again)
1067 * Relations in flexform fields are processed in setFlexFormRelations() after this function
1068 *
1069 * @return void
1070 * @see setFlexFormRelations()
1071 */
1072 public function setRelations()
1073 {
1074 $updateData = array();
1075 // import_newId contains a register of all records that was in the import memorys "records" key
1076 foreach ($this->import_newId as $nId => $dat) {
1077 $table = $dat['table'];
1078 $uid = $dat['uid'];
1079 // original UID - NOT the new one!
1080 // If the record has been written and received a new id, then proceed:
1081 if (is_array($this->import_mapId[$table]) && isset($this->import_mapId[$table][$uid])) {
1082 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
1083 if (is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {
1084 $thisNewPageUid = 0;
1085 if ($this->legacyImport) {
1086 if ($table != 'pages') {
1087 $oldPid = $this->dat['records'][$table . ':' . $uid]['data']['pid'];
1088 $thisNewPageUid = BackendUtility::wsMapId($table, $this->import_mapId['pages'][$oldPid]);
1089 } else {
1090 $thisNewPageUid = $thisNewUid;
1091 }
1092 }
1093 // Traverse relation fields of each record
1094 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
1095 // uid_local of sys_file_reference needs no update because the correct reference uid was already written
1096 // @see ImportExport::fixUidLocalInSysFileReferenceRecords()
1097 if ($table === 'sys_file_reference' && $field === 'uid_local') {
1098 continue;
1099 }
1100 switch ((string)$config['type']) {
1101 case 'db':
1102 if (is_array($config['itemArray']) && !empty($config['itemArray'])) {
1103 $itemConfig = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
1104 $valArray = $this->setRelations_db($config['itemArray'], $itemConfig);
1105 $updateData[$table][$thisNewUid][$field] = implode(',', $valArray);
1106 }
1107 break;
1108 case 'file':
1109 if (is_array($config['newValueFiles']) && !empty($config['newValueFiles'])) {
1110 $valArr = array();
1111 foreach ($config['newValueFiles'] as $fI) {
1112 $valArr[] = $this->import_addFileNameToBeCopied($fI);
1113 }
1114 if ($this->legacyImport && $this->legacyImportFolder === null && isset($this->legacyImportMigrationTables[$table][$field])) {
1115 // Do nothing - the legacy import folder is missing
1116 } elseif ($this->legacyImport && $this->legacyImportFolder !== null && isset($this->legacyImportMigrationTables[$table][$field])) {
1117 $refIds = array();
1118 foreach ($valArr as $tempFile) {
1119 $fileName = $this->alternativeFileName[$tempFile];
1120 $fileObject = null;
1121
1122 try {
1123 // check, if there is alreay the same file in the folder
1124 if ($this->legacyImportFolder->hasFile($fileName)) {
1125 $fileStorage = $this->legacyImportFolder->getStorage();
1126 $file = $fileStorage->getFile($this->legacyImportFolder->getIdentifier() . $fileName);
1127 if ($file->getSha1() === sha1_file($tempFile)) {
1128 $fileObject = $file;
1129 }
1130 }
1131 } catch (Exception $e) {
1132 }
1133
1134 if ($fileObject === null) {
1135 try {
1136 $fileObject = $this->legacyImportFolder->addFile($tempFile, $fileName, DuplicationBehavior::RENAME);
1137 } catch (Exception $e) {
1138 $this->error('Error: no file could be added to the storage for file name' . $this->alternativeFileName[$tempFile]);
1139 }
1140 }
1141 if ($fileObject !== null) {
1142 $refId = StringUtility::getUniqueId('NEW');
1143 $refIds[] = $refId;
1144 $updateData['sys_file_reference'][$refId] = array(
1145 'uid_local' => $fileObject->getUid(),
1146 'uid_foreign' => $thisNewUid, // uid of your content record
1147 'tablenames' => $table,
1148 'fieldname' => $field,
1149 'pid' => $thisNewPageUid, // parent id of the parent page
1150 'table_local' => 'sys_file',
1151 );
1152 }
1153 }
1154 $updateData[$table][$thisNewUid][$field] = implode(',', $refIds);
1155 if (!empty($this->legacyImportMigrationTables[$table][$field])) {
1156 $this->legacyImportMigrationRecords[$table][$thisNewUid][$field] = $refIds;
1157 }
1158 } else {
1159 $updateData[$table][$thisNewUid][$field] = implode(',', $valArr);
1160 }
1161 }
1162 break;
1163 }
1164 }
1165 } else {
1166 $this->error('Error: no record was found in data array!');
1167 }
1168 } else {
1169 $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')');
1170 }
1171 }
1172 if (!empty($updateData)) {
1173 $tce = $this->getNewTCE();
1174 $tce->isImporting = true;
1175 $this->callHook('before_setRelation', array(
1176 'tce' => &$tce,
1177 'data' => &$updateData
1178 ));
1179 $tce->start($updateData, array());
1180 $tce->process_datamap();
1181 // Replace the temporary "NEW" ids with the final ones.
1182 foreach ($this->legacyImportMigrationRecords as $table => $records) {
1183 foreach ($records as $uid => $fields) {
1184 foreach ($fields as $field => $referenceIds) {
1185 foreach ($referenceIds as $key => $referenceId) {
1186 $this->legacyImportMigrationRecords[$table][$uid][$field][$key] = $tce->substNEWwithIDs[$referenceId];
1187 }
1188 }
1189 }
1190 }
1191 $this->callHook('after_setRelations', array(
1192 'tce' => &$tce
1193 ));
1194 }
1195 }
1196
1197 /**
1198 * Maps relations for database
1199 *
1200 * @param array $itemArray Array of item sets (table/uid) from a dbAnalysis object
1201 * @param array $itemConfig Array of TCA config of the field the relation to be set on
1202 * @return array Array with values [table]_[uid] or [uid] for field of type group / internal_type file_reference. These values have the regular tcemain-input group/select type which means they will automatically be processed into a uid-list or MM relations.
1203 */
1204 public function setRelations_db($itemArray, $itemConfig)
1205 {
1206 $valArray = array();
1207 foreach ($itemArray as $relDat) {
1208 if (is_array($this->import_mapId[$relDat['table']]) && isset($this->import_mapId[$relDat['table']][$relDat['id']])) {
1209 // Since non FAL file relation type group internal_type file_reference are handled as reference to
1210 // sys_file records Datahandler requires the value as uid of the the related sys_file record only
1211 if ($itemConfig['type'] === 'group' && $itemConfig['internal_type'] === 'file_reference') {
1212 $value = $this->import_mapId[$relDat['table']][$relDat['id']];
1213 } elseif ($itemConfig['type'] === 'input' && isset($itemConfig['wizards']['link'])) {
1214 // If an input field has a relation to a sys_file record this need to be converted back to
1215 // the public path. But use getPublicUrl here, because could normally only be a local file path.
1216 $fileUid = $this->import_mapId[$relDat['table']][$relDat['id']];
1217 // Fallback value
1218 $value = 'file:' . $fileUid;
1219 try {
1220 $file = ResourceFactory::getInstance()->retrieveFileOrFolderObject($fileUid);
1221 } catch (\Exception $e) {
1222 $file = null;
1223 }
1224 if ($file instanceof FileInterface) {
1225 $value = $file->getPublicUrl();
1226 }
1227 } else {
1228 $value = $relDat['table'] . '_' . $this->import_mapId[$relDat['table']][$relDat['id']];
1229 }
1230 $valArray[] = $value;
1231 } elseif ($this->isTableStatic($relDat['table']) || $this->isExcluded($relDat['table'], $relDat['id']) || $relDat['id'] < 0) {
1232 // Checking for less than zero because some select types could contain negative values,
1233 // eg. fe_groups (-1, -2) and sys_language (-1 = ALL languages). This must be handled on both export and import.
1234 $valArray[] = $relDat['table'] . '_' . $relDat['id'];
1235 } else {
1236 $this->error('Lost relation: ' . $relDat['table'] . ':' . $relDat['id']);
1237 }
1238 }
1239 return $valArray;
1240 }
1241
1242 /**
1243 * Writes the file from import array to temp dir and returns the filename of it.
1244 *
1245 * @param array $fI File information with three keys: "filename" = filename without path, "ID_absFile" = absolute filepath to the file (including the filename), "ID" = md5 hash of "ID_absFile
1246 * @return string|NULL Absolute filename of the temporary filename of the file. In ->alternativeFileName the original name is set.
1247 */
1248 public function import_addFileNameToBeCopied($fI)
1249 {
1250 if (is_array($this->dat['files'][$fI['ID']])) {
1251 $tmpFile = null;
1252 // check if there is the right file already in the local folder
1253 if ($this->filesPathForImport !== null) {
1254 if (is_file($this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5']) &&
1255 md5_file($this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5']) === $this->dat['files'][$fI['ID']]['content_md5']) {
1256 $tmpFile = $this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5'];
1257 }
1258 }
1259 if ($tmpFile === null) {
1260 $tmpFile = GeneralUtility::tempnam('import_temp_');
1261 GeneralUtility::writeFile($tmpFile, $this->dat['files'][$fI['ID']]['content']);
1262 }
1263 clearstatcache();
1264 if (@is_file($tmpFile)) {
1265 $this->unlinkFiles[] = $tmpFile;
1266 if (filesize($tmpFile) == $this->dat['files'][$fI['ID']]['filesize']) {
1267 $this->alternativeFileName[$tmpFile] = $fI['filename'];
1268 $this->alternativeFilePath[$tmpFile] = $this->dat['files'][$fI['ID']]['relFileRef'];
1269 return $tmpFile;
1270 } else {
1271 $this->error('Error: temporary file ' . $tmpFile . ' had a size (' . filesize($tmpFile) . ') different from the original (' . $this->dat['files'][$fI['ID']]['filesize'] . ')');
1272 }
1273 } else {
1274 $this->error('Error: temporary file ' . $tmpFile . ' was not written as it should have been!');
1275 }
1276 } else {
1277 $this->error('Error: No file found for ID ' . $fI['ID']);
1278 }
1279 return null;
1280 }
1281
1282 /**
1283 * After all DB relations has been set in the end of the import (see setRelations()) then it is time to correct all relations inside of FlexForm fields.
1284 * The reason for doing this after is that the setting of relations may affect (quite often!) which data structure is used for the flexforms field!
1285 *
1286 * @return void
1287 * @see setRelations()
1288 */
1289 public function setFlexFormRelations()
1290 {
1291 $updateData = array();
1292 // import_newId contains a register of all records that was in the import memorys "records" key
1293 foreach ($this->import_newId as $nId => $dat) {
1294 $table = $dat['table'];
1295 $uid = $dat['uid'];
1296 // original UID - NOT the new one!
1297 // If the record has been written and received a new id, then proceed:
1298 if (!isset($this->import_mapId[$table][$uid])) {
1299 $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')');
1300 continue;
1301 }
1302
1303 if (!is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {
1304 $this->error('Error: no record was found in data array!');
1305 continue;
1306 }
1307 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
1308 // Traverse relation fields of each record
1309 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
1310 switch ((string)$config['type']) {
1311 case 'flex':
1312 // Get XML content and set as default value (string, non-processed):
1313 $updateData[$table][$thisNewUid][$field] = $this->dat['records'][$table . ':' . $uid]['data'][$field];
1314 // If there has been registered relations inside the flex form field, run processing on the content:
1315 if (!empty($config['flexFormRels']['db']) || !empty($config['flexFormRels']['file'])) {
1316 $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');
1317 // This will fetch the new row for the element (which should be updated with any references to data structures etc.)
1318 $conf = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
1319 if (is_array($origRecordRow) && is_array($conf) && $conf['type'] === 'flex') {
1320 // Get current data structure and value array:
1321 $dataStructArray = BackendUtility::getFlexFormDS($conf, $origRecordRow, $table, $field);
1322 $currentValueArray = GeneralUtility::xml2array($updateData[$table][$thisNewUid][$field]);
1323 // Do recursive processing of the XML data:
1324 $iteratorObj = GeneralUtility::makeInstance(DataHandler::class);
1325 $iteratorObj->callBackObj = $this;
1326 $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData(
1327 $currentValueArray['data'],
1328 array(),
1329 array(),
1330 $dataStructArray,
1331 array($table, $thisNewUid, $field, $config),
1332 'remapListedDBRecords_flexFormCallBack'
1333 );
1334 // The return value is set as an array which means it will be processed by tcemain for file and DB references!
1335 if (is_array($currentValueArray['data'])) {
1336 $updateData[$table][$thisNewUid][$field] = $currentValueArray;
1337 }
1338 }
1339 }
1340 break;
1341 }
1342 }
1343 }
1344 if (!empty($updateData)) {
1345 $tce = $this->getNewTCE();
1346 $tce->isImporting = true;
1347 $this->callHook('before_setFlexFormRelations', array(
1348 'tce' => &$tce,
1349 'data' => &$updateData
1350 ));
1351 $tce->start($updateData, array());
1352 $tce->process_datamap();
1353 $this->callHook('after_setFlexFormRelations', array(
1354 'tce' => &$tce
1355 ));
1356 }
1357 }
1358
1359 /**
1360 * Callback function for traversing the FlexForm structure in relation to remapping database relations
1361 *
1362 * @param array $pParams Set of parameters in numeric array: table, uid, field
1363 * @param array $dsConf TCA config for field (from Data Structure of course)
1364 * @param string $dataValue Field value (from FlexForm XML)
1365 * @param string $dataValue_ext1 Not used
1366 * @param string $dataValue_ext2 Not used
1367 * @param string $path Path of where the data structure of the element is found
1368 * @return array Array where the "value" key carries the value.
1369 * @see setFlexFormRelations()
1370 */
1371 public function remapListedDBRecords_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path)
1372 {
1373 // Extract parameters:
1374 list(, , , $config) = $pParams;
1375 // In case the $path is used as index without a trailing slash we will remove that
1376 if (!is_array($config['flexFormRels']['db'][$path]) && is_array($config['flexFormRels']['db'][rtrim($path, '/')])) {
1377 $path = rtrim($path, '/');
1378 }
1379 if (is_array($config['flexFormRels']['db'][$path])) {
1380 $valArray = $this->setRelations_db($config['flexFormRels']['db'][$path], $dsConf);
1381 $dataValue = implode(',', $valArray);
1382 }
1383 if (is_array($config['flexFormRels']['file'][$path])) {
1384 $valArr = array();
1385 foreach ($config['flexFormRels']['file'][$path] as $fI) {
1386 $valArr[] = $this->import_addFileNameToBeCopied($fI);
1387 }
1388 $dataValue = implode(',', $valArr);
1389 }
1390 return array('value' => $dataValue);
1391 }
1392
1393 /**************************
1394 * Import / Soft References
1395 *************************/
1396
1397 /**
1398 * Processing of soft references
1399 *
1400 * @return void
1401 */
1402 public function processSoftReferences()
1403 {
1404 // Initialize:
1405 $inData = array();
1406 // Traverse records:
1407 if (is_array($this->dat['header']['records'])) {
1408 foreach ($this->dat['header']['records'] as $table => $recs) {
1409 foreach ($recs as $uid => $thisRec) {
1410 // If there are soft references defined, traverse those:
1411 if (isset($GLOBALS['TCA'][$table]) && is_array($thisRec['softrefs'])) {
1412 // First traversal is to collect softref configuration and split them up based on fields.
1413 // This could probably also have been done with the "records" key instead of the header.
1414 $fieldsIndex = array();
1415 foreach ($thisRec['softrefs'] as $softrefDef) {
1416 // If a substitution token is set:
1417 if ($softrefDef['field'] && is_array($softrefDef['subst']) && $softrefDef['subst']['tokenID']) {
1418 $fieldsIndex[$softrefDef['field']][$softrefDef['subst']['tokenID']] = $softrefDef;
1419 }
1420 }
1421 // The new id:
1422 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
1423 // Now, if there are any fields that require substitution to be done, lets go for that:
1424 foreach ($fieldsIndex as $field => $softRefCfgs) {
1425 if (is_array($GLOBALS['TCA'][$table]['columns'][$field])) {
1426 $conf = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
1427 if ($conf['type'] === 'flex') {
1428 // This will fetch the new row for the element (which should be updated with any references to data structures etc.)
1429 $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');
1430 if (is_array($origRecordRow)) {
1431 // Get current data structure and value array:
1432 $dataStructArray = BackendUtility::getFlexFormDS($conf, $origRecordRow, $table, $field);
1433 $currentValueArray = GeneralUtility::xml2array($origRecordRow[$field]);
1434 // Do recursive processing of the XML data:
1435 /** @var $iteratorObj DataHandler */
1436 $iteratorObj = GeneralUtility::makeInstance(DataHandler::class);
1437 $iteratorObj->callBackObj = $this;
1438 $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData($currentValueArray['data'], array(), array(), $dataStructArray, array($table, $uid, $field, $softRefCfgs), 'processSoftReferences_flexFormCallBack');
1439 // The return value is set as an array which means it will be processed by tcemain for file and DB references!
1440 if (is_array($currentValueArray['data'])) {
1441 $inData[$table][$thisNewUid][$field] = $currentValueArray;
1442 }
1443 }
1444 } else {
1445 // Get tokenizedContent string and proceed only if that is not blank:
1446 $tokenizedContent = $this->dat['records'][$table . ':' . $uid]['rels'][$field]['softrefs']['tokenizedContent'];
1447 if (strlen($tokenizedContent) && is_array($softRefCfgs)) {
1448 $inData[$table][$thisNewUid][$field] = $this->processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid);
1449 }
1450 }
1451 }
1452 }
1453 }
1454 }
1455 }
1456 }
1457 // Now write to database:
1458 $tce = $this->getNewTCE();
1459 $tce->isImporting = true;
1460 $this->callHook('before_processSoftReferences', array(
1461 'tce' => $tce,
1462 'data' => &$inData
1463 ));
1464 $tce->enableLogging = true;
1465 $tce->start($inData, array());
1466 $tce->process_datamap();
1467 $this->callHook('after_processSoftReferences', array(
1468 'tce' => $tce
1469 ));
1470 }
1471
1472 /**
1473 * Callback function for traversing the FlexForm structure in relation to remapping softreference relations
1474 *
1475 * @param array $pParams Set of parameters in numeric array: table, uid, field
1476 * @param array $dsConf TCA config for field (from Data Structure of course)
1477 * @param string $dataValue Field value (from FlexForm XML)
1478 * @param string $dataValue_ext1 Not used
1479 * @param string $dataValue_ext2 Not used
1480 * @param string $path Path of where the data structure where the element is found
1481 * @return array Array where the "value" key carries the value.
1482 * @see setFlexFormRelations()
1483 */
1484 public function processSoftReferences_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path)
1485 {
1486 // Extract parameters:
1487 list($table, $origUid, $field, $softRefCfgs) = $pParams;
1488 if (is_array($softRefCfgs)) {
1489 // First, find all soft reference configurations for this structure path (they are listed flat in the header):
1490 $thisSoftRefCfgList = array();
1491 foreach ($softRefCfgs as $sK => $sV) {
1492 if ($sV['structurePath'] === $path) {
1493 $thisSoftRefCfgList[$sK] = $sV;
1494 }
1495 }
1496 // If any was found, do processing:
1497 if (!empty($thisSoftRefCfgList)) {
1498 // Get tokenizedContent string and proceed only if that is not blank:
1499 $tokenizedContent = $this->dat['records'][$table . ':' . $origUid]['rels'][$field]['flexFormRels']['softrefs'][$path]['tokenizedContent'];
1500 if (strlen($tokenizedContent)) {
1501 $dataValue = $this->processSoftReferences_substTokens($tokenizedContent, $thisSoftRefCfgList, $table, $origUid);
1502 }
1503 }
1504 }
1505 // Return
1506 return array('value' => $dataValue);
1507 }
1508
1509 /**
1510 * Substition of softreference tokens
1511 *
1512 * @param string $tokenizedContent Content of field with soft reference tokens in.
1513 * @param array $softRefCfgs Soft reference configurations
1514 * @param string $table Table for which the processing occurs
1515 * @param string $uid UID of record from table
1516 * @return string The input content with tokens substituted according to entries in softRefCfgs
1517 */
1518 public function processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid)
1519 {
1520 // traverse each softref type for this field:
1521 foreach ($softRefCfgs as $cfg) {
1522 // Get token ID:
1523 $tokenID = $cfg['subst']['tokenID'];
1524 // Default is current token value:
1525 $insertValue = $cfg['subst']['tokenValue'];
1526 // Based on mode:
1527 switch ((string)$this->softrefCfg[$tokenID]['mode']) {
1528 case 'exclude':
1529 // Exclude is a simple passthrough of the value
1530 break;
1531 case 'editable':
1532 // Editable always picks up the value from this input array:
1533 $insertValue = $this->softrefInputValues[$tokenID];
1534 break;
1535 default:
1536 // Mapping IDs/creating files: Based on type, look up new value:
1537 switch ((string)$cfg['subst']['type']) {
1538 case 'file':
1539 // Create / Overwrite file:
1540 $insertValue = $this->processSoftReferences_saveFile($cfg['subst']['relFileName'], $cfg, $table, $uid);
1541 break;
1542 case 'db':
1543 default:
1544 // Trying to map database element if found in the mapID array:
1545 list($tempTable, $tempUid) = explode(':', $cfg['subst']['recordRef']);
1546 if (isset($this->import_mapId[$tempTable][$tempUid])) {
1547 $insertValue = BackendUtility::wsMapId($tempTable, $this->import_mapId[$tempTable][$tempUid]);
1548 // Look if reference is to a page and the original token value was NOT an integer - then we assume is was an alias and try to look up the new one!
1549 if ($tempTable === 'pages' && !MathUtility::canBeInterpretedAsInteger($cfg['subst']['tokenValue'])) {
1550 $recWithUniqueValue = BackendUtility::getRecord($tempTable, $insertValue, 'alias');
1551 if ($recWithUniqueValue['alias']) {
1552 $insertValue = $recWithUniqueValue['alias'];
1553 }
1554 } elseif (strpos($cfg['subst']['tokenValue'], ':') !== false) {
1555 list($tokenKey) = explode(':', $cfg['subst']['tokenValue']);
1556 $insertValue = $tokenKey . ':' . $insertValue;
1557 }
1558 }
1559 }
1560 }
1561 // Finally, swap the soft reference token in tokenized content with the insert value:
1562 $tokenizedContent = str_replace('{softref:' . $tokenID . '}', $insertValue, $tokenizedContent);
1563 }
1564 return $tokenizedContent;
1565 }
1566
1567 /**
1568 * Process a soft reference file
1569 *
1570 * @param string $relFileName Old Relative filename
1571 * @param array $cfg soft reference configuration array
1572 * @param string $table Table for which the processing occurs
1573 * @param string $uid UID of record from table
1574 * @return string New relative filename (value to insert instead of the softref token)
1575 */
1576 public function processSoftReferences_saveFile($relFileName, $cfg, $table, $uid)
1577 {
1578 if ($fileHeaderInfo = $this->dat['header']['files'][$cfg['file_ID']]) {
1579 // Initialize; Get directory prefix for file and find possible RTE filename
1580 $dirPrefix = PathUtility::dirname($relFileName) . '/';
1581 $rteOrigName = $this->getRTEoriginalFilename(PathUtility::basename($relFileName));
1582 // If filename looks like an RTE file, and the directory is in "uploads/", then process as a RTE file!
1583 if ($rteOrigName && GeneralUtility::isFirstPartOfStr($dirPrefix, 'uploads/')) {
1584 // RTE:
1585 // First, find unique RTE file name:
1586 if (@is_dir((PATH_site . $dirPrefix))) {
1587 // From the "original" RTE filename, produce a new "original" destination filename which is unused.
1588 // Even if updated, the image should be unique. Currently the problem with this is that it leaves a lot of unused RTE images...
1589 $fileProcObj = $this->getFileProcObj();
1590 $origDestName = $fileProcObj->getUniqueName($rteOrigName, PATH_site . $dirPrefix);
1591 // Create copy file name:
1592 $pI = pathinfo($relFileName);
1593 $copyDestName = PathUtility::dirname($origDestName) . '/RTEmagicC_' . substr(PathUtility::basename($origDestName), 10) . '.' . $pI['extension'];
1594 if (
1595 !@is_file($copyDestName) && !@is_file($origDestName)
1596 && $origDestName === GeneralUtility::getFileAbsFileName($origDestName)
1597 && $copyDestName === GeneralUtility::getFileAbsFileName($copyDestName)
1598 ) {
1599 if ($this->dat['header']['files'][$fileHeaderInfo['RTE_ORIG_ID']]) {
1600 if ($this->legacyImport) {
1601 $fileName = PathUtility::basename($copyDestName);
1602 $this->writeSysFileResourceForLegacyImport($fileName, $cfg['file_ID']);
1603 $relFileName = $this->filePathMap[$cfg['file_ID']] . '" data-htmlarea-file-uid="' . $fileName . '" data-htmlarea-file-table="sys_file';
1604 // Also save the original file
1605 $originalFileName = PathUtility::basename($origDestName);
1606 $this->writeSysFileResourceForLegacyImport($originalFileName, $fileHeaderInfo['RTE_ORIG_ID']);
1607 } else {
1608 // Write the copy and original RTE file to the respective filenames:
1609 $this->writeFileVerify($copyDestName, $cfg['file_ID'], true);
1610 $this->writeFileVerify($origDestName, $fileHeaderInfo['RTE_ORIG_ID'], true);
1611 // Return the relative path of the copy file name:
1612 return PathUtility::stripPathSitePrefix($copyDestName);
1613 }
1614 } else {
1615 $this->error('ERROR: Could not find original file ID');
1616 }
1617 } else {
1618 $this->error('ERROR: The destination filenames "' . $copyDestName . '" and "' . $origDestName . '" either existed or have non-valid names');
1619 }
1620 } else {
1621 $this->error('ERROR: "' . PATH_site . $dirPrefix . '" was not a directory, so could not process file "' . $relFileName . '"');
1622 }
1623 } elseif (GeneralUtility::isFirstPartOfStr($dirPrefix, $this->fileadminFolderName . '/')) {
1624 // File in fileadmin/ folder:
1625 // Create file (and possible resources)
1626 $newFileName = $this->processSoftReferences_saveFile_createRelFile($dirPrefix, PathUtility::basename($relFileName), $cfg['file_ID'], $table, $uid);
1627 if (strlen($newFileName)) {
1628 $relFileName = $newFileName;
1629 } else {
1630 $this->error('ERROR: No new file created for "' . $relFileName . '"');
1631 }
1632 } else {
1633 $this->error('ERROR: Sorry, cannot operate on non-RTE files which are outside the fileadmin folder.');
1634 }
1635 } else {
1636 $this->error('ERROR: Could not find file ID in header.');
1637 }
1638 // Return (new) filename relative to PATH_site:
1639 return $relFileName;
1640 }
1641
1642 /**
1643 * Create file in directory and return the new (unique) filename
1644 *
1645 * @param string $origDirPrefix Directory prefix, relative, with trailing slash
1646 * @param string $fileName Filename (without path)
1647 * @param string $fileID File ID from import memory
1648 * @param string $table Table for which the processing occurs
1649 * @param string $uid UID of record from table
1650 * @return string|NULL New relative filename, if any
1651 */
1652 public function processSoftReferences_saveFile_createRelFile($origDirPrefix, $fileName, $fileID, $table, $uid)
1653 {
1654 // If the fileID map contains an entry for this fileID then just return the relative filename of that entry;
1655 // we don't want to write another unique filename for this one!
1656 if (isset($this->fileIDMap[$fileID])) {
1657 return PathUtility::stripPathSitePrefix($this->fileIDMap[$fileID]);
1658 }
1659 if ($this->legacyImport) {
1660 // set dirPrefix to fileadmin because the right target folder is set and checked for permissions later
1661 $dirPrefix = $this->fileadminFolderName . '/';
1662 } else {
1663 // Verify FileMount access to dir-prefix. Returns the best alternative relative path if any
1664 $dirPrefix = $this->verifyFolderAccess($origDirPrefix);
1665 }
1666 if ($dirPrefix && (!$this->update || $origDirPrefix === $dirPrefix) && $this->checkOrCreateDir($dirPrefix)) {
1667 $fileHeaderInfo = $this->dat['header']['files'][$fileID];
1668 $updMode = $this->update && $this->import_mapId[$table][$uid] === $uid && $this->import_mode[$table . ':' . $uid] !== 'as_new';
1669 // Create new name for file:
1670 // Must have same ID in map array (just for security, is not really needed) and NOT be set "as_new".
1671
1672 // Write main file:
1673 if ($this->legacyImport) {
1674 $fileWritten = $this->writeSysFileResourceForLegacyImport($fileName, $fileID);
1675 if ($fileWritten) {
1676 $newName = 'file:' . $fileName;
1677 return $newName;
1678 // no support for HTML/CSS file resources attached ATM - see below
1679 }
1680 } else {
1681 if ($updMode) {
1682 $newName = PATH_site . $dirPrefix . $fileName;
1683 } else {
1684 // Create unique filename:
1685 $fileProcObj = $this->getFileProcObj();
1686 $newName = $fileProcObj->getUniqueName($fileName, PATH_site . $dirPrefix);
1687 }
1688 if ($this->writeFileVerify($newName, $fileID)) {
1689 // If the resource was an HTML/CSS file with resources attached, we will write those as well!
1690 if (is_array($fileHeaderInfo['EXT_RES_ID'])) {
1691 $tokenizedContent = $this->dat['files'][$fileID]['tokenizedContent'];
1692 $tokenSubstituted = false;
1693 $fileProcObj = $this->getFileProcObj();
1694 if ($updMode) {
1695 foreach ($fileHeaderInfo['EXT_RES_ID'] as $res_fileID) {
1696 if ($this->dat['files'][$res_fileID]['filename']) {
1697 // Resolve original filename:
1698 $relResourceFileName = $this->dat['files'][$res_fileID]['parentRelFileName'];
1699 $absResourceFileName = GeneralUtility::resolveBackPath(PATH_site . $origDirPrefix . $relResourceFileName);
1700 $absResourceFileName = GeneralUtility::getFileAbsFileName($absResourceFileName);
1701 if ($absResourceFileName && GeneralUtility::isFirstPartOfStr($absResourceFileName, PATH_site . $this->fileadminFolderName . '/')) {
1702 $destDir = PathUtility::stripPathSitePrefix(PathUtility::dirname($absResourceFileName) . '/');
1703 if ($this->verifyFolderAccess($destDir, true) && $this->checkOrCreateDir($destDir)) {
1704 $this->writeFileVerify($absResourceFileName, $res_fileID);
1705 } else {
1706 $this->error('ERROR: Could not create file in directory "' . $destDir . '"');
1707 }
1708 } else {
1709 $this->error('ERROR: Could not resolve path for "' . $relResourceFileName . '"');
1710 }
1711 $tokenizedContent = str_replace('{EXT_RES_ID:' . $res_fileID . '}', $relResourceFileName, $tokenizedContent);
1712 $tokenSubstituted = true;
1713 }
1714 }
1715 } else {
1716 // Create the resouces directory name (filename without extension, suffixed "_FILES")
1717 $resourceDir = PathUtility::dirname($newName) . '/' . preg_replace('/\\.[^.]*$/', '', PathUtility::basename($newName)) . '_FILES';
1718 if (GeneralUtility::mkdir($resourceDir)) {
1719 foreach ($fileHeaderInfo['EXT_RES_ID'] as $res_fileID) {
1720 if ($this->dat['files'][$res_fileID]['filename']) {
1721 $absResourceFileName = $fileProcObj->getUniqueName($this->dat['files'][$res_fileID]['filename'], $resourceDir);
1722 $relResourceFileName = substr($absResourceFileName, strlen(PathUtility::dirname($resourceDir)) + 1);
1723 $this->writeFileVerify($absResourceFileName, $res_fileID);
1724 $tokenizedContent = str_replace('{EXT_RES_ID:' . $res_fileID . '}', $relResourceFileName, $tokenizedContent);
1725 $tokenSubstituted = true;
1726 }
1727 }
1728 }
1729 }
1730 // If substitutions has been made, write the content to the file again:
1731 if ($tokenSubstituted) {
1732 GeneralUtility::writeFile($newName, $tokenizedContent);
1733 }
1734 }
1735 return PathUtility::stripPathSitePrefix($newName);
1736 }
1737 }
1738 }
1739 return null;
1740 }
1741
1742 /**
1743 * Writes a file from the import memory having $fileID to file name $fileName which must be an absolute path inside PATH_site
1744 *
1745 * @param string $fileName Absolute filename inside PATH_site to write to
1746 * @param string $fileID File ID from import memory
1747 * @param bool $bypassMountCheck Bypasses the checking against filemounts - only for RTE files!
1748 * @return bool Returns TRUE if it went well. Notice that the content of the file is read again, and md5 from import memory is validated.
1749 */
1750 public function writeFileVerify($fileName, $fileID, $bypassMountCheck = false)
1751 {
1752 $fileProcObj = $this->getFileProcObj();
1753 if (!$fileProcObj->actionPerms['addFile']) {
1754 $this->error('ERROR: You did not have sufficient permissions to write the file "' . $fileName . '"');
1755 return false;
1756 }
1757 // Just for security, check again. Should actually not be necessary.
1758 if (!$fileProcObj->checkPathAgainstMounts($fileName) && !$bypassMountCheck) {
1759 $this->error('ERROR: Filename "' . $fileName . '" was not allowed in destination path!');
1760 return false;
1761 }
1762 $fI = GeneralUtility::split_fileref($fileName);
1763 if (!$fileProcObj->checkIfAllowed($fI['fileext'], $fI['path'], $fI['file']) && (!$this->allowPHPScripts || !$this->getBackendUser()->isAdmin())) {
1764 $this->error('ERROR: Filename "' . $fileName . '" failed against extension check or deny-pattern!');
1765 return false;
1766 }
1767 if (!GeneralUtility::getFileAbsFileName($fileName)) {
1768 $this->error('ERROR: Filename "' . $fileName . '" was not a valid relative file path!');
1769 return false;
1770 }
1771 if (!$this->dat['files'][$fileID]) {
1772 $this->error('ERROR: File ID "' . $fileID . '" could not be found');
1773 return false;
1774 }
1775 GeneralUtility::writeFile($fileName, $this->dat['files'][$fileID]['content']);
1776 $this->fileIDMap[$fileID] = $fileName;
1777 if (md5(GeneralUtility::getUrl($fileName)) == $this->dat['files'][$fileID]['content_md5']) {
1778 return true;
1779 } else {
1780 $this->error('ERROR: File content "' . $fileName . '" was corrupted');
1781 return false;
1782 }
1783 }
1784
1785 /**
1786 * Writes the file with the is $fileId to the legacy import folder. The file name will used from
1787 * argument $fileName and the file was successfully created or an identical file was already found,
1788 * $fileName will held the uid of the new created file record.
1789 *
1790 * @param string $fileName The file name for the new file. Value would be changed to the uid of the new created file record.
1791 * @param int $fileId The id of the file in data array
1792 * @return bool
1793 */
1794 protected function writeSysFileResourceForLegacyImport(&$fileName, $fileId)
1795 {
1796 if ($this->legacyImportFolder === null) {
1797 return false;
1798 }
1799
1800 if (!isset($this->dat['files'][$fileId])) {
1801 $this->error('ERROR: File ID "' . $fileId . '" could not be found');
1802 return false;
1803 }
1804
1805 $temporaryFile = $this->writeTemporaryFileFromData($fileId, 'files');
1806 if ($temporaryFile === null) {
1807 // error on writing the file. Error message was already added
1808 return false;
1809 }
1810
1811 $importFolder = $this->legacyImportFolder;
1812
1813 if (isset($this->dat['files'][$fileId]['relFileName'])) {
1814 $relativeFilePath = PathUtility::dirname($this->dat['files'][$fileId]['relFileName']);
1815
1816 if (!$this->legacyImportFolder->hasFolder($relativeFilePath)) {
1817 $this->legacyImportFolder->createFolder($relativeFilePath);
1818 }
1819 $importFolder = $this->legacyImportFolder->getSubfolder($relativeFilePath);
1820 }
1821
1822 $fileObject = null;
1823
1824 try {
1825 // check, if there is alreay the same file in the folder
1826 if ($importFolder->hasFile($fileName)) {
1827 $fileStorage = $importFolder->getStorage();
1828 $file = $fileStorage->getFile($importFolder->getIdentifier() . $fileName);
1829 if ($file->getSha1() === sha1_file($temporaryFile)) {
1830 $fileObject = $file;
1831 }
1832 }
1833 } catch (Exception $e) {
1834 }
1835
1836 if ($fileObject === null) {
1837 try {
1838 $fileObject = $importFolder->addFile($temporaryFile, $fileName, DuplicationBehavior::RENAME);
1839 } catch (Exception $e) {
1840 $this->error('Error: no file could be added to the storage for file name ' . $this->alternativeFileName[$temporaryFile]);
1841 }
1842 }
1843
1844 if (md5_file(PATH_site . $fileObject->getPublicUrl()) == $this->dat['files'][$fileId]['content_md5']) {
1845 $fileName = $fileObject->getUid();
1846 $this->fileIDMap[$fileId] = $fileName;
1847 $this->filePathMap[$fileId] = $fileObject->getPublicUrl();
1848 return true;
1849 } else {
1850 $this->error('ERROR: File content "' . $this->dat['files'][$fileId]['relFileName'] . '" was corrupted');
1851 }
1852
1853 return false;
1854 }
1855
1856 /**
1857 * Migrate legacy import records
1858 *
1859 * @return void
1860 */
1861 protected function migrateLegacyImportRecords()
1862 {
1863 $updateData= array();
1864
1865 foreach ($this->legacyImportMigrationRecords as $table => $records) {
1866 foreach ($records as $uid => $fields) {
1867 $row = BackendUtility::getRecord($table, $uid);
1868 if (empty($row)) {
1869 continue;
1870 }
1871
1872 foreach ($fields as $field => $referenceIds) {
1873 $fieldConfiguration = $this->legacyImportMigrationTables[$table][$field];
1874
1875 if (isset($fieldConfiguration['titleTexts'])) {
1876 $titleTextField = $fieldConfiguration['titleTexts'];
1877 if (isset($row[$titleTextField]) && $row[$titleTextField] !== '') {
1878 $titleTextContents = explode(LF, $row[$titleTextField]);
1879 $updateData[$table][$uid][$titleTextField] = '';
1880 }
1881 }
1882
1883 if (isset($fieldConfiguration['alternativeTexts'])) {
1884 $alternativeTextField = $fieldConfiguration['alternativeTexts'];
1885 if (isset($row[$alternativeTextField]) && $row[$alternativeTextField] !== '') {
1886 $alternativeTextContents = explode(LF, $row[$alternativeTextField]);
1887 $updateData[$table][$uid][$alternativeTextField] = '';
1888 }
1889 }
1890 if (isset($fieldConfiguration['description'])) {
1891 $descriptionField = $fieldConfiguration['description'];
1892 if ($row[$descriptionField] !== '') {
1893 $descriptionContents = explode(LF, $row[$descriptionField]);
1894 $updateData[$table][$uid][$descriptionField] = '';
1895 }
1896 }
1897 if (isset($fieldConfiguration['links'])) {
1898 $linkField = $fieldConfiguration['links'];
1899 if ($row[$linkField] !== '') {
1900 $linkContents = explode(LF, $row[$linkField]);
1901 $updateData[$table][$uid][$linkField] = '';
1902 }
1903 }
1904
1905 foreach ($referenceIds as $key => $referenceId) {
1906 if (isset($titleTextContents[$key])) {
1907 $updateData['sys_file_reference'][$referenceId]['title'] = trim($titleTextContents[$key]);
1908 }
1909 if (isset($alternativeTextContents[$key])) {
1910 $updateData['sys_file_reference'][$referenceId]['alternative'] = trim($alternativeTextContents[$key]);
1911 }
1912 if (isset($descriptionContents[$key])) {
1913 $updateData['sys_file_reference'][$referenceId]['description'] = trim($descriptionContents[$key]);
1914 }
1915 if (isset($linkContents[$key])) {
1916 $updateData['sys_file_reference'][$referenceId]['link'] = trim($linkContents[$key]);
1917 }
1918 }
1919 }
1920 }
1921 }
1922
1923 // update
1924 $tce = $this->getNewTCE();
1925 $tce->isImporting = true;
1926 $tce->start($updateData, array());
1927 $tce->process_datamap();
1928 }
1929
1930 /**
1931 * Returns TRUE if directory exists and if it doesn't it will create directory and return TRUE if that succeeded.
1932 *
1933 * @param string $dirPrefix Directory to create. Having a trailing slash. Must be in fileadmin/. Relative to PATH_site
1934 * @return bool TRUE, if directory exists (was created)
1935 */
1936 public function checkOrCreateDir($dirPrefix)
1937 {
1938 // Split dir path and remove first directory (which should be "fileadmin")
1939 $filePathParts = explode('/', $dirPrefix);
1940 $firstDir = array_shift($filePathParts);
1941 if ($firstDir === $this->fileadminFolderName && GeneralUtility::getFileAbsFileName($dirPrefix)) {
1942 $pathAcc = '';
1943 foreach ($filePathParts as $dirname) {
1944 $pathAcc .= '/' . $dirname;
1945 if (strlen($dirname)) {
1946 if (!@is_dir((PATH_site . $this->fileadminFolderName . $pathAcc))) {
1947 if (!GeneralUtility::mkdir((PATH_site . $this->fileadminFolderName . $pathAcc))) {
1948 $this->error('ERROR: Directory could not be created....B');
1949 return false;
1950 }
1951 }
1952 } elseif ($dirPrefix === $this->fileadminFolderName . $pathAcc) {
1953 return true;
1954 } else {
1955 $this->error('ERROR: Directory could not be created....A');
1956 }
1957 }
1958 }
1959 return false;
1960 }
1961
1962 /**************************
1963 * File Input
1964 *************************/
1965
1966 /**
1967 * Loads the header section/all of the $filename into memory
1968 *
1969 * @param string $filename Filename, absolute
1970 * @param bool $all If set, all information is loaded (header, records and files). Otherwise the default is to read only the header information
1971 * @return bool TRUE if the operation went well
1972 */
1973 public function loadFile($filename, $all = false)
1974 {
1975 if (!@is_file($filename)) {
1976 $this->error('Filename not found: ' . $filename);
1977 return false;
1978 }
1979 $fI = pathinfo($filename);
1980 if (@is_dir($filename . '.files')) {
1981 if (GeneralUtility::isAllowedAbsPath($filename . '.files')) {
1982 // copy the folder lowlevel to typo3temp, because the files would be deleted after import
1983 $temporaryFolderName = $this->getTemporaryFolderName();
1984 GeneralUtility::copyDirectory($filename . '.files', $temporaryFolderName);
1985 $this->filesPathForImport = $temporaryFolderName;
1986 } else {
1987 $this->error('External import files for the given import source is currently not supported.');
1988 }
1989 }
1990 if (strtolower($fI['extension']) == 'xml') {
1991 // XML:
1992 $xmlContent = GeneralUtility::getUrl($filename);
1993 if (strlen($xmlContent)) {
1994 $this->dat = GeneralUtility::xml2array($xmlContent, '', true);
1995 if (is_array($this->dat)) {
1996 if ($this->dat['_DOCUMENT_TAG'] === 'T3RecordDocument' && is_array($this->dat['header']) && is_array($this->dat['records'])) {
1997 $this->loadInit();
1998 return true;
1999 } else {
2000 $this->error('XML file did not contain proper XML for TYPO3 Import');
2001 }
2002 } else {
2003 $this->error('XML could not be parsed: ' . $this->dat);
2004 }
2005 } else {
2006 $this->error('Error opening file: ' . $filename);
2007 }
2008 } else {
2009 // T3D
2010 if ($fd = fopen($filename, 'rb')) {
2011 $this->dat['header'] = $this->getNextFilePart($fd, 1, 'header');
2012 if ($all) {
2013 $this->dat['records'] = $this->getNextFilePart($fd, 1, 'records');
2014 $this->dat['files'] = $this->getNextFilePart($fd, 1, 'files');
2015 $this->dat['files_fal'] = $this->getNextFilePart($fd, 1, 'files_fal');
2016 }
2017 $this->loadInit();
2018 return true;
2019 } else {
2020 $this->error('Error opening file: ' . $filename);
2021 }
2022 fclose($fd);
2023 }
2024 return false;
2025 }
2026
2027 /**
2028 * Returns the next content part form the fileresource (t3d), $fd
2029 *
2030 * @param resource $fd File pointer
2031 * @param bool $unserialize If set, the returned content is unserialized into an array, otherwise you get the raw string
2032 * @param string $name For error messages this indicates the section of the problem.
2033 * @return string|NULL Data string or NULL in case of an error
2034 * @access private
2035 * @see loadFile()
2036 */
2037 public function getNextFilePart($fd, $unserialize = false, $name = '')
2038 {
2039 $initStrLen = 32 + 1 + 1 + 1 + 10 + 1;
2040 // Getting header data
2041 $initStr = fread($fd, $initStrLen);
2042 if (empty($initStr)) {
2043 $this->error('File does not contain data for "' . $name . '"');
2044 return null;
2045 }
2046 $initStrDat = explode(':', $initStr);
2047 if (strstr($initStrDat[0], 'Warning')) {
2048 $this->error('File read error: Warning message in file. (' . $initStr . fgets($fd) . ')');
2049 return null;
2050 }
2051 if ((string)$initStrDat[3] !== '') {
2052 $this->error('File read error: InitString had a wrong length. (' . $name . ')');
2053 return null;
2054 }
2055 $datString = fread($fd, (int)$initStrDat[2]);
2056 fread($fd, 1);
2057 if (md5($datString) === $initStrDat[0]) {
2058 if ($initStrDat[1]) {
2059 if ($this->compress) {
2060 $datString = gzuncompress($datString);
2061 return $unserialize ? unserialize($datString) : $datString;
2062 } else {
2063 $this->error('Content read error: This file requires decompression, but this server does not offer gzcompress()/gzuncompress() functions.');
2064 }
2065 }
2066 } else {
2067 $this->error('MD5 check failed (' . $name . ')');
2068 }
2069 return null;
2070 }
2071
2072 /**
2073 * Loads T3D file content into the $this->dat array
2074 * (This function can be used to test the output strings from ->compileMemoryToFileContent())
2075 *
2076 * @param string $filecontent File content
2077 * @return void
2078 */
2079 public function loadContent($filecontent)
2080 {
2081 $pointer = 0;
2082 $this->dat['header'] = $this->getNextContentPart($filecontent, $pointer, 1, 'header');
2083 $this->dat['records'] = $this->getNextContentPart($filecontent, $pointer, 1, 'records');
2084 $this->dat['files'] = $this->getNextContentPart($filecontent, $pointer, 1, 'files');
2085 $this->loadInit();
2086 }
2087
2088 /**
2089 * Returns the next content part from the $filecontent
2090 *
2091 * @param string $filecontent File content string
2092 * @param int $pointer File pointer (where to read from)
2093 * @param bool $unserialize If set, the returned content is unserialized into an array, otherwise you get the raw string
2094 * @param string $name For error messages this indicates the section of the problem.
2095 * @return string|NULL Data string
2096 */
2097 public function getNextContentPart($filecontent, &$pointer, $unserialize = false, $name = '')
2098 {
2099 $initStrLen = 32 + 1 + 1 + 1 + 10 + 1;
2100 // getting header data
2101 $initStr = substr($filecontent, $pointer, $initStrLen);
2102 $pointer += $initStrLen;
2103 $initStrDat = explode(':', $initStr);
2104 if ((string)$initStrDat[3] !== '') {
2105 $this->error('Content read error: InitString had a wrong length. (' . $name . ')');
2106 return null;
2107 }
2108 $datString = substr($filecontent, $pointer, (int)$initStrDat[2]);
2109 $pointer += (int)$initStrDat[2] + 1;
2110 if (md5($datString) === $initStrDat[0]) {
2111 if ($initStrDat[1]) {
2112 if ($this->compress) {
2113 $datString = gzuncompress($datString);
2114 return $unserialize ? unserialize($datString) : $datString;
2115 } else {
2116 $this->error('Content read error: This file requires decompression, but this server does not offer gzcompress()/gzuncompress() functions.');
2117 }
2118 }
2119 } else {
2120 $this->error('MD5 check failed (' . $name . ')');
2121 }
2122 return null;
2123 }
2124
2125 /**
2126 * Setting up the object based on the recently loaded ->dat array
2127 *
2128 * @return void
2129 */
2130 public function loadInit()
2131 {
2132 $this->cleanupImportData();
2133 $this->relStaticTables = (array)$this->dat['header']['relStaticTables'];
2134 $this->excludeMap = (array)$this->dat['header']['excludeMap'];
2135 $this->softrefCfg = (array)$this->dat['header']['softrefCfg'];
2136 $this->fixCharsets();
2137 if (
2138 isset($this->dat['header']['meta']['TYPO3_version'])
2139 && VersionNumberUtility::convertVersionNumberToInteger($this->dat['header']['meta']['TYPO3_version']) < 6000000
2140 ) {
2141 $this->legacyImport = true;
2142 $this->initializeLegacyImportFolder();
2143 }
2144 }
2145
2146 /**
2147 * Cleanses any inconsistent states which can occur in imported T3D/XML
2148 *
2149 * @return void
2150 */
2151 protected function cleanupImportData()
2152 {
2153 if (is_array($this->dat['header']['extensionDependencies'])) {
2154 $this->dat['header']['extensionDependencies'] = array_filter($this->dat['header']['extensionDependencies']);
2155 } else {
2156 $this->dat['header']['extensionDependencies'] = array();
2157 }
2158 }
2159
2160 /**
2161 * Fix charset of import memory if different from system charset
2162 *
2163 * @return void
2164 * @see loadInit()
2165 */
2166 public function fixCharsets()
2167 {
2168 $importCharset = $this->dat['header']['charset'];
2169 if ($importCharset) {
2170 if ($importCharset !== $this->getLanguageService()->charSet) {
2171 $this->error('CHARSET: Converting charset of input file (' . $importCharset . ') to the system charset (' . $this->getLanguageService()->charSet . ')');
2172 // Convert meta data:
2173 if (is_array($this->dat['header']['meta'])) {
2174 $this->getLanguageService()->csConvObj->convArray($this->dat['header']['meta'], $importCharset, $this->getLanguageService()->charSet);
2175 }
2176 // Convert record headers:
2177 if (is_array($this->dat['header']['records'])) {
2178 $this->getLanguageService()->csConvObj->convArray($this->dat['header']['records'], $importCharset, $this->getLanguageService()->charSet);
2179 }
2180 // Convert records themselves:
2181 if (is_array($this->dat['records'])) {
2182 $this->getLanguageService()->csConvObj->convArray($this->dat['records'], $importCharset, $this->getLanguageService()->charSet);
2183 }
2184 }
2185 } else {
2186 $this->error('CHARSET: No charset found in import file!');
2187 }
2188 }
2189 }