[FOLLOWUP][TASK] Import/Export (impexp) move to Fluid standalone view
[Packages/TYPO3.CMS.git] / typo3 / sysext / impexp / Classes / Import.php
1 <?php
2 namespace TYPO3\CMS\Impexp;
3
4 /*
5 * This file is part of the TYPO3 CMS project.
6 *
7 * It is free software; you can redistribute it and/or modify it under
8 * the terms of the GNU General Public License, either version 2
9 * of the License, or any later version.
10 *
11 * For the full copyright and license information, please read the
12 * LICENSE.txt file that was distributed with this source code.
13 *
14 * The TYPO3 project - inspiring people to share!
15 */
16
17 use TYPO3\CMS\Backend\Utility\BackendUtility;
18 use TYPO3\CMS\Core\DataHandling\DataHandler;
19 use TYPO3\CMS\Core\Exception;
20 use TYPO3\CMS\Core\Resource\DuplicationBehavior;
21 use TYPO3\CMS\Core\Resource\File;
22 use TYPO3\CMS\Core\Resource\FileInterface;
23 use TYPO3\CMS\Core\Resource\ResourceFactory;
24 use TYPO3\CMS\Core\Resource\StorageRepository;
25 use TYPO3\CMS\Core\Utility\GeneralUtility;
26 use TYPO3\CMS\Core\Utility\MathUtility;
27 use TYPO3\CMS\Core\Utility\PathUtility;
28 use TYPO3\CMS\Core\Utility\StringUtility;
29 use TYPO3\CMS\Core\Utility\VersionNumberUtility;
30
31 /**
32 * T3D file Import library (TYPO3 Record Document)
33 */
34 class Import extends ImportExport
35 {
36 /**
37 * Used to register the forged UID values for imported records that we want
38 * to create with the same UIDs as in the import file. Admin-only feature.
39 *
40 * @var array
41 */
42 public $suggestedInsertUids = array();
43
44 /**
45 * Disable logging when importing
46 *
47 * @var bool
48 */
49 public $enableLogging = false;
50
51 /**
52 * Keys are [tablename]:[new NEWxxx ids (or when updating it is uids)]
53 * while values are arrays with table/uid of the original record it is based on.
54 * With the array keys the new ids can be looked up inside tcemain
55 *
56 * @var array
57 */
58 public $import_newId = array();
59
60 /**
61 * Page id map for page tree (import)
62 *
63 * @var array
64 */
65 public $import_newId_pids = array();
66
67 /**
68 * Internal data accumulation for writing records during import
69 *
70 * @var array
71 */
72 public $import_data = array();
73
74 /**
75 * Array of current registered storage objects
76 *
77 * @var array
78 */
79 protected $storageObjects = array();
80
81 /**
82 * Is set, if the import file has a TYPO3 version below 6.0
83 *
84 * @var bool
85 */
86 protected $legacyImport = false;
87
88 /**
89 * @var \TYPO3\CMS\Core\Resource\Folder
90 */
91 protected $legacyImportFolder = null;
92
93 /**
94 * Related to the default storage root
95 *
96 * @var string
97 */
98 protected $legacyImportTargetPath = '_imported/';
99
100 /**
101 * Table fields to migrate
102 *
103 * @var array
104 */
105 protected $legacyImportMigrationTables = array(
106 'tt_content' => array(
107 'image' => array(
108 'titleTexts' => 'titleText',
109 'description' => 'imagecaption',
110 'links' => 'image_link',
111 'alternativeTexts' => 'altText'
112 ),
113 'media' => array(
114 'description' => 'imagecaption',
115 )
116 ),
117 'pages' => array(
118 'media' => array()
119 ),
120 'pages_language_overlay' => array(
121 'media' => array()
122 )
123 );
124
125 /**
126 * Records to be migrated after all
127 * Multidimensional array [table][uid][field] = array([related sys_file_reference uids])
128 *
129 * @var array
130 */
131 protected $legacyImportMigrationRecords = array();
132
133 /**
134 * @var NULL|string
135 */
136 protected $filesPathForImport = null;
137
138 /**
139 * @var array
140 */
141 protected $unlinkFiles = array();
142
143 /**
144 * @var array
145 */
146 protected $alternativeFileName = array();
147
148 /**
149 * @var array
150 */
151 protected $alternativeFilePath = array();
152
153 /**
154 * @var array
155 */
156 protected $filePathMap = array();
157
158 /**************************
159 * Initialize
160 *************************/
161
162 /**
163 * Init the object
164 *
165 * @return void
166 */
167 public function init()
168 {
169 parent::init();
170 $this->mode = 'import';
171 }
172
173 /***********************
174 * Import
175 ***********************/
176
177 /**
178 * Initialize all settings for the import
179 *
180 * @return void
181 */
182 protected function initializeImport()
183 {
184 // Set this flag to indicate that an import is being/has been done.
185 $this->doesImport = 1;
186 // Initialize:
187 // These vars MUST last for the whole section not being cleared. They are used by the method setRelations() which are called at the end of the import session.
188 $this->import_mapId = array();
189 $this->import_newId = array();
190 $this->import_newId_pids = array();
191 // Temporary files stack initialized:
192 $this->unlinkFiles = array();
193 $this->alternativeFileName = array();
194 $this->alternativeFilePath = array();
195
196 $this->initializeStorageObjects();
197 }
198
199 /**
200 * Initialize the all present storage objects
201 *
202 * @return void
203 */
204 protected function initializeStorageObjects()
205 {
206 /** @var $storageRepository StorageRepository */
207 $storageRepository = GeneralUtility::makeInstance(StorageRepository::class);
208 $this->storageObjects = $storageRepository->findAll();
209 }
210
211 /**
212 * Imports the internal data array to $pid.
213 *
214 * @param int $pid Page ID in which to import the content
215 * @return void
216 */
217 public function importData($pid)
218 {
219 $this->initializeImport();
220
221 // Write sys_file_storages first
222 $this->writeSysFileStorageRecords();
223 // Write sys_file records and write the binary file data
224 $this->writeSysFileRecords();
225 // Write records, first pages, then the rest
226 // Fields with "hard" relations to database, files and flexform fields are kept empty during this run
227 $this->writeRecords_pages($pid);
228 $this->writeRecords_records($pid);
229 // Finally all the file and DB record references must be fixed. This is done after all records have supposedly been written to database:
230 // $this->import_mapId will indicate two things: 1) that a record WAS written to db and 2) that it has got a new id-number.
231 $this->setRelations();
232 // And when all DB relations are in place, we can fix file and DB relations in flexform fields (since data structures often depends on relations to a DS record):
233 $this->setFlexFormRelations();
234 // Unlink temporary files:
235 $this->unlinkTempFiles();
236 // Finally, traverse all records and process softreferences with substitution attributes.
237 $this->processSoftReferences();
238 // After all migrate records using sys_file_reference now
239 if ($this->legacyImport) {
240 $this->migrateLegacyImportRecords();
241 }
242 }
243
244 /**
245 * Imports the sys_file_storage records from internal data array.
246 *
247 * @return void
248 */
249 protected function writeSysFileStorageRecords()
250 {
251 if (!isset($this->dat['header']['records']['sys_file_storage'])) {
252 return;
253 }
254 $sysFileStorageUidsToBeResetToDefaultStorage = array();
255 foreach ($this->dat['header']['records']['sys_file_storage'] as $sysFileStorageUid => $_) {
256 $storageRecord = $this->dat['records']['sys_file_storage:' . $sysFileStorageUid]['data'];
257 // continue with Local, writable and online storage only
258 if ($storageRecord['driver'] === 'Local' && $storageRecord['is_writable'] && $storageRecord['is_online']) {
259 $useThisStorageUidInsteadOfTheOneInImport = 0;
260 /** @var $localStorage \TYPO3\CMS\Core\Resource\ResourceStorage */
261 foreach ($this->storageObjects as $localStorage) {
262 // check the available storage for Local, writable and online ones
263 if ($localStorage->getDriverType() === 'Local' && $localStorage->isWritable() && $localStorage->isOnline()) {
264 // check if there is already an identical storage present (same pathType and basePath)
265 $storageRecordConfiguration = ResourceFactory::getInstance()->convertFlexFormDataToConfigurationArray($storageRecord['configuration']);
266 $localStorageRecordConfiguration = $localStorage->getConfiguration();
267 if (
268 $storageRecordConfiguration['pathType'] === $localStorageRecordConfiguration['pathType']
269 && $storageRecordConfiguration['basePath'] === $localStorageRecordConfiguration['basePath']
270 ) {
271 // same storage is already present
272 $useThisStorageUidInsteadOfTheOneInImport = $localStorage->getUid();
273 break;
274 }
275 }
276 }
277 if ($useThisStorageUidInsteadOfTheOneInImport > 0) {
278 // same storage is already present; map the to be imported one to the present one
279 $this->import_mapId['sys_file_storage'][$sysFileStorageUid] = $useThisStorageUidInsteadOfTheOneInImport;
280 } else {
281 // Local, writable and online storage. Is allowed to be used to later write files in.
282 $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);
283 }
284 } else {
285 // Storage with non Local drivers could be imported but must not be used to saves files in, because you
286 // could not be sure, that this is supported. The default storage will be used in this case.
287 // It could happen that non writable and non online storage will be created as dupes because you could not
288 // check the detailed configuration options at this point
289 $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);
290 $sysFileStorageUidsToBeResetToDefaultStorage[] = $sysFileStorageUid;
291 }
292 }
293
294 // Importing the added ones
295 $tce = $this->getNewTCE();
296 // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.
297 $tce->reverseOrder = 1;
298 $tce->isImporting = true;
299 $tce->start($this->import_data, array());
300 $tce->process_datamap();
301 $this->addToMapId($tce->substNEWwithIDs);
302
303 $defaultStorageUid = null;
304 // get default storage
305 $defaultStorage = ResourceFactory::getInstance()->getDefaultStorage();
306 if ($defaultStorage !== null) {
307 $defaultStorageUid = $defaultStorage->getUid();
308 }
309 foreach ($sysFileStorageUidsToBeResetToDefaultStorage as $sysFileStorageUidToBeResetToDefaultStorage) {
310 $this->import_mapId['sys_file_storage'][$sysFileStorageUidToBeResetToDefaultStorage] = $defaultStorageUid;
311 }
312
313 // unset the sys_file_storage records to prevent an import in writeRecords_records
314 unset($this->dat['header']['records']['sys_file_storage']);
315 }
316
317 /**
318 * Imports the sys_file records and the binary files data from internal data array.
319 *
320 * @return void
321 */
322 protected function writeSysFileRecords()
323 {
324 if (!isset($this->dat['header']['records']['sys_file'])) {
325 return;
326 }
327 $this->addGeneralErrorsByTable('sys_file');
328
329 // fetch fresh storage records from database
330 $storageRecords = $this->fetchStorageRecords();
331
332 $defaultStorage = ResourceFactory::getInstance()->getDefaultStorage();
333
334 $sanitizedFolderMappings = array();
335
336 foreach ($this->dat['header']['records']['sys_file'] as $sysFileUid => $_) {
337 $fileRecord = $this->dat['records']['sys_file:' . $sysFileUid]['data'];
338
339 $temporaryFile = null;
340 // check if there is the right file already in the local folder
341 if ($this->filesPathForImport !== null) {
342 if (is_file($this->filesPathForImport . '/' . $fileRecord['sha1']) && sha1_file($this->filesPathForImport . '/' . $fileRecord['sha1']) === $fileRecord['sha1']) {
343 $temporaryFile = $this->filesPathForImport . '/' . $fileRecord['sha1'];
344 }
345 }
346
347 // save file to disk
348 if ($temporaryFile === null) {
349 $fileId = md5($fileRecord['storage'] . ':' . $fileRecord['identifier_hash']);
350 $temporaryFile = $this->writeTemporaryFileFromData($fileId);
351 if ($temporaryFile === null) {
352 // error on writing the file. Error message was already added
353 continue;
354 }
355 }
356
357 $originalStorageUid = $fileRecord['storage'];
358 $useStorageFromStorageRecords = false;
359
360 // replace storage id, if an alternative one was registered
361 if (isset($this->import_mapId['sys_file_storage'][$fileRecord['storage']])) {
362 $fileRecord['storage'] = $this->import_mapId['sys_file_storage'][$fileRecord['storage']];
363 $useStorageFromStorageRecords = true;
364 }
365
366 if (empty($fileRecord['storage']) && !$this->isFallbackStorage($fileRecord['storage'])) {
367 // no storage for the file is defined, mostly because of a missing default storage.
368 $this->error('Error: No storage for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $originalStorageUid . '"');
369 continue;
370 }
371
372 // using a storage from the local storage is only allowed, if the uid is present in the
373 // mapping. Only in this case we could be sure, that it's a local, online and writable storage.
374 if ($useStorageFromStorageRecords && isset($storageRecords[$fileRecord['storage']])) {
375 /** @var $storage \TYPO3\CMS\Core\Resource\ResourceStorage */
376 $storage = ResourceFactory::getInstance()->getStorageObject($fileRecord['storage'], $storageRecords[$fileRecord['storage']]);
377 } elseif ($this->isFallbackStorage($fileRecord['storage'])) {
378 $storage = ResourceFactory::getInstance()->getStorageObject(0);
379 } elseif ($defaultStorage !== null) {
380 $storage = $defaultStorage;
381 } else {
382 $this->error('Error: No storage available for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
383 continue;
384 }
385
386 $newFile = null;
387
388 // check, if there is an identical file
389 try {
390 if ($storage->hasFile($fileRecord['identifier'])) {
391 $file = $storage->getFile($fileRecord['identifier']);
392 if ($file->getSha1() === $fileRecord['sha1']) {
393 $newFile = $file;
394 }
395 }
396 } catch (Exception $e) {
397 }
398
399 if ($newFile === null) {
400 $folderName = PathUtility::dirname(ltrim($fileRecord['identifier'], '/'));
401 if (in_array($folderName, $sanitizedFolderMappings)) {
402 $folderName = $sanitizedFolderMappings[$folderName];
403 }
404 if (!$storage->hasFolder($folderName)) {
405 try {
406 $importFolder = $storage->createFolder($folderName);
407 if ($importFolder->getIdentifier() !== $folderName && !in_array($folderName, $sanitizedFolderMappings)) {
408 $sanitizedFolderMappings[$folderName] = $importFolder->getIdentifier();
409 }
410 } catch (Exception $e) {
411 $this->error('Error: Folder could not be created for file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
412 continue;
413 }
414 } else {
415 $importFolder = $storage->getFolder($folderName);
416 }
417
418 try {
419 /** @var $newFile File */
420 $newFile = $storage->addFile($temporaryFile, $importFolder, $fileRecord['name']);
421 } catch (Exception $e) {
422 $this->error('Error: File could not be added to the storage: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
423 continue;
424 }
425
426 if ($newFile->getSha1() !== $fileRecord['sha1']) {
427 $this->error('Error: The hash of the written file is not identical to the import data! File could be corrupted! File: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
428 }
429 }
430
431 // save the new uid in the import id map
432 $this->import_mapId['sys_file'][$fileRecord['uid']] = $newFile->getUid();
433 $this->fixUidLocalInSysFileReferenceRecords($fileRecord['uid'], $newFile->getUid());
434 }
435
436 // unset the sys_file records to prevent an import in writeRecords_records
437 unset($this->dat['header']['records']['sys_file']);
438 }
439
440 /**
441 * Checks if the $storageId is the id of the fallback storage
442 *
443 * @param int|string $storageId
444 * @return bool
445 */
446 protected function isFallbackStorage($storageId)
447 {
448 return $storageId === 0 || $storageId === '0';
449 }
450
451 /**
452 * Normally the importer works like the following:
453 * Step 1: import the records with cleared field values of relation fields (see addSingle())
454 * Step 2: update the records with the right relation ids (see setRelations())
455 *
456 * In step 2 the saving fields of type "relation to sys_file_reference" checks the related sys_file_reference
457 * record (created in step 1) with the FileExtensionFilter for matching file extensions of the related file.
458 * To make this work correct, the uid_local of sys_file_reference records has to be not empty AND has to
459 * relate to the correct (imported) sys_file record uid!!!
460 *
461 * This is fixed here.
462 *
463 * @param int $oldFileUid
464 * @param int $newFileUid
465 * @return void
466 */
467 protected function fixUidLocalInSysFileReferenceRecords($oldFileUid, $newFileUid)
468 {
469 if (!isset($this->dat['header']['records']['sys_file_reference'])) {
470 return;
471 }
472
473 foreach ($this->dat['header']['records']['sys_file_reference'] as $sysFileReferenceUid => $_) {
474 $fileReferenceRecord = $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'];
475 if ($fileReferenceRecord['uid_local'] == $oldFileUid) {
476 $fileReferenceRecord['uid_local'] = $newFileUid;
477 $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'] = $fileReferenceRecord;
478 }
479 }
480 }
481
482 /**
483 * Initializes the folder for legacy imports as subfolder of backend users default upload folder
484 *
485 * @return void
486 */
487 protected function initializeLegacyImportFolder()
488 {
489 /** @var \TYPO3\CMS\Core\Resource\Folder $folder */
490 $folder = $this->getBackendUser()->getDefaultUploadFolder();
491 if ($folder === false) {
492 $this->error('Error: the backend users default upload folder is missing! No files will be imported!');
493 }
494 if (!$folder->hasFolder($this->legacyImportTargetPath)) {
495 try {
496 $this->legacyImportFolder = $folder->createFolder($this->legacyImportTargetPath);
497 } catch (Exception $e) {
498 $this->error('Error: the import folder in the default upload folder could not be created! No files will be imported!');
499 }
500 } else {
501 $this->legacyImportFolder = $folder->getSubFolder($this->legacyImportTargetPath);
502 }
503 }
504
505 /**
506 * Fetched fresh storage records from database because the new imported
507 * ones are not in cached data of the StorageRepository
508 *
509 * @return bool|array
510 */
511 protected function fetchStorageRecords()
512 {
513 $whereClause = BackendUtility::BEenableFields('sys_file_storage');
514 $whereClause .= BackendUtility::deleteClause('sys_file_storage');
515
516 $rows = $this->getDatabaseConnection()->exec_SELECTgetRows(
517 '*',
518 'sys_file_storage',
519 '1=1' . $whereClause,
520 '',
521 '',
522 '',
523 'uid'
524 );
525
526 return $rows;
527 }
528
529 /**
530 * Writes the file from import array to temp dir and returns the filename of it.
531 *
532 * @param string $fileId
533 * @param string $dataKey
534 * @return string Absolute filename of the temporary filename of the file
535 */
536 protected function writeTemporaryFileFromData($fileId, $dataKey = 'files_fal')
537 {
538 $temporaryFilePath = null;
539 if (is_array($this->dat[$dataKey][$fileId])) {
540 $temporaryFilePathInternal = GeneralUtility::tempnam('import_temp_');
541 GeneralUtility::writeFile($temporaryFilePathInternal, $this->dat[$dataKey][$fileId]['content']);
542 clearstatcache();
543 if (@is_file($temporaryFilePathInternal)) {
544 $this->unlinkFiles[] = $temporaryFilePathInternal;
545 if (filesize($temporaryFilePathInternal) == $this->dat[$dataKey][$fileId]['filesize']) {
546 $temporaryFilePath = $temporaryFilePathInternal;
547 } else {
548 $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' had a size (' . filesize($temporaryFilePathInternal) . ') different from the original (' . $this->dat[$dataKey][$fileId]['filesize'] . ')');
549 }
550 } else {
551 $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' was not written as it should have been!');
552 }
553 } else {
554 $this->error('Error: No file found for ID ' . $fileId);
555 }
556 return $temporaryFilePath;
557 }
558
559 /**
560 * Writing pagetree/pages to database:
561 *
562 * @param int $pid PID in which to import. If the operation is an update operation, the root of the page tree inside will be moved to this PID unless it is the same as the root page from the import
563 * @return void
564 * @see writeRecords_records()
565 */
566 public function writeRecords_pages($pid)
567 {
568 // First, write page structure if any:
569 if (is_array($this->dat['header']['records']['pages'])) {
570 $this->addGeneralErrorsByTable('pages');
571 // $pageRecords is a copy of the pages array in the imported file. Records here are unset one by one when the addSingle function is called.
572 $pageRecords = $this->dat['header']['records']['pages'];
573 $this->import_data = array();
574 // First add page tree if any
575 if (is_array($this->dat['header']['pagetree'])) {
576 $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);
577 foreach ($pagesFromTree as $uid) {
578 $thisRec = $this->dat['header']['records']['pages'][$uid];
579 // PID: Set the main $pid, unless a NEW-id is found
580 $setPid = isset($this->import_newId_pids[$thisRec['pid']]) ? $this->import_newId_pids[$thisRec['pid']] : $pid;
581 $this->addSingle('pages', $uid, $setPid);
582 unset($pageRecords[$uid]);
583 }
584 }
585 // Then add all remaining pages not in tree on root level:
586 if (!empty($pageRecords)) {
587 $remainingPageUids = array_keys($pageRecords);
588 foreach ($remainingPageUids as $pUid) {
589 $this->addSingle('pages', $pUid, $pid);
590 }
591 }
592 // Now write to database:
593 $tce = $this->getNewTCE();
594 $tce->isImporting = true;
595 $this->callHook('before_writeRecordsPages', array(
596 'tce' => &$tce,
597 'data' => &$this->import_data
598 ));
599 $tce->suggestedInsertUids = $this->suggestedInsertUids;
600 $tce->start($this->import_data, array());
601 $tce->process_datamap();
602 $this->callHook('after_writeRecordsPages', array(
603 'tce' => &$tce
604 ));
605 // post-processing: Registering new ids (end all tcemain sessions with this)
606 $this->addToMapId($tce->substNEWwithIDs);
607 // In case of an update, order pages from the page tree correctly:
608 if ($this->update && is_array($this->dat['header']['pagetree'])) {
609 $this->writeRecords_pages_order();
610 }
611 }
612 }
613
614 /**
615 * Organize all updated pages in page tree so they are related like in the import file
616 * Only used for updates and when $this->dat['header']['pagetree'] is an array.
617 *
618 * @return void
619 * @access private
620 * @see writeRecords_pages(), writeRecords_records_order()
621 */
622 public function writeRecords_pages_order()
623 {
624 $cmd_data = array();
625 // Get uid-pid relations and traverse them in order to map to possible new IDs
626 $pidsFromTree = $this->flatInversePageTree_pid($this->dat['header']['pagetree']);
627 foreach ($pidsFromTree as $origPid => $newPid) {
628 if ($newPid >= 0 && $this->dontIgnorePid('pages', $origPid)) {
629 // If the page had a new id (because it was created) use that instead!
630 if (substr($this->import_newId_pids[$origPid], 0, 3) === 'NEW') {
631 if ($this->import_mapId['pages'][$origPid]) {
632 $mappedPid = $this->import_mapId['pages'][$origPid];
633 $cmd_data['pages'][$mappedPid]['move'] = $newPid;
634 }
635 } else {
636 $cmd_data['pages'][$origPid]['move'] = $newPid;
637 }
638 }
639 }
640 // Execute the move commands if any:
641 if (!empty($cmd_data)) {
642 $tce = $this->getNewTCE();
643 $this->callHook('before_writeRecordsPagesOrder', array(
644 'tce' => &$tce,
645 'data' => &$cmd_data
646 ));
647 $tce->start(array(), $cmd_data);
648 $tce->process_cmdmap();
649 $this->callHook('after_writeRecordsPagesOrder', array(
650 'tce' => &$tce
651 ));
652 }
653 }
654
655 /**
656 * Recursively flattening the idH array, setting PIDs as values
657 *
658 * @param array $idH Page uid hierarchy
659 * @param array $a Accumulation array of pages (internal, don't set from outside)
660 * @param int $pid PID value (internal)
661 * @return array Array with uid-pid pairs for all pages in the page tree.
662 * @see ImportExport::flatInversePageTree()
663 */
664 public function flatInversePageTree_pid($idH, $a = array(), $pid = -1)
665 {
666 if (is_array($idH)) {
667 $idH = array_reverse($idH);
668 foreach ($idH as $v) {
669 $a[$v['uid']] = $pid;
670 if (is_array($v['subrow'])) {
671 $a = $this->flatInversePageTree_pid($v['subrow'], $a, $v['uid']);
672 }
673 }
674 }
675 return $a;
676 }
677
678 /**
679 * Write all database records except pages (writtein in writeRecords_pages())
680 *
681 * @param int $pid Page id in which to import
682 * @return void
683 * @see writeRecords_pages()
684 */
685 public function writeRecords_records($pid)
686 {
687 // Write the rest of the records
688 $this->import_data = array();
689 if (is_array($this->dat['header']['records'])) {
690 foreach ($this->dat['header']['records'] as $table => $recs) {
691 $this->addGeneralErrorsByTable($table);
692 if ($table != 'pages') {
693 foreach ($recs as $uid => $thisRec) {
694 // PID: Set the main $pid, unless a NEW-id is found
695 $setPid = isset($this->import_mapId['pages'][$thisRec['pid']])
696 ? (int)$this->import_mapId['pages'][$thisRec['pid']]
697 : (int)$pid;
698 if (is_array($GLOBALS['TCA'][$table]) && isset($GLOBALS['TCA'][$table]['ctrl']['rootLevel'])) {
699 $rootLevelSetting = (int)$GLOBALS['TCA'][$table]['ctrl']['rootLevel'];
700 if ($rootLevelSetting === 1) {
701 $setPid = 0;
702 } elseif ($rootLevelSetting === 0 && $setPid === 0) {
703 $this->error('Error: Record type ' . $table . ' is not allowed on pid 0');
704 continue;
705 }
706 }
707 // Add record:
708 $this->addSingle($table, $uid, $setPid);
709 }
710 }
711 }
712 } else {
713 $this->error('Error: No records defined in internal data array.');
714 }
715 // Now write to database:
716 $tce = $this->getNewTCE();
717 $this->callHook('before_writeRecordsRecords', array(
718 'tce' => &$tce,
719 'data' => &$this->import_data
720 ));
721 $tce->suggestedInsertUids = $this->suggestedInsertUids;
722 // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.
723 $tce->reverseOrder = 1;
724 $tce->isImporting = true;
725 $tce->start($this->import_data, array());
726 $tce->process_datamap();
727 $this->callHook('after_writeRecordsRecords', array(
728 'tce' => &$tce
729 ));
730 // post-processing: Removing files and registering new ids (end all tcemain sessions with this)
731 $this->addToMapId($tce->substNEWwithIDs);
732 // In case of an update, order pages from the page tree correctly:
733 if ($this->update) {
734 $this->writeRecords_records_order($pid);
735 }
736 }
737
738 /**
739 * Organize all updated record to their new positions.
740 * Only used for updates
741 *
742 * @param int $mainPid Main PID into which we import.
743 * @return void
744 * @access private
745 * @see writeRecords_records(), writeRecords_pages_order()
746 */
747 public function writeRecords_records_order($mainPid)
748 {
749 $cmd_data = array();
750 if (is_array($this->dat['header']['pagetree'])) {
751 $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);
752 } else {
753 $pagesFromTree = array();
754 }
755 if (is_array($this->dat['header']['pid_lookup'])) {
756 foreach ($this->dat['header']['pid_lookup'] as $pid => $recList) {
757 $newPid = isset($this->import_mapId['pages'][$pid]) ? $this->import_mapId['pages'][$pid] : $mainPid;
758 if (MathUtility::canBeInterpretedAsInteger($newPid)) {
759 foreach ($recList as $tableName => $uidList) {
760 // If $mainPid===$newPid then we are on root level and we can consider to move pages as well!
761 // (they will not be in the page tree!)
762 if (($tableName != 'pages' || !$pagesFromTree[$pid]) && is_array($uidList)) {
763 $uidList = array_reverse(array_keys($uidList));
764 foreach ($uidList as $uid) {
765 if ($this->dontIgnorePid($tableName, $uid)) {
766 $cmd_data[$tableName][$uid]['move'] = $newPid;
767 } else {
768 }
769 }
770 }
771 }
772 }
773 }
774 }
775 // Execute the move commands if any:
776 if (!empty($cmd_data)) {
777 $tce = $this->getNewTCE();
778 $this->callHook('before_writeRecordsRecordsOrder', array(
779 'tce' => &$tce,
780 'data' => &$cmd_data
781 ));
782 $tce->start(array(), $cmd_data);
783 $tce->process_cmdmap();
784 $this->callHook('after_writeRecordsRecordsOrder', array(
785 'tce' => &$tce
786 ));
787 }
788 }
789
790 /**
791 * Adds a single record to the $importData array. Also copies files to tempfolder.
792 * However all File/DB-references and flexform field contents are set to blank for now!
793 * That is done with setRelations() later
794 *
795 * @param string $table Table name (from import memory)
796 * @param int $uid Record UID (from import memory)
797 * @param int $pid Page id
798 * @return void
799 * @see writeRecords()
800 */
801 public function addSingle($table, $uid, $pid)
802 {
803 if ($this->import_mode[$table . ':' . $uid] === 'exclude') {
804 return;
805 }
806 $record = $this->dat['records'][$table . ':' . $uid]['data'];
807 if (is_array($record)) {
808 if ($this->update && $this->doesRecordExist($table, $uid) && $this->import_mode[$table . ':' . $uid] !== 'as_new') {
809 $ID = $uid;
810 } elseif ($table === 'sys_file_metadata' && $record['sys_language_uid'] == '0' && $this->import_mapId['sys_file'][$record['file']]) {
811 // on adding sys_file records the belonging sys_file_metadata record was also created
812 // if there is one the record need to be overwritten instead of creating a new one.
813 $recordInDatabase = $this->getDatabaseConnection()->exec_SELECTgetSingleRow(
814 'uid',
815 'sys_file_metadata',
816 'file = ' . $this->import_mapId['sys_file'][$record['file']] . ' AND sys_language_uid = 0 AND pid = 0'
817 );
818 // if no record could be found, $this->import_mapId['sys_file'][$record['file']] is pointing
819 // to a file, that was already there, thus a new metadata record should be created
820 if (is_array($recordInDatabase)) {
821 $this->import_mapId['sys_file_metadata'][$record['uid']] = $recordInDatabase['uid'];
822 $ID = $recordInDatabase['uid'];
823 } else {
824 $ID = StringUtility::getUniqueId('NEW');
825 }
826 } else {
827 $ID = StringUtility::getUniqueId('NEW');
828 }
829 $this->import_newId[$table . ':' . $ID] = array('table' => $table, 'uid' => $uid);
830 if ($table == 'pages') {
831 $this->import_newId_pids[$uid] = $ID;
832 }
833 // Set main record data:
834 $this->import_data[$table][$ID] = $record;
835 $this->import_data[$table][$ID]['tx_impexp_origuid'] = $this->import_data[$table][$ID]['uid'];
836 // Reset permission data:
837 if ($table === 'pages') {
838 // Have to reset the user/group IDs so pages are owned by importing user. Otherwise strange things may happen for non-admins!
839 unset($this->import_data[$table][$ID]['perms_userid']);
840 unset($this->import_data[$table][$ID]['perms_groupid']);
841 }
842 // PID and UID:
843 unset($this->import_data[$table][$ID]['uid']);
844 // Updates:
845 if (MathUtility::canBeInterpretedAsInteger($ID)) {
846 unset($this->import_data[$table][$ID]['pid']);
847 } else {
848 // Inserts:
849 $this->import_data[$table][$ID]['pid'] = $pid;
850 if (($this->import_mode[$table . ':' . $uid] === 'force_uid' && $this->update || $this->force_all_UIDS) && $this->getBackendUser()->isAdmin()) {
851 $this->import_data[$table][$ID]['uid'] = $uid;
852 $this->suggestedInsertUids[$table . ':' . $uid] = 'DELETE';
853 }
854 }
855 // Setting db/file blank:
856 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
857 switch ((string)$config['type']) {
858 case 'db':
859
860 case 'file':
861 // Fixed later in ->setRelations() [because we need to know ALL newly created IDs before we can map relations!]
862 // In the meantime we set NO values for relations.
863 //
864 // BUT for field uid_local of table sys_file_reference the relation MUST not be cleared here,
865 // because the value is already the uid of the right imported sys_file record.
866 // @see fixUidLocalInSysFileReferenceRecords()
867 // If it's empty or a uid to another record the FileExtensionFilter will throw an exception or
868 // delete the reference record if the file extension of the related record doesn't match.
869 if ($table !== 'sys_file_reference' && $field !== 'uid_local') {
870 $this->import_data[$table][$ID][$field] = '';
871 }
872 break;
873 case 'flex':
874 // Fixed later in setFlexFormRelations()
875 // In the meantime we set NO value for flexforms - this is mainly because file references
876 // inside will not be processed properly; In fact references will point to no file
877 // or existing files (in which case there will be double-references which is a big problem of course!)
878 $this->import_data[$table][$ID][$field] = '';
879 break;
880 }
881 }
882 } elseif ($table . ':' . $uid != 'pages:0') {
883 // On root level we don't want this error message.
884 $this->error('Error: no record was found in data array!');
885 }
886 }
887
888 /**
889 * Registers the substNEWids in memory.
890 *
891 * @param array $substNEWwithIDs From tcemain to be merged into internal mapping variable in this object
892 * @return void
893 * @see writeRecords()
894 */
895 public function addToMapId($substNEWwithIDs)
896 {
897 foreach ($this->import_data as $table => $recs) {
898 foreach ($recs as $id => $value) {
899 $old_uid = $this->import_newId[$table . ':' . $id]['uid'];
900 if (isset($substNEWwithIDs[$id])) {
901 $this->import_mapId[$table][$old_uid] = $substNEWwithIDs[$id];
902 } elseif ($this->update) {
903 // Map same ID to same ID....
904 $this->import_mapId[$table][$old_uid] = $id;
905 } else {
906 // if $this->import_mapId contains already the right mapping, skip the error msg.
907 // See special handling of sys_file_metadata in addSingle() => nothing to do
908 if (!($table === 'sys_file_metadata' && isset($this->import_mapId[$table][$old_uid]) && $this->import_mapId[$table][$old_uid] == $id)) {
909 $this->error('Possible error: ' . $table . ':' . $old_uid . ' had no new id assigned to it. This indicates that the record was not added to database during import. Please check changelog!');
910 }
911 }
912 }
913 }
914 }
915
916 /**
917 * Returns a new $TCE object
918 *
919 * @return DataHandler $TCE object
920 */
921 public function getNewTCE()
922 {
923 $tce = GeneralUtility::makeInstance(DataHandler::class);
924 $tce->dontProcessTransformations = 1;
925 $tce->enableLogging = $this->enableLogging;
926 $tce->alternativeFileName = $this->alternativeFileName;
927 $tce->alternativeFilePath = $this->alternativeFilePath;
928 return $tce;
929 }
930
931 /**
932 * Cleaning up all the temporary files stored in typo3temp/ folder
933 *
934 * @return void
935 */
936 public function unlinkTempFiles()
937 {
938 foreach ($this->unlinkFiles as $fileName) {
939 if (GeneralUtility::isFirstPartOfStr($fileName, PATH_site . 'typo3temp/')) {
940 GeneralUtility::unlink_tempfile($fileName);
941 clearstatcache();
942 if (is_file($fileName)) {
943 $this->error('Error: ' . $fileName . ' was NOT unlinked as it should have been!');
944 }
945 } else {
946 $this->error('Error: ' . $fileName . ' was not in temp-path. Not removed!');
947 }
948 }
949 $this->unlinkFiles = array();
950 }
951
952 /***************************
953 * Import / Relations setting
954 ***************************/
955
956 /**
957 * At the end of the import process all file and DB relations should be set properly (that is relations
958 * to imported records are all re-created so imported records are correctly related again)
959 * Relations in flexform fields are processed in setFlexFormRelations() after this function
960 *
961 * @return void
962 * @see setFlexFormRelations()
963 */
964 public function setRelations()
965 {
966 $updateData = array();
967 // import_newId contains a register of all records that was in the import memorys "records" key
968 foreach ($this->import_newId as $nId => $dat) {
969 $table = $dat['table'];
970 $uid = $dat['uid'];
971 // original UID - NOT the new one!
972 // If the record has been written and received a new id, then proceed:
973 if (is_array($this->import_mapId[$table]) && isset($this->import_mapId[$table][$uid])) {
974 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
975 if (is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {
976 $thisNewPageUid = 0;
977 if ($this->legacyImport) {
978 if ($table != 'pages') {
979 $oldPid = $this->dat['records'][$table . ':' . $uid]['data']['pid'];
980 $thisNewPageUid = BackendUtility::wsMapId($table, $this->import_mapId['pages'][$oldPid]);
981 } else {
982 $thisNewPageUid = $thisNewUid;
983 }
984 }
985 // Traverse relation fields of each record
986 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
987 // uid_local of sys_file_reference needs no update because the correct reference uid was already written
988 // @see ImportExport::fixUidLocalInSysFileReferenceRecords()
989 if ($table === 'sys_file_reference' && $field === 'uid_local') {
990 continue;
991 }
992 switch ((string)$config['type']) {
993 case 'db':
994 if (is_array($config['itemArray']) && !empty($config['itemArray'])) {
995 $itemConfig = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
996 $valArray = $this->setRelations_db($config['itemArray'], $itemConfig);
997 $updateData[$table][$thisNewUid][$field] = implode(',', $valArray);
998 }
999 break;
1000 case 'file':
1001 if (is_array($config['newValueFiles']) && !empty($config['newValueFiles'])) {
1002 $valArr = array();
1003 foreach ($config['newValueFiles'] as $fI) {
1004 $valArr[] = $this->import_addFileNameToBeCopied($fI);
1005 }
1006 if ($this->legacyImport && $this->legacyImportFolder === null && isset($this->legacyImportMigrationTables[$table][$field])) {
1007 // Do nothing - the legacy import folder is missing
1008 } elseif ($this->legacyImport && $this->legacyImportFolder !== null && isset($this->legacyImportMigrationTables[$table][$field])) {
1009 $refIds = array();
1010 foreach ($valArr as $tempFile) {
1011 $fileName = $this->alternativeFileName[$tempFile];
1012 $fileObject = null;
1013
1014 try {
1015 // check, if there is alreay the same file in the folder
1016 if ($this->legacyImportFolder->hasFile($fileName)) {
1017 $fileStorage = $this->legacyImportFolder->getStorage();
1018 $file = $fileStorage->getFile($this->legacyImportFolder->getIdentifier() . $fileName);
1019 if ($file->getSha1() === sha1_file($tempFile)) {
1020 $fileObject = $file;
1021 }
1022 }
1023 } catch (Exception $e) {
1024 }
1025
1026 if ($fileObject === null) {
1027 try {
1028 $fileObject = $this->legacyImportFolder->addFile($tempFile, $fileName, DuplicationBehavior::RENAME);
1029 } catch (Exception $e) {
1030 $this->error('Error: no file could be added to the storage for file name' . $this->alternativeFileName[$tempFile]);
1031 }
1032 }
1033 if ($fileObject !== null) {
1034 $refId = StringUtility::getUniqueId('NEW');
1035 $refIds[] = $refId;
1036 $updateData['sys_file_reference'][$refId] = array(
1037 'uid_local' => $fileObject->getUid(),
1038 'uid_foreign' => $thisNewUid, // uid of your content record
1039 'tablenames' => $table,
1040 'fieldname' => $field,
1041 'pid' => $thisNewPageUid, // parent id of the parent page
1042 'table_local' => 'sys_file',
1043 );
1044 }
1045 }
1046 $updateData[$table][$thisNewUid][$field] = implode(',', $refIds);
1047 if (!empty($this->legacyImportMigrationTables[$table][$field])) {
1048 $this->legacyImportMigrationRecords[$table][$thisNewUid][$field] = $refIds;
1049 }
1050 } else {
1051 $updateData[$table][$thisNewUid][$field] = implode(',', $valArr);
1052 }
1053 }
1054 break;
1055 }
1056 }
1057 } else {
1058 $this->error('Error: no record was found in data array!');
1059 }
1060 } else {
1061 $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')');
1062 }
1063 }
1064 if (!empty($updateData)) {
1065 $tce = $this->getNewTCE();
1066 $tce->isImporting = true;
1067 $this->callHook('before_setRelation', array(
1068 'tce' => &$tce,
1069 'data' => &$updateData
1070 ));
1071 $tce->start($updateData, array());
1072 $tce->process_datamap();
1073 // Replace the temporary "NEW" ids with the final ones.
1074 foreach ($this->legacyImportMigrationRecords as $table => $records) {
1075 foreach ($records as $uid => $fields) {
1076 foreach ($fields as $field => $referenceIds) {
1077 foreach ($referenceIds as $key => $referenceId) {
1078 $this->legacyImportMigrationRecords[$table][$uid][$field][$key] = $tce->substNEWwithIDs[$referenceId];
1079 }
1080 }
1081 }
1082 }
1083 $this->callHook('after_setRelations', array(
1084 'tce' => &$tce
1085 ));
1086 }
1087 }
1088
1089 /**
1090 * Maps relations for database
1091 *
1092 * @param array $itemArray Array of item sets (table/uid) from a dbAnalysis object
1093 * @param array $itemConfig Array of TCA config of the field the relation to be set on
1094 * @return array Array with values [table]_[uid] or [uid] for field of type group / internal_type file_reference. These values have the regular tcemain-input group/select type which means they will automatically be processed into a uid-list or MM relations.
1095 */
1096 public function setRelations_db($itemArray, $itemConfig)
1097 {
1098 $valArray = array();
1099 foreach ($itemArray as $relDat) {
1100 if (is_array($this->import_mapId[$relDat['table']]) && isset($this->import_mapId[$relDat['table']][$relDat['id']])) {
1101 // Since non FAL file relation type group internal_type file_reference are handled as reference to
1102 // sys_file records Datahandler requires the value as uid of the the related sys_file record only
1103 if ($itemConfig['type'] === 'group' && $itemConfig['internal_type'] === 'file_reference') {
1104 $value = $this->import_mapId[$relDat['table']][$relDat['id']];
1105 } elseif ($itemConfig['type'] === 'input' && isset($itemConfig['wizards']['link'])) {
1106 // If an input field has a relation to a sys_file record this need to be converted back to
1107 // the public path. But use getPublicUrl here, because could normally only be a local file path.
1108 $fileUid = $this->import_mapId[$relDat['table']][$relDat['id']];
1109 // Fallback value
1110 $value = 'file:' . $fileUid;
1111 try {
1112 $file = ResourceFactory::getInstance()->retrieveFileOrFolderObject($fileUid);
1113 } catch (\Exception $e) {
1114 $file = null;
1115 }
1116 if ($file instanceof FileInterface) {
1117 $value = $file->getPublicUrl();
1118 }
1119 } else {
1120 $value = $relDat['table'] . '_' . $this->import_mapId[$relDat['table']][$relDat['id']];
1121 }
1122 $valArray[] = $value;
1123 } elseif ($this->isTableStatic($relDat['table']) || $this->isExcluded($relDat['table'], $relDat['id']) || $relDat['id'] < 0) {
1124 // Checking for less than zero because some select types could contain negative values,
1125 // eg. fe_groups (-1, -2) and sys_language (-1 = ALL languages). This must be handled on both export and import.
1126 $valArray[] = $relDat['table'] . '_' . $relDat['id'];
1127 } else {
1128 $this->error('Lost relation: ' . $relDat['table'] . ':' . $relDat['id']);
1129 }
1130 }
1131 return $valArray;
1132 }
1133
1134 /**
1135 * Writes the file from import array to temp dir and returns the filename of it.
1136 *
1137 * @param array $fI File information with three keys: "filename" = filename without path, "ID_absFile" = absolute filepath to the file (including the filename), "ID" = md5 hash of "ID_absFile
1138 * @return string|NULL Absolute filename of the temporary filename of the file. In ->alternativeFileName the original name is set.
1139 */
1140 public function import_addFileNameToBeCopied($fI)
1141 {
1142 if (is_array($this->dat['files'][$fI['ID']])) {
1143 $tmpFile = null;
1144 // check if there is the right file already in the local folder
1145 if ($this->filesPathForImport !== null) {
1146 if (is_file($this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5']) &&
1147 md5_file($this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5']) === $this->dat['files'][$fI['ID']]['content_md5']) {
1148 $tmpFile = $this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5'];
1149 }
1150 }
1151 if ($tmpFile === null) {
1152 $tmpFile = GeneralUtility::tempnam('import_temp_');
1153 GeneralUtility::writeFile($tmpFile, $this->dat['files'][$fI['ID']]['content']);
1154 }
1155 clearstatcache();
1156 if (@is_file($tmpFile)) {
1157 $this->unlinkFiles[] = $tmpFile;
1158 if (filesize($tmpFile) == $this->dat['files'][$fI['ID']]['filesize']) {
1159 $this->alternativeFileName[$tmpFile] = $fI['filename'];
1160 $this->alternativeFilePath[$tmpFile] = $this->dat['files'][$fI['ID']]['relFileRef'];
1161 return $tmpFile;
1162 } else {
1163 $this->error('Error: temporary file ' . $tmpFile . ' had a size (' . filesize($tmpFile) . ') different from the original (' . $this->dat['files'][$fI['ID']]['filesize'] . ')');
1164 }
1165 } else {
1166 $this->error('Error: temporary file ' . $tmpFile . ' was not written as it should have been!');
1167 }
1168 } else {
1169 $this->error('Error: No file found for ID ' . $fI['ID']);
1170 }
1171 return null;
1172 }
1173
1174 /**
1175 * After all DB relations has been set in the end of the import (see setRelations()) then it is time to correct all relations inside of FlexForm fields.
1176 * The reason for doing this after is that the setting of relations may affect (quite often!) which data structure is used for the flexforms field!
1177 *
1178 * @return void
1179 * @see setRelations()
1180 */
1181 public function setFlexFormRelations()
1182 {
1183 $updateData = array();
1184 // import_newId contains a register of all records that was in the import memorys "records" key
1185 foreach ($this->import_newId as $nId => $dat) {
1186 $table = $dat['table'];
1187 $uid = $dat['uid'];
1188 // original UID - NOT the new one!
1189 // If the record has been written and received a new id, then proceed:
1190 if (!isset($this->import_mapId[$table][$uid])) {
1191 $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')');
1192 continue;
1193 }
1194
1195 if (!is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {
1196 $this->error('Error: no record was found in data array!');
1197 continue;
1198 }
1199 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
1200 // Traverse relation fields of each record
1201 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
1202 switch ((string)$config['type']) {
1203 case 'flex':
1204 // Get XML content and set as default value (string, non-processed):
1205 $updateData[$table][$thisNewUid][$field] = $this->dat['records'][$table . ':' . $uid]['data'][$field];
1206 // If there has been registered relations inside the flex form field, run processing on the content:
1207 if (!empty($config['flexFormRels']['db']) || !empty($config['flexFormRels']['file'])) {
1208 $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');
1209 // This will fetch the new row for the element (which should be updated with any references to data structures etc.)
1210 $conf = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
1211 if (is_array($origRecordRow) && is_array($conf) && $conf['type'] === 'flex') {
1212 // Get current data structure and value array:
1213 $dataStructArray = BackendUtility::getFlexFormDS($conf, $origRecordRow, $table, $field);
1214 $currentValueArray = GeneralUtility::xml2array($updateData[$table][$thisNewUid][$field]);
1215 // Do recursive processing of the XML data:
1216 $iteratorObj = GeneralUtility::makeInstance(DataHandler::class);
1217 $iteratorObj->callBackObj = $this;
1218 $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData(
1219 $currentValueArray['data'],
1220 array(),
1221 array(),
1222 $dataStructArray,
1223 array($table, $thisNewUid, $field, $config),
1224 'remapListedDBRecords_flexFormCallBack'
1225 );
1226 // The return value is set as an array which means it will be processed by tcemain for file and DB references!
1227 if (is_array($currentValueArray['data'])) {
1228 $updateData[$table][$thisNewUid][$field] = $currentValueArray;
1229 }
1230 }
1231 }
1232 break;
1233 }
1234 }
1235 }
1236 if (!empty($updateData)) {
1237 $tce = $this->getNewTCE();
1238 $tce->isImporting = true;
1239 $this->callHook('before_setFlexFormRelations', array(
1240 'tce' => &$tce,
1241 'data' => &$updateData
1242 ));
1243 $tce->start($updateData, array());
1244 $tce->process_datamap();
1245 $this->callHook('after_setFlexFormRelations', array(
1246 'tce' => &$tce
1247 ));
1248 }
1249 }
1250
1251 /**
1252 * Callback function for traversing the FlexForm structure in relation to remapping database relations
1253 *
1254 * @param array $pParams Set of parameters in numeric array: table, uid, field
1255 * @param array $dsConf TCA config for field (from Data Structure of course)
1256 * @param string $dataValue Field value (from FlexForm XML)
1257 * @param string $dataValue_ext1 Not used
1258 * @param string $dataValue_ext2 Not used
1259 * @param string $path Path of where the data structure of the element is found
1260 * @return array Array where the "value" key carries the value.
1261 * @see setFlexFormRelations()
1262 */
1263 public function remapListedDBRecords_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path)
1264 {
1265 // Extract parameters:
1266 list(, , , $config) = $pParams;
1267 // In case the $path is used as index without a trailing slash we will remove that
1268 if (!is_array($config['flexFormRels']['db'][$path]) && is_array($config['flexFormRels']['db'][rtrim($path, '/')])) {
1269 $path = rtrim($path, '/');
1270 }
1271 if (is_array($config['flexFormRels']['db'][$path])) {
1272 $valArray = $this->setRelations_db($config['flexFormRels']['db'][$path], $dsConf);
1273 $dataValue = implode(',', $valArray);
1274 }
1275 if (is_array($config['flexFormRels']['file'][$path])) {
1276 $valArr = array();
1277 foreach ($config['flexFormRels']['file'][$path] as $fI) {
1278 $valArr[] = $this->import_addFileNameToBeCopied($fI);
1279 }
1280 $dataValue = implode(',', $valArr);
1281 }
1282 return array('value' => $dataValue);
1283 }
1284
1285 /**************************
1286 * Import / Soft References
1287 *************************/
1288
1289 /**
1290 * Processing of soft references
1291 *
1292 * @return void
1293 */
1294 public function processSoftReferences()
1295 {
1296 // Initialize:
1297 $inData = array();
1298 // Traverse records:
1299 if (is_array($this->dat['header']['records'])) {
1300 foreach ($this->dat['header']['records'] as $table => $recs) {
1301 foreach ($recs as $uid => $thisRec) {
1302 // If there are soft references defined, traverse those:
1303 if (isset($GLOBALS['TCA'][$table]) && is_array($thisRec['softrefs'])) {
1304 // First traversal is to collect softref configuration and split them up based on fields.
1305 // This could probably also have been done with the "records" key instead of the header.
1306 $fieldsIndex = array();
1307 foreach ($thisRec['softrefs'] as $softrefDef) {
1308 // If a substitution token is set:
1309 if ($softrefDef['field'] && is_array($softrefDef['subst']) && $softrefDef['subst']['tokenID']) {
1310 $fieldsIndex[$softrefDef['field']][$softrefDef['subst']['tokenID']] = $softrefDef;
1311 }
1312 }
1313 // The new id:
1314 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
1315 // Now, if there are any fields that require substitution to be done, lets go for that:
1316 foreach ($fieldsIndex as $field => $softRefCfgs) {
1317 if (is_array($GLOBALS['TCA'][$table]['columns'][$field])) {
1318 $conf = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
1319 if ($conf['type'] === 'flex') {
1320 // This will fetch the new row for the element (which should be updated with any references to data structures etc.)
1321 $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');
1322 if (is_array($origRecordRow)) {
1323 // Get current data structure and value array:
1324 $dataStructArray = BackendUtility::getFlexFormDS($conf, $origRecordRow, $table, $field);
1325 $currentValueArray = GeneralUtility::xml2array($origRecordRow[$field]);
1326 // Do recursive processing of the XML data:
1327 /** @var $iteratorObj DataHandler */
1328 $iteratorObj = GeneralUtility::makeInstance(DataHandler::class);
1329 $iteratorObj->callBackObj = $this;
1330 $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData($currentValueArray['data'], array(), array(), $dataStructArray, array($table, $uid, $field, $softRefCfgs), 'processSoftReferences_flexFormCallBack');
1331 // The return value is set as an array which means it will be processed by tcemain for file and DB references!
1332 if (is_array($currentValueArray['data'])) {
1333 $inData[$table][$thisNewUid][$field] = $currentValueArray;
1334 }
1335 }
1336 } else {
1337 // Get tokenizedContent string and proceed only if that is not blank:
1338 $tokenizedContent = $this->dat['records'][$table . ':' . $uid]['rels'][$field]['softrefs']['tokenizedContent'];
1339 if (strlen($tokenizedContent) && is_array($softRefCfgs)) {
1340 $inData[$table][$thisNewUid][$field] = $this->processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid);
1341 }
1342 }
1343 }
1344 }
1345 }
1346 }
1347 }
1348 }
1349 // Now write to database:
1350 $tce = $this->getNewTCE();
1351 $tce->isImporting = true;
1352 $this->callHook('before_processSoftReferences', array(
1353 'tce' => $tce,
1354 'data' => &$inData
1355 ));
1356 $tce->enableLogging = true;
1357 $tce->start($inData, array());
1358 $tce->process_datamap();
1359 $this->callHook('after_processSoftReferences', array(
1360 'tce' => $tce
1361 ));
1362 }
1363
1364 /**
1365 * Callback function for traversing the FlexForm structure in relation to remapping softreference relations
1366 *
1367 * @param array $pParams Set of parameters in numeric array: table, uid, field
1368 * @param array $dsConf TCA config for field (from Data Structure of course)
1369 * @param string $dataValue Field value (from FlexForm XML)
1370 * @param string $dataValue_ext1 Not used
1371 * @param string $dataValue_ext2 Not used
1372 * @param string $path Path of where the data structure where the element is found
1373 * @return array Array where the "value" key carries the value.
1374 * @see setFlexFormRelations()
1375 */
1376 public function processSoftReferences_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path)
1377 {
1378 // Extract parameters:
1379 list($table, $origUid, $field, $softRefCfgs) = $pParams;
1380 if (is_array($softRefCfgs)) {
1381 // First, find all soft reference configurations for this structure path (they are listed flat in the header):
1382 $thisSoftRefCfgList = array();
1383 foreach ($softRefCfgs as $sK => $sV) {
1384 if ($sV['structurePath'] === $path) {
1385 $thisSoftRefCfgList[$sK] = $sV;
1386 }
1387 }
1388 // If any was found, do processing:
1389 if (!empty($thisSoftRefCfgList)) {
1390 // Get tokenizedContent string and proceed only if that is not blank:
1391 $tokenizedContent = $this->dat['records'][$table . ':' . $origUid]['rels'][$field]['flexFormRels']['softrefs'][$path]['tokenizedContent'];
1392 if (strlen($tokenizedContent)) {
1393 $dataValue = $this->processSoftReferences_substTokens($tokenizedContent, $thisSoftRefCfgList, $table, $origUid);
1394 }
1395 }
1396 }
1397 // Return
1398 return array('value' => $dataValue);
1399 }
1400
1401 /**
1402 * Substition of softreference tokens
1403 *
1404 * @param string $tokenizedContent Content of field with soft reference tokens in.
1405 * @param array $softRefCfgs Soft reference configurations
1406 * @param string $table Table for which the processing occurs
1407 * @param string $uid UID of record from table
1408 * @return string The input content with tokens substituted according to entries in softRefCfgs
1409 */
1410 public function processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid)
1411 {
1412 // traverse each softref type for this field:
1413 foreach ($softRefCfgs as $cfg) {
1414 // Get token ID:
1415 $tokenID = $cfg['subst']['tokenID'];
1416 // Default is current token value:
1417 $insertValue = $cfg['subst']['tokenValue'];
1418 // Based on mode:
1419 switch ((string)$this->softrefCfg[$tokenID]['mode']) {
1420 case 'exclude':
1421 // Exclude is a simple passthrough of the value
1422 break;
1423 case 'editable':
1424 // Editable always picks up the value from this input array:
1425 $insertValue = $this->softrefInputValues[$tokenID];
1426 break;
1427 default:
1428 // Mapping IDs/creating files: Based on type, look up new value:
1429 switch ((string)$cfg['subst']['type']) {
1430 case 'file':
1431 // Create / Overwrite file:
1432 $insertValue = $this->processSoftReferences_saveFile($cfg['subst']['relFileName'], $cfg, $table, $uid);
1433 break;
1434 case 'db':
1435 default:
1436 // Trying to map database element if found in the mapID array:
1437 list($tempTable, $tempUid) = explode(':', $cfg['subst']['recordRef']);
1438 if (isset($this->import_mapId[$tempTable][$tempUid])) {
1439 $insertValue = BackendUtility::wsMapId($tempTable, $this->import_mapId[$tempTable][$tempUid]);
1440 // Look if reference is to a page and the original token value was NOT an integer - then we assume is was an alias and try to look up the new one!
1441 if ($tempTable === 'pages' && !MathUtility::canBeInterpretedAsInteger($cfg['subst']['tokenValue'])) {
1442 $recWithUniqueValue = BackendUtility::getRecord($tempTable, $insertValue, 'alias');
1443 if ($recWithUniqueValue['alias']) {
1444 $insertValue = $recWithUniqueValue['alias'];
1445 }
1446 } elseif (strpos($cfg['subst']['tokenValue'], ':') !== false) {
1447 list($tokenKey) = explode(':', $cfg['subst']['tokenValue']);
1448 $insertValue = $tokenKey . ':' . $insertValue;
1449 }
1450 }
1451 }
1452 }
1453 // Finally, swap the soft reference token in tokenized content with the insert value:
1454 $tokenizedContent = str_replace('{softref:' . $tokenID . '}', $insertValue, $tokenizedContent);
1455 }
1456 return $tokenizedContent;
1457 }
1458
1459 /**
1460 * Process a soft reference file
1461 *
1462 * @param string $relFileName Old Relative filename
1463 * @param array $cfg soft reference configuration array
1464 * @param string $table Table for which the processing occurs
1465 * @param string $uid UID of record from table
1466 * @return string New relative filename (value to insert instead of the softref token)
1467 */
1468 public function processSoftReferences_saveFile($relFileName, $cfg, $table, $uid)
1469 {
1470 if ($fileHeaderInfo = $this->dat['header']['files'][$cfg['file_ID']]) {
1471 // Initialize; Get directory prefix for file and find possible RTE filename
1472 $dirPrefix = PathUtility::dirname($relFileName) . '/';
1473 $rteOrigName = $this->getRTEoriginalFilename(PathUtility::basename($relFileName));
1474 // If filename looks like an RTE file, and the directory is in "uploads/", then process as a RTE file!
1475 if ($rteOrigName && GeneralUtility::isFirstPartOfStr($dirPrefix, 'uploads/')) {
1476 // RTE:
1477 // First, find unique RTE file name:
1478 if (@is_dir((PATH_site . $dirPrefix))) {
1479 // From the "original" RTE filename, produce a new "original" destination filename which is unused.
1480 // Even if updated, the image should be unique. Currently the problem with this is that it leaves a lot of unused RTE images...
1481 $fileProcObj = $this->getFileProcObj();
1482 $origDestName = $fileProcObj->getUniqueName($rteOrigName, PATH_site . $dirPrefix);
1483 // Create copy file name:
1484 $pI = pathinfo($relFileName);
1485 $copyDestName = PathUtility::dirname($origDestName) . '/RTEmagicC_' . substr(PathUtility::basename($origDestName), 10) . '.' . $pI['extension'];
1486 if (
1487 !@is_file($copyDestName) && !@is_file($origDestName)
1488 && $origDestName === GeneralUtility::getFileAbsFileName($origDestName)
1489 && $copyDestName === GeneralUtility::getFileAbsFileName($copyDestName)
1490 ) {
1491 if ($this->dat['header']['files'][$fileHeaderInfo['RTE_ORIG_ID']]) {
1492 if ($this->legacyImport) {
1493 $fileName = PathUtility::basename($copyDestName);
1494 $this->writeSysFileResourceForLegacyImport($fileName, $cfg['file_ID']);
1495 $relFileName = $this->filePathMap[$cfg['file_ID']] . '" data-htmlarea-file-uid="' . $fileName . '" data-htmlarea-file-table="sys_file';
1496 // Also save the original file
1497 $originalFileName = PathUtility::basename($origDestName);
1498 $this->writeSysFileResourceForLegacyImport($originalFileName, $fileHeaderInfo['RTE_ORIG_ID']);
1499 } else {
1500 // Write the copy and original RTE file to the respective filenames:
1501 $this->writeFileVerify($copyDestName, $cfg['file_ID'], true);
1502 $this->writeFileVerify($origDestName, $fileHeaderInfo['RTE_ORIG_ID'], true);
1503 // Return the relative path of the copy file name:
1504 return PathUtility::stripPathSitePrefix($copyDestName);
1505 }
1506 } else {
1507 $this->error('ERROR: Could not find original file ID');
1508 }
1509 } else {
1510 $this->error('ERROR: The destination filenames "' . $copyDestName . '" and "' . $origDestName . '" either existed or have non-valid names');
1511 }
1512 } else {
1513 $this->error('ERROR: "' . PATH_site . $dirPrefix . '" was not a directory, so could not process file "' . $relFileName . '"');
1514 }
1515 } elseif (GeneralUtility::isFirstPartOfStr($dirPrefix, $this->fileadminFolderName . '/')) {
1516 // File in fileadmin/ folder:
1517 // Create file (and possible resources)
1518 $newFileName = $this->processSoftReferences_saveFile_createRelFile($dirPrefix, PathUtility::basename($relFileName), $cfg['file_ID'], $table, $uid);
1519 if (strlen($newFileName)) {
1520 $relFileName = $newFileName;
1521 } else {
1522 $this->error('ERROR: No new file created for "' . $relFileName . '"');
1523 }
1524 } else {
1525 $this->error('ERROR: Sorry, cannot operate on non-RTE files which are outside the fileadmin folder.');
1526 }
1527 } else {
1528 $this->error('ERROR: Could not find file ID in header.');
1529 }
1530 // Return (new) filename relative to PATH_site:
1531 return $relFileName;
1532 }
1533
1534 /**
1535 * Create file in directory and return the new (unique) filename
1536 *
1537 * @param string $origDirPrefix Directory prefix, relative, with trailing slash
1538 * @param string $fileName Filename (without path)
1539 * @param string $fileID File ID from import memory
1540 * @param string $table Table for which the processing occurs
1541 * @param string $uid UID of record from table
1542 * @return string|NULL New relative filename, if any
1543 */
1544 public function processSoftReferences_saveFile_createRelFile($origDirPrefix, $fileName, $fileID, $table, $uid)
1545 {
1546 // If the fileID map contains an entry for this fileID then just return the relative filename of that entry;
1547 // we don't want to write another unique filename for this one!
1548 if (isset($this->fileIDMap[$fileID])) {
1549 return PathUtility::stripPathSitePrefix($this->fileIDMap[$fileID]);
1550 }
1551 if ($this->legacyImport) {
1552 // set dirPrefix to fileadmin because the right target folder is set and checked for permissions later
1553 $dirPrefix = $this->fileadminFolderName . '/';
1554 } else {
1555 // Verify FileMount access to dir-prefix. Returns the best alternative relative path if any
1556 $dirPrefix = $this->verifyFolderAccess($origDirPrefix);
1557 }
1558 if ($dirPrefix && (!$this->update || $origDirPrefix === $dirPrefix) && $this->checkOrCreateDir($dirPrefix)) {
1559 $fileHeaderInfo = $this->dat['header']['files'][$fileID];
1560 $updMode = $this->update && $this->import_mapId[$table][$uid] === $uid && $this->import_mode[$table . ':' . $uid] !== 'as_new';
1561 // Create new name for file:
1562 // Must have same ID in map array (just for security, is not really needed) and NOT be set "as_new".
1563
1564 // Write main file:
1565 if ($this->legacyImport) {
1566 $fileWritten = $this->writeSysFileResourceForLegacyImport($fileName, $fileID);
1567 if ($fileWritten) {
1568 $newName = 'file:' . $fileName;
1569 return $newName;
1570 // no support for HTML/CSS file resources attached ATM - see below
1571 }
1572 } else {
1573 if ($updMode) {
1574 $newName = PATH_site . $dirPrefix . $fileName;
1575 } else {
1576 // Create unique filename:
1577 $fileProcObj = $this->getFileProcObj();
1578 $newName = $fileProcObj->getUniqueName($fileName, PATH_site . $dirPrefix);
1579 }
1580 if ($this->writeFileVerify($newName, $fileID)) {
1581 // If the resource was an HTML/CSS file with resources attached, we will write those as well!
1582 if (is_array($fileHeaderInfo['EXT_RES_ID'])) {
1583 $tokenizedContent = $this->dat['files'][$fileID]['tokenizedContent'];
1584 $tokenSubstituted = false;
1585 $fileProcObj = $this->getFileProcObj();
1586 if ($updMode) {
1587 foreach ($fileHeaderInfo['EXT_RES_ID'] as $res_fileID) {
1588 if ($this->dat['files'][$res_fileID]['filename']) {
1589 // Resolve original filename:
1590 $relResourceFileName = $this->dat['files'][$res_fileID]['parentRelFileName'];
1591 $absResourceFileName = GeneralUtility::resolveBackPath(PATH_site . $origDirPrefix . $relResourceFileName);
1592 $absResourceFileName = GeneralUtility::getFileAbsFileName($absResourceFileName);
1593 if ($absResourceFileName && GeneralUtility::isFirstPartOfStr($absResourceFileName, PATH_site . $this->fileadminFolderName . '/')) {
1594 $destDir = PathUtility::stripPathSitePrefix(PathUtility::dirname($absResourceFileName) . '/');
1595 if ($this->verifyFolderAccess($destDir, true) && $this->checkOrCreateDir($destDir)) {
1596 $this->writeFileVerify($absResourceFileName, $res_fileID);
1597 } else {
1598 $this->error('ERROR: Could not create file in directory "' . $destDir . '"');
1599 }
1600 } else {
1601 $this->error('ERROR: Could not resolve path for "' . $relResourceFileName . '"');
1602 }
1603 $tokenizedContent = str_replace('{EXT_RES_ID:' . $res_fileID . '}', $relResourceFileName, $tokenizedContent);
1604 $tokenSubstituted = true;
1605 }
1606 }
1607 } else {
1608 // Create the resouces directory name (filename without extension, suffixed "_FILES")
1609 $resourceDir = PathUtility::dirname($newName) . '/' . preg_replace('/\\.[^.]*$/', '', PathUtility::basename($newName)) . '_FILES';
1610 if (GeneralUtility::mkdir($resourceDir)) {
1611 foreach ($fileHeaderInfo['EXT_RES_ID'] as $res_fileID) {
1612 if ($this->dat['files'][$res_fileID]['filename']) {
1613 $absResourceFileName = $fileProcObj->getUniqueName($this->dat['files'][$res_fileID]['filename'], $resourceDir);
1614 $relResourceFileName = substr($absResourceFileName, strlen(PathUtility::dirname($resourceDir)) + 1);
1615 $this->writeFileVerify($absResourceFileName, $res_fileID);
1616 $tokenizedContent = str_replace('{EXT_RES_ID:' . $res_fileID . '}', $relResourceFileName, $tokenizedContent);
1617 $tokenSubstituted = true;
1618 }
1619 }
1620 }
1621 }
1622 // If substitutions has been made, write the content to the file again:
1623 if ($tokenSubstituted) {
1624 GeneralUtility::writeFile($newName, $tokenizedContent);
1625 }
1626 }
1627 return PathUtility::stripPathSitePrefix($newName);
1628 }
1629 }
1630 }
1631 return null;
1632 }
1633
1634 /**
1635 * Writes a file from the import memory having $fileID to file name $fileName which must be an absolute path inside PATH_site
1636 *
1637 * @param string $fileName Absolute filename inside PATH_site to write to
1638 * @param string $fileID File ID from import memory
1639 * @param bool $bypassMountCheck Bypasses the checking against filemounts - only for RTE files!
1640 * @return bool Returns TRUE if it went well. Notice that the content of the file is read again, and md5 from import memory is validated.
1641 */
1642 public function writeFileVerify($fileName, $fileID, $bypassMountCheck = false)
1643 {
1644 $fileProcObj = $this->getFileProcObj();
1645 if (!$fileProcObj->actionPerms['addFile']) {
1646 $this->error('ERROR: You did not have sufficient permissions to write the file "' . $fileName . '"');
1647 return false;
1648 }
1649 // Just for security, check again. Should actually not be necessary.
1650 if (!$fileProcObj->checkPathAgainstMounts($fileName) && !$bypassMountCheck) {
1651 $this->error('ERROR: Filename "' . $fileName . '" was not allowed in destination path!');
1652 return false;
1653 }
1654 $fI = GeneralUtility::split_fileref($fileName);
1655 if (!$fileProcObj->checkIfAllowed($fI['fileext'], $fI['path'], $fI['file']) && (!$this->allowPHPScripts || !$this->getBackendUser()->isAdmin())) {
1656 $this->error('ERROR: Filename "' . $fileName . '" failed against extension check or deny-pattern!');
1657 return false;
1658 }
1659 if (!GeneralUtility::getFileAbsFileName($fileName)) {
1660 $this->error('ERROR: Filename "' . $fileName . '" was not a valid relative file path!');
1661 return false;
1662 }
1663 if (!$this->dat['files'][$fileID]) {
1664 $this->error('ERROR: File ID "' . $fileID . '" could not be found');
1665 return false;
1666 }
1667 GeneralUtility::writeFile($fileName, $this->dat['files'][$fileID]['content']);
1668 $this->fileIDMap[$fileID] = $fileName;
1669 if (md5(GeneralUtility::getUrl($fileName)) == $this->dat['files'][$fileID]['content_md5']) {
1670 return true;
1671 } else {
1672 $this->error('ERROR: File content "' . $fileName . '" was corrupted');
1673 return false;
1674 }
1675 }
1676
1677 /**
1678 * Writes the file with the is $fileId to the legacy import folder. The file name will used from
1679 * argument $fileName and the file was successfully created or an identical file was already found,
1680 * $fileName will held the uid of the new created file record.
1681 *
1682 * @param string $fileName The file name for the new file. Value would be changed to the uid of the new created file record.
1683 * @param int $fileId The id of the file in data array
1684 * @return bool
1685 */
1686 protected function writeSysFileResourceForLegacyImport(&$fileName, $fileId)
1687 {
1688 if ($this->legacyImportFolder === null) {
1689 return false;
1690 }
1691
1692 if (!isset($this->dat['files'][$fileId])) {
1693 $this->error('ERROR: File ID "' . $fileId . '" could not be found');
1694 return false;
1695 }
1696
1697 $temporaryFile = $this->writeTemporaryFileFromData($fileId, 'files');
1698 if ($temporaryFile === null) {
1699 // error on writing the file. Error message was already added
1700 return false;
1701 }
1702
1703 $importFolder = $this->legacyImportFolder;
1704
1705 if (isset($this->dat['files'][$fileId]['relFileName'])) {
1706 $relativeFilePath = PathUtility::dirname($this->dat['files'][$fileId]['relFileName']);
1707
1708 if (!$this->legacyImportFolder->hasFolder($relativeFilePath)) {
1709 $this->legacyImportFolder->createFolder($relativeFilePath);
1710 }
1711 $importFolder = $this->legacyImportFolder->getSubfolder($relativeFilePath);
1712 }
1713
1714 $fileObject = null;
1715
1716 try {
1717 // check, if there is alreay the same file in the folder
1718 if ($importFolder->hasFile($fileName)) {
1719 $fileStorage = $importFolder->getStorage();
1720 $file = $fileStorage->getFile($importFolder->getIdentifier() . $fileName);
1721 if ($file->getSha1() === sha1_file($temporaryFile)) {
1722 $fileObject = $file;
1723 }
1724 }
1725 } catch (Exception $e) {
1726 }
1727
1728 if ($fileObject === null) {
1729 try {
1730 $fileObject = $importFolder->addFile($temporaryFile, $fileName, DuplicationBehavior::RENAME);
1731 } catch (Exception $e) {
1732 $this->error('Error: no file could be added to the storage for file name ' . $this->alternativeFileName[$temporaryFile]);
1733 }
1734 }
1735
1736 if (md5_file(PATH_site . $fileObject->getPublicUrl()) == $this->dat['files'][$fileId]['content_md5']) {
1737 $fileName = $fileObject->getUid();
1738 $this->fileIDMap[$fileId] = $fileName;
1739 $this->filePathMap[$fileId] = $fileObject->getPublicUrl();
1740 return true;
1741 } else {
1742 $this->error('ERROR: File content "' . $this->dat['files'][$fileId]['relFileName'] . '" was corrupted');
1743 }
1744
1745 return false;
1746 }
1747
1748 /**
1749 * Migrate legacy import records
1750 *
1751 * @return void
1752 */
1753 protected function migrateLegacyImportRecords()
1754 {
1755 $updateData= array();
1756
1757 foreach ($this->legacyImportMigrationRecords as $table => $records) {
1758 foreach ($records as $uid => $fields) {
1759 $row = BackendUtility::getRecord($table, $uid);
1760 if (empty($row)) {
1761 continue;
1762 }
1763
1764 foreach ($fields as $field => $referenceIds) {
1765 $fieldConfiguration = $this->legacyImportMigrationTables[$table][$field];
1766
1767 if (isset($fieldConfiguration['titleTexts'])) {
1768 $titleTextField = $fieldConfiguration['titleTexts'];
1769 if (isset($row[$titleTextField]) && $row[$titleTextField] !== '') {
1770 $titleTextContents = explode(LF, $row[$titleTextField]);
1771 $updateData[$table][$uid][$titleTextField] = '';
1772 }
1773 }
1774
1775 if (isset($fieldConfiguration['alternativeTexts'])) {
1776 $alternativeTextField = $fieldConfiguration['alternativeTexts'];
1777 if (isset($row[$alternativeTextField]) && $row[$alternativeTextField] !== '') {
1778 $alternativeTextContents = explode(LF, $row[$alternativeTextField]);
1779 $updateData[$table][$uid][$alternativeTextField] = '';
1780 }
1781 }
1782 if (isset($fieldConfiguration['description'])) {
1783 $descriptionField = $fieldConfiguration['description'];
1784 if ($row[$descriptionField] !== '') {
1785 $descriptionContents = explode(LF, $row[$descriptionField]);
1786 $updateData[$table][$uid][$descriptionField] = '';
1787 }
1788 }
1789 if (isset($fieldConfiguration['links'])) {
1790 $linkField = $fieldConfiguration['links'];
1791 if ($row[$linkField] !== '') {
1792 $linkContents = explode(LF, $row[$linkField]);
1793 $updateData[$table][$uid][$linkField] = '';
1794 }
1795 }
1796
1797 foreach ($referenceIds as $key => $referenceId) {
1798 if (isset($titleTextContents[$key])) {
1799 $updateData['sys_file_reference'][$referenceId]['title'] = trim($titleTextContents[$key]);
1800 }
1801 if (isset($alternativeTextContents[$key])) {
1802 $updateData['sys_file_reference'][$referenceId]['alternative'] = trim($alternativeTextContents[$key]);
1803 }
1804 if (isset($descriptionContents[$key])) {
1805 $updateData['sys_file_reference'][$referenceId]['description'] = trim($descriptionContents[$key]);
1806 }
1807 if (isset($linkContents[$key])) {
1808 $updateData['sys_file_reference'][$referenceId]['link'] = trim($linkContents[$key]);
1809 }
1810 }
1811 }
1812 }
1813 }
1814
1815 // update
1816 $tce = $this->getNewTCE();
1817 $tce->isImporting = true;
1818 $tce->start($updateData, array());
1819 $tce->process_datamap();
1820 }
1821
1822 /**
1823 * Returns TRUE if directory exists and if it doesn't it will create directory and return TRUE if that succeeded.
1824 *
1825 * @param string $dirPrefix Directory to create. Having a trailing slash. Must be in fileadmin/. Relative to PATH_site
1826 * @return bool TRUE, if directory exists (was created)
1827 */
1828 public function checkOrCreateDir($dirPrefix)
1829 {
1830 // Split dir path and remove first directory (which should be "fileadmin")
1831 $filePathParts = explode('/', $dirPrefix);
1832 $firstDir = array_shift($filePathParts);
1833 if ($firstDir === $this->fileadminFolderName && GeneralUtility::getFileAbsFileName($dirPrefix)) {
1834 $pathAcc = '';
1835 foreach ($filePathParts as $dirname) {
1836 $pathAcc .= '/' . $dirname;
1837 if (strlen($dirname)) {
1838 if (!@is_dir((PATH_site . $this->fileadminFolderName . $pathAcc))) {
1839 if (!GeneralUtility::mkdir((PATH_site . $this->fileadminFolderName . $pathAcc))) {
1840 $this->error('ERROR: Directory could not be created....B');
1841 return false;
1842 }
1843 }
1844 } elseif ($dirPrefix === $this->fileadminFolderName . $pathAcc) {
1845 return true;
1846 } else {
1847 $this->error('ERROR: Directory could not be created....A');
1848 }
1849 }
1850 }
1851 return false;
1852 }
1853
1854 /**************************
1855 * File Input
1856 *************************/
1857
1858 /**
1859 * Loads the header section/all of the $filename into memory
1860 *
1861 * @param string $filename Filename, absolute
1862 * @param bool $all If set, all information is loaded (header, records and files). Otherwise the default is to read only the header information
1863 * @return bool TRUE if the operation went well
1864 */
1865 public function loadFile($filename, $all = false)
1866 {
1867 if (!@is_file($filename)) {
1868 $this->error('Filename not found: ' . $filename);
1869 return false;
1870 }
1871 $fI = pathinfo($filename);
1872 if (@is_dir($filename . '.files')) {
1873 if (GeneralUtility::isAllowedAbsPath($filename . '.files')) {
1874 // copy the folder lowlevel to typo3temp, because the files would be deleted after import
1875 $temporaryFolderName = $this->getTemporaryFolderName();
1876 GeneralUtility::copyDirectory($filename . '.files', $temporaryFolderName);
1877 $this->filesPathForImport = $temporaryFolderName;
1878 } else {
1879 $this->error('External import files for the given import source is currently not supported.');
1880 }
1881 }
1882 if (strtolower($fI['extension']) == 'xml') {
1883 // XML:
1884 $xmlContent = GeneralUtility::getUrl($filename);
1885 if (strlen($xmlContent)) {
1886 $this->dat = GeneralUtility::xml2array($xmlContent, '', true);
1887 if (is_array($this->dat)) {
1888 if ($this->dat['_DOCUMENT_TAG'] === 'T3RecordDocument' && is_array($this->dat['header']) && is_array($this->dat['records'])) {
1889 $this->loadInit();
1890 return true;
1891 } else {
1892 $this->error('XML file did not contain proper XML for TYPO3 Import');
1893 }
1894 } else {
1895 $this->error('XML could not be parsed: ' . $this->dat);
1896 }
1897 } else {
1898 $this->error('Error opening file: ' . $filename);
1899 }
1900 } else {
1901 // T3D
1902 if ($fd = fopen($filename, 'rb')) {
1903 $this->dat['header'] = $this->getNextFilePart($fd, 1, 'header');
1904 if ($all) {
1905 $this->dat['records'] = $this->getNextFilePart($fd, 1, 'records');
1906 $this->dat['files'] = $this->getNextFilePart($fd, 1, 'files');
1907 $this->dat['files_fal'] = $this->getNextFilePart($fd, 1, 'files_fal');
1908 }
1909 $this->loadInit();
1910 return true;
1911 } else {
1912 $this->error('Error opening file: ' . $filename);
1913 }
1914 fclose($fd);
1915 }
1916 return false;
1917 }
1918
1919 /**
1920 * Returns the next content part form the fileresource (t3d), $fd
1921 *
1922 * @param resource $fd File pointer
1923 * @param bool $unserialize If set, the returned content is unserialized into an array, otherwise you get the raw string
1924 * @param string $name For error messages this indicates the section of the problem.
1925 * @return string|NULL Data string or NULL in case of an error
1926 * @access private
1927 * @see loadFile()
1928 */
1929 public function getNextFilePart($fd, $unserialize = false, $name = '')
1930 {
1931 $initStrLen = 32 + 1 + 1 + 1 + 10 + 1;
1932 // Getting header data
1933 $initStr = fread($fd, $initStrLen);
1934 if (empty($initStr)) {
1935 $this->error('File does not contain data for "' . $name . '"');
1936 return null;
1937 }
1938 $initStrDat = explode(':', $initStr);
1939 if (strstr($initStrDat[0], 'Warning')) {
1940 $this->error('File read error: Warning message in file. (' . $initStr . fgets($fd) . ')');
1941 return null;
1942 }
1943 if ((string)$initStrDat[3] !== '') {
1944 $this->error('File read error: InitString had a wrong length. (' . $name . ')');
1945 return null;
1946 }
1947 $datString = fread($fd, (int)$initStrDat[2]);
1948 fread($fd, 1);
1949 if (md5($datString) === $initStrDat[0]) {
1950 if ($initStrDat[1]) {
1951 if ($this->compress) {
1952 $datString = gzuncompress($datString);
1953 return $unserialize ? unserialize($datString) : $datString;
1954 } else {
1955 $this->error('Content read error: This file requires decompression, but this server does not offer gzcompress()/gzuncompress() functions.');
1956 }
1957 }
1958 } else {
1959 $this->error('MD5 check failed (' . $name . ')');
1960 }
1961 return null;
1962 }
1963
1964 /**
1965 * Loads T3D file content into the $this->dat array
1966 * (This function can be used to test the output strings from ->compileMemoryToFileContent())
1967 *
1968 * @param string $filecontent File content
1969 * @return void
1970 */
1971 public function loadContent($filecontent)
1972 {
1973 $pointer = 0;
1974 $this->dat['header'] = $this->getNextContentPart($filecontent, $pointer, 1, 'header');
1975 $this->dat['records'] = $this->getNextContentPart($filecontent, $pointer, 1, 'records');
1976 $this->dat['files'] = $this->getNextContentPart($filecontent, $pointer, 1, 'files');
1977 $this->loadInit();
1978 }
1979
1980 /**
1981 * Returns the next content part from the $filecontent
1982 *
1983 * @param string $filecontent File content string
1984 * @param int $pointer File pointer (where to read from)
1985 * @param bool $unserialize If set, the returned content is unserialized into an array, otherwise you get the raw string
1986 * @param string $name For error messages this indicates the section of the problem.
1987 * @return string|NULL Data string
1988 */
1989 public function getNextContentPart($filecontent, &$pointer, $unserialize = false, $name = '')
1990 {
1991 $initStrLen = 32 + 1 + 1 + 1 + 10 + 1;
1992 // getting header data
1993 $initStr = substr($filecontent, $pointer, $initStrLen);
1994 $pointer += $initStrLen;
1995 $initStrDat = explode(':', $initStr);
1996 if ((string)$initStrDat[3] !== '') {
1997 $this->error('Content read error: InitString had a wrong length. (' . $name . ')');
1998 return null;
1999 }
2000 $datString = substr($filecontent, $pointer, (int)$initStrDat[2]);
2001 $pointer += (int)$initStrDat[2] + 1;
2002 if (md5($datString) === $initStrDat[0]) {
2003 if ($initStrDat[1]) {
2004 if ($this->compress) {
2005 $datString = gzuncompress($datString);
2006 return $unserialize ? unserialize($datString) : $datString;
2007 } else {
2008 $this->error('Content read error: This file requires decompression, but this server does not offer gzcompress()/gzuncompress() functions.');
2009 }
2010 }
2011 } else {
2012 $this->error('MD5 check failed (' . $name . ')');
2013 }
2014 return null;
2015 }
2016
2017 /**
2018 * Setting up the object based on the recently loaded ->dat array
2019 *
2020 * @return void
2021 */
2022 public function loadInit()
2023 {
2024 $this->relStaticTables = (array)$this->dat['header']['relStaticTables'];
2025 $this->excludeMap = (array)$this->dat['header']['excludeMap'];
2026 $this->softrefCfg = (array)$this->dat['header']['softrefCfg'];
2027 $this->extensionDependencies = (array)$this->dat['header']['extensionDependencies'];
2028 $this->fixCharsets();
2029 if (
2030 isset($this->dat['header']['meta']['TYPO3_version'])
2031 && VersionNumberUtility::convertVersionNumberToInteger($this->dat['header']['meta']['TYPO3_version']) < 6000000
2032 ) {
2033 $this->legacyImport = true;
2034 $this->initializeLegacyImportFolder();
2035 }
2036 }
2037
2038 /**
2039 * Fix charset of import memory if different from system charset
2040 *
2041 * @return void
2042 * @see loadInit()
2043 */
2044 public function fixCharsets()
2045 {
2046 $importCharset = $this->dat['header']['charset'];
2047 if ($importCharset) {
2048 if ($importCharset !== $this->getLanguageService()->charSet) {
2049 $this->error('CHARSET: Converting charset of input file (' . $importCharset . ') to the system charset (' . $this->getLanguageService()->charSet . ')');
2050 // Convert meta data:
2051 if (is_array($this->dat['header']['meta'])) {
2052 $this->getLanguageService()->csConvObj->convArray($this->dat['header']['meta'], $importCharset, $this->getLanguageService()->charSet);
2053 }
2054 // Convert record headers:
2055 if (is_array($this->dat['header']['records'])) {
2056 $this->getLanguageService()->csConvObj->convArray($this->dat['header']['records'], $importCharset, $this->getLanguageService()->charSet);
2057 }
2058 // Convert records themselves:
2059 if (is_array($this->dat['records'])) {
2060 $this->getLanguageService()->csConvObj->convArray($this->dat['records'], $importCharset, $this->getLanguageService()->charSet);
2061 }
2062 }
2063 } else {
2064 $this->error('CHARSET: No charset found in import file!');
2065 }
2066 }
2067 }