88157bb22e20b2336aea8c591a4497c13e90a3e4
[Packages/TYPO3.CMS.git] / typo3 / sysext / impexp / Classes / ImportExport.php
1 <?php
2 namespace TYPO3\CMS\Impexp;
3
4 /*
5 * This file is part of the TYPO3 CMS project.
6 *
7 * It is free software; you can redistribute it and/or modify it under
8 * the terms of the GNU General Public License, either version 2
9 * of the License, or any later version.
10 *
11 * For the full copyright and license information, please read the
12 * LICENSE.txt file that was distributed with this source code.
13 *
14 * The TYPO3 project - inspiring people to share!
15 */
16
17 use TYPO3\CMS\Backend\Utility\BackendUtility;
18 use TYPO3\CMS\Core\DataHandling\DataHandler;
19 use TYPO3\CMS\Core\Exception;
20 use TYPO3\CMS\Core\Imaging\Icon;
21 use TYPO3\CMS\Core\Imaging\IconFactory;
22 use TYPO3\CMS\Core\Resource\DuplicationBehavior;
23 use TYPO3\CMS\Core\Resource\ResourceFactory;
24 use TYPO3\CMS\Core\Utility\File\ExtendedFileUtility;
25 use TYPO3\CMS\Core\Utility\GeneralUtility;
26 use TYPO3\CMS\Core\Utility\PathUtility;
27
28 /**
29 * EXAMPLE for using the impexp-class for exporting stuff:
30 *
31 * Create and initialize:
32 * $this->export = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Impexp\ImportExport::class);
33 * $this->export->init();
34 * Set which tables relations we will allow:
35 * $this->export->relOnlyTables[]="tt_news"; // exclusively includes. See comment in the class
36 *
37 * Adding records:
38 * $this->export->export_addRecord("pages", $this->pageinfo);
39 * $this->export->export_addRecord("pages", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("pages", 38));
40 * $this->export->export_addRecord("pages", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("pages", 39));
41 * $this->export->export_addRecord("tt_content", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("tt_content", 12));
42 * $this->export->export_addRecord("tt_content", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("tt_content", 74));
43 * $this->export->export_addRecord("sys_template", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("sys_template", 20));
44 *
45 * Adding all the relations (recursively in 5 levels so relations has THEIR relations registered as well)
46 * for($a=0;$a<5;$a++) {
47 * $addR = $this->export->export_addDBRelations($a);
48 * if (empty($addR)) break;
49 * }
50 *
51 * Finally load all the files.
52 * $this->export->export_addFilesFromRelations(); // MUST be after the DBrelations are set so that file from ALL added records are included!
53 *
54 * Write export
55 * $out = $this->export->compileMemoryToFileContent();
56 */
57
58 /**
59 * T3D file Import/Export library (TYPO3 Record Document)
60 */
61 class ImportExport {
62
63 /**
64 * If set, static relations (not exported) will be shown in overview as well
65 *
66 * @var bool
67 */
68 public $showStaticRelations = FALSE;
69
70 /**
71 * Name of the "fileadmin" folder where files for export/import should be located
72 *
73 * @var string
74 */
75 public $fileadminFolderName = '';
76
77 /**
78 * Whether "import" or "export" mode of object. Set through init() function
79 *
80 * @var string
81 */
82 public $mode = '';
83
84 /**
85 * Updates all records that has same UID instead of creating new!
86 *
87 * @var bool
88 */
89 public $update = FALSE;
90
91 /**
92 * Is set by importData() when an import has been done.
93 *
94 * @var bool
95 */
96 public $doesImport = FALSE;
97
98 /**
99 * If set to a page-record, then the preview display of the content will expect this page-record to be the target
100 * for the import and accordingly display validation information. This triggers the visual view of the
101 * import/export memory to validate if import is possible
102 *
103 * @var array
104 */
105 public $display_import_pid_record = array();
106
107 /**
108 * Used to register the forged UID values for imported records that we want
109 * to create with the same UIDs as in the import file. Admin-only feature.
110 *
111 * @var array
112 */
113 public $suggestedInsertUids = array();
114
115 /**
116 * Setting import modes during update state: as_new, exclude, force_uid
117 *
118 * @var array
119 */
120 public $import_mode = array();
121
122 /**
123 * If set, PID correct is ignored globally
124 *
125 * @var bool
126 */
127 public $global_ignore_pid = FALSE;
128
129 /**
130 * If set, all UID values are forced! (update or import)
131 *
132 * @var bool
133 */
134 public $force_all_UIDS = FALSE;
135
136 /**
137 * If set, a diff-view column is added to the overview.
138 *
139 * @var bool
140 */
141 public $showDiff = FALSE;
142
143 /**
144 * If set, and if the user is admin, allow the writing of PHP scripts to fileadmin/ area.
145 *
146 * @var bool
147 */
148 public $allowPHPScripts = FALSE;
149
150 /**
151 * Disable logging when importing
152 *
153 * @var bool
154 */
155 public $enableLogging = FALSE;
156
157 /**
158 * Array of values to substitute in editable softreferences.
159 *
160 * @var array
161 */
162 public $softrefInputValues = array();
163
164 /**
165 * Mapping between the fileID from import memory and the final filenames they are written to.
166 *
167 * @var array
168 */
169 public $fileIDMap = array();
170
171 /**
172 * 1MB max file size
173 *
174 * @var int
175 */
176 public $maxFileSize = 1000000;
177
178 /**
179 * 1MB max record size
180 *
181 * @var int
182 */
183 public $maxRecordSize = 1000000;
184
185 /**
186 * 10MB max export size
187 *
188 * @var int
189 */
190 public $maxExportSize = 10000000;
191
192 /**
193 * Add table names here which are THE ONLY ones which will be included
194 * into export if found as relations. '_ALL' will allow all tables.
195 *
196 * @var array
197 */
198 public $relOnlyTables = array();
199
200 /**
201 * Add tables names here which should not be exported with the file.
202 * (Where relations should be mapped to same UIDs in target system).
203 *
204 * @var array
205 */
206 public $relStaticTables = array();
207
208 /**
209 * Exclude map. Keys are table:uid pairs and if set, records are not added to the export.
210 *
211 * @var array
212 */
213 public $excludeMap = array();
214
215 /**
216 * Soft Reference Token ID modes.
217 *
218 * @var array
219 */
220 public $softrefCfg = array();
221
222 /**
223 * Listing extension dependencies.
224 *
225 * @var array
226 */
227 public $extensionDependencies = array();
228
229 /**
230 * Set by user: If set, compression in t3d files is disabled
231 *
232 * @var bool
233 */
234 public $dontCompress = FALSE;
235
236 /**
237 * If set, HTML file resources are included.
238 *
239 * @var bool
240 */
241 public $includeExtFileResources = FALSE;
242
243 /**
244 * Files with external media (HTML/css style references inside)
245 *
246 * @var string
247 */
248 public $extFileResourceExtensions = 'html,htm,css';
249
250 /**
251 * After records are written this array is filled with [table][original_uid] = [new_uid]
252 *
253 * @var array
254 */
255 public $import_mapId = array();
256
257 /**
258 * Keys are [tablename]:[new NEWxxx ids (or when updating it is uids)]
259 * while values are arrays with table/uid of the original record it is based on.
260 * With the array keys the new ids can be looked up inside tcemain
261 *
262 * @var array
263 */
264 public $import_newId = array();
265
266 /**
267 * Page id map for page tree (import)
268 *
269 * @var array
270 */
271 public $import_newId_pids = array();
272
273 /**
274 * Internal data accumulation for writing records during import
275 *
276 * @var array
277 */
278 public $import_data = array();
279
280 /**
281 * Error log.
282 *
283 * @var array
284 */
285 public $errorLog = array();
286
287 /**
288 * Cache for record paths
289 *
290 * @var array
291 */
292 public $cache_getRecordPath = array();
293
294 /**
295 * Cache of checkPID values.
296 *
297 * @var array
298 */
299 public $checkPID_cache = array();
300
301 /**
302 * Set internally if the gzcompress function exists
303 * Used by ImportExportController
304 *
305 * @var bool
306 */
307 public $compress = FALSE;
308
309 /**
310 * Internal import/export memory
311 *
312 * @var array
313 */
314 public $dat = array();
315
316 /**
317 * File processing object
318 *
319 * @var \TYPO3\CMS\Core\Utility\File\ExtendedFileUtility
320 */
321 protected $fileProcObj = NULL;
322
323 /**
324 * Keys are [recordname], values are an array of fields to be included
325 * in the export
326 *
327 * @var array
328 */
329 protected $recordTypesIncludeFields = array();
330
331 /**
332 * Default array of fields to be included in the export
333 *
334 * @var array
335 */
336 protected $defaultRecordIncludeFields = array('uid', 'pid');
337
338 /**
339 * Array of current registered storage objects
340 *
341 * @var array
342 */
343 protected $storageObjects = array();
344
345 /**
346 * Is set, if the import file has a TYPO3 version below 6.0
347 *
348 * @var bool
349 */
350 protected $legacyImport = FALSE;
351
352 /**
353 * @var \TYPO3\CMS\Core\Resource\Folder
354 */
355 protected $legacyImportFolder = NULL;
356
357 /**
358 * Related to the default storage root
359 *
360 * @var string
361 */
362 protected $legacyImportTargetPath = '_imported/';
363
364 /**
365 * Table fields to migrate
366 *
367 * @var array
368 */
369 protected $legacyImportMigrationTables = array(
370 'tt_content' => array(
371 'image' => array(
372 'titleTexts' => 'titleText',
373 'description' => 'imagecaption',
374 'links' => 'image_link',
375 'alternativeTexts' => 'altText'
376 ),
377 'media' => array(
378 'description' => 'imagecaption',
379 )
380 ),
381 'pages' => array(
382 'media' => array()
383 ),
384 'pages_language_overlay' => array(
385 'media' => array()
386 )
387 );
388
389
390 /**
391 * Records to be migrated after all
392 * Multidimensional array [table][uid][field] = array([related sys_file_reference uids])
393 *
394 * @var array
395 */
396 protected $legacyImportMigrationRecords = array();
397
398 /**
399 * @var bool
400 */
401 protected $saveFilesOutsideExportFile = FALSE;
402
403 /**
404 * @var NULL|string
405 */
406 protected $temporaryFilesPathForExport = NULL;
407
408 /**
409 * @var NULL|string
410 */
411 protected $filesPathForImport = NULL;
412
413 /**
414 * @var array
415 */
416 protected $unlinkFiles = array();
417
418 /**
419 * @var array
420 */
421 protected $alternativeFileName = array();
422
423 /**
424 * @var array
425 */
426 protected $alternativeFilePath = array();
427
428 /**
429 * @var array
430 */
431 protected $filePathMap = array();
432
433 /**
434 * @var array
435 */
436 protected $remainHeader = array();
437
438 /**************************
439 * Initialize
440 *************************/
441
442 /**
443 * Init the object, both import and export
444 *
445 * @param bool $dontCompress If set, compression of t3d files is disabled
446 * @param string $mode Mode of usage, either "import" or "export
447 * @return void
448 */
449 public function init($dontCompress = FALSE, $mode = '') {
450 $this->compress = function_exists('gzcompress');
451 $this->dontCompress = $dontCompress;
452 $this->mode = $mode;
453 $this->fileadminFolderName = !empty($GLOBALS['TYPO3_CONF_VARS']['BE']['fileadminDir']) ? rtrim($GLOBALS['TYPO3_CONF_VARS']['BE']['fileadminDir'], '/') : 'fileadmin';
454 }
455
456 /**************************
457 * Export / Init + Meta Data
458 *************************/
459
460 /**
461 * Set header basics
462 *
463 * @return void
464 */
465 public function setHeaderBasics() {
466 // Initializing:
467 if (is_array($this->softrefCfg)) {
468 foreach ($this->softrefCfg as $key => $value) {
469 if (!strlen($value['mode'])) {
470 unset($this->softrefCfg[$key]);
471 }
472 }
473 }
474 // Setting in header memory:
475 // Version of file format
476 $this->dat['header']['XMLversion'] = '1.0';
477 // Initialize meta data array (to put it in top of file)
478 $this->dat['header']['meta'] = array();
479 // Add list of tables to consider static
480 $this->dat['header']['relStaticTables'] = $this->relStaticTables;
481 // The list of excluded records
482 $this->dat['header']['excludeMap'] = $this->excludeMap;
483 // Soft Reference mode for elements
484 $this->dat['header']['softrefCfg'] = $this->softrefCfg;
485 // List of extensions the import depends on.
486 $this->dat['header']['extensionDependencies'] = $this->extensionDependencies;
487 }
488
489 /**
490 * Set charset
491 *
492 * @param string $charset Charset for the content in the export. During import the character set will be converted if the target system uses another charset.
493 * @return void
494 */
495 public function setCharset($charset) {
496 $this->dat['header']['charset'] = $charset;
497 }
498
499 /**
500 * Sets meta data
501 *
502 * @param string $title Title of the export
503 * @param string $description Description of the export
504 * @param string $notes Notes about the contents
505 * @param string $packager_username Backend Username of the packager (the guy making the export)
506 * @param string $packager_name Real name of the packager
507 * @param string $packager_email Email of the packager
508 * @return void
509 */
510 public function setMetaData($title, $description, $notes, $packager_username, $packager_name, $packager_email) {
511 $this->dat['header']['meta'] = array(
512 'title' => $title,
513 'description' => $description,
514 'notes' => $notes,
515 'packager_username' => $packager_username,
516 'packager_name' => $packager_name,
517 'packager_email' => $packager_email,
518 'TYPO3_version' => TYPO3_version,
519 'created' => strftime('%A %e. %B %Y', $GLOBALS['EXEC_TIME'])
520 );
521 }
522
523 /**
524 * Option to enable having the files not included in the export file.
525 * The files are saved to a temporary folder instead.
526 *
527 * @param bool $saveFilesOutsideExportFile
528 * @see getTemporaryFilesPathForExport()
529 */
530 public function setSaveFilesOutsideExportFile($saveFilesOutsideExportFile) {
531 $this->saveFilesOutsideExportFile = $saveFilesOutsideExportFile;
532 }
533
534 /**
535 * Sets a thumbnail image to the exported file
536 *
537 * @param string $imgFilepath Filename reference, gif, jpg, png. Absolute path.
538 * @return void
539 */
540 public function addThumbnail($imgFilepath) {
541 if (@is_file($imgFilepath)) {
542 $imgInfo = @getimagesize($imgFilepath);
543 if (is_array($imgInfo)) {
544 $fileContent = GeneralUtility::getUrl($imgFilepath);
545 $this->dat['header']['thumbnail'] = array(
546 'imgInfo' => $imgInfo,
547 'content' => $fileContent,
548 'filesize' => strlen($fileContent),
549 'filemtime' => filemtime($imgFilepath),
550 'filename' => PathUtility::basename($imgFilepath)
551 );
552 }
553 }
554 }
555
556 /**************************
557 * Export / Init Page tree
558 *************************/
559
560 /**
561 * Sets the page-tree array in the export header and returns the array in a flattened version
562 *
563 * @param array $idH Hierarchy of ids, the page tree: array([uid] => array("uid" => [uid], "subrow" => array(.....)), [uid] => ....)
564 * @return array The hierarchical page tree converted to a one-dimensional list of pages
565 */
566 public function setPageTree($idH) {
567 $this->dat['header']['pagetree'] = $this->unsetExcludedSections($idH);
568 return $this->flatInversePageTree($this->dat['header']['pagetree']);
569 }
570
571 /**
572 * Removes entries in the page tree which are found in ->excludeMap[]
573 *
574 * @param array $idH Page uid hierarchy
575 * @return array Modified input array
576 * @access private
577 * @see setPageTree()
578 */
579 public function unsetExcludedSections($idH) {
580 if (is_array($idH)) {
581 foreach ($idH as $k => $v) {
582 if ($this->excludeMap['pages:' . $idH[$k]['uid']]) {
583 unset($idH[$k]);
584 } elseif (is_array($idH[$k]['subrow'])) {
585 $idH[$k]['subrow'] = $this->unsetExcludedSections($idH[$k]['subrow']);
586 }
587 }
588 }
589 return $idH;
590 }
591
592 /**
593 * Recursively flattening the idH array (for setPageTree() function)
594 *
595 * @param array $idH Page uid hierarchy
596 * @param array $a Accumulation array of pages (internal, don't set from outside)
597 * @return array Array with uid-uid pairs for all pages in the page tree.
598 * @see flatInversePageTree_pid()
599 */
600 public function flatInversePageTree($idH, $a = array()) {
601 if (is_array($idH)) {
602 $idH = array_reverse($idH);
603 foreach ($idH as $k => $v) {
604 $a[$v['uid']] = $v['uid'];
605 if (is_array($v['subrow'])) {
606 $a = $this->flatInversePageTree($v['subrow'], $a);
607 }
608 }
609 }
610 return $a;
611 }
612
613 /**
614 * Recursively flattening the idH array (for setPageTree() function), setting PIDs as values
615 *
616 * @param array $idH Page uid hierarchy
617 * @param array $a Accumulation array of pages (internal, don't set from outside)
618 * @param int $pid PID value (internal)
619 * @return array Array with uid-pid pairs for all pages in the page tree.
620 * @see flatInversePageTree()
621 */
622 public function flatInversePageTree_pid($idH, $a = array(), $pid = -1) {
623 if (is_array($idH)) {
624 $idH = array_reverse($idH);
625 foreach ($idH as $v) {
626 $a[$v['uid']] = $pid;
627 if (is_array($v['subrow'])) {
628 $a = $this->flatInversePageTree_pid($v['subrow'], $a, $v['uid']);
629 }
630 }
631 }
632 return $a;
633 }
634
635 /**************************
636 * Export
637 *************************/
638
639 /**
640 * Sets the fields of record types to be included in the export
641 *
642 * @param array $recordTypesIncludeFields Keys are [recordname], values are an array of fields to be included in the export
643 * @throws \TYPO3\CMS\Core\Exception if an array value is not type of array
644 * @return void
645 */
646 public function setRecordTypesIncludeFields(array $recordTypesIncludeFields) {
647 foreach ($recordTypesIncludeFields as $table => $fields) {
648 if (!is_array($fields)) {
649 throw new \TYPO3\CMS\Core\Exception('The include fields for record type ' . htmlspecialchars($table) . ' are not defined by an array.', 1391440658);
650 }
651 $this->setRecordTypeIncludeFields($table, $fields);
652 }
653 }
654
655 /**
656 * Sets the fields of a record type to be included in the export
657 *
658 * @param string $table The record type
659 * @param array $fields The fields to be included
660 * @return void
661 */
662 public function setRecordTypeIncludeFields($table, array $fields) {
663 $this->recordTypesIncludeFields[$table] = $fields;
664 }
665
666 /**
667 * Adds the record $row from $table.
668 * No checking for relations done here. Pure data.
669 *
670 * @param string $table Table name
671 * @param array $row Record row.
672 * @param int $relationLevel (Internal) if the record is added as a relation, this is set to the "level" it was on.
673 * @return void
674 */
675 public function export_addRecord($table, $row, $relationLevel = 0) {
676 BackendUtility::workspaceOL($table, $row);
677 if ((string)$table !== '' && is_array($row) && $row['uid'] > 0 && !$this->excludeMap[($table . ':' . $row['uid'])]) {
678 if ($this->checkPID($table === 'pages' ? $row['uid'] : $row['pid'])) {
679 if (!isset($this->dat['records'][($table . ':' . $row['uid'])])) {
680 // Prepare header info:
681 $row = $this->filterRecordFields($table, $row);
682 $headerInfo = array();
683 $headerInfo['uid'] = $row['uid'];
684 $headerInfo['pid'] = $row['pid'];
685 $headerInfo['title'] = GeneralUtility::fixed_lgd_cs(BackendUtility::getRecordTitle($table, $row), 40);
686 $headerInfo['size'] = strlen(serialize($row));
687 if ($relationLevel) {
688 $headerInfo['relationLevel'] = $relationLevel;
689 }
690 // If record content is not too large in size, set the header content and add the rest:
691 if ($headerInfo['size'] < $this->maxRecordSize) {
692 // Set the header summary:
693 $this->dat['header']['records'][$table][$row['uid']] = $headerInfo;
694 // Create entry in the PID lookup:
695 $this->dat['header']['pid_lookup'][$row['pid']][$table][$row['uid']] = 1;
696 // Initialize reference index object:
697 $refIndexObj = GeneralUtility::makeInstance(\TYPO3\CMS\Core\Database\ReferenceIndex::class);
698 // Yes to workspace overlays for exporting....
699 $refIndexObj->WSOL = TRUE;
700 $relations = $refIndexObj->getRelations($table, $row);
701 $relations = $this->fixFileIDsInRelations($relations);
702 $relations = $this->removeSoftrefsHavingTheSameDatabaseRelation($relations);
703 // Data:
704 $this->dat['records'][$table . ':' . $row['uid']] = array();
705 $this->dat['records'][$table . ':' . $row['uid']]['data'] = $row;
706 $this->dat['records'][$table . ':' . $row['uid']]['rels'] = $relations;
707 $this->errorLog = array_merge($this->errorLog, $refIndexObj->errorLog);
708 // Merge error logs.
709 // Add information about the relations in the record in the header:
710 $this->dat['header']['records'][$table][$row['uid']]['rels'] = $this->flatDBrels($this->dat['records'][$table . ':' . $row['uid']]['rels']);
711 // Add information about the softrefs to header:
712 $this->dat['header']['records'][$table][$row['uid']]['softrefs'] = $this->flatSoftRefs($this->dat['records'][$table . ':' . $row['uid']]['rels']);
713 } else {
714 $this->error('Record ' . $table . ':' . $row['uid'] . ' was larger than maxRecordSize (' . GeneralUtility::formatSize($this->maxRecordSize) . ')');
715 }
716 } else {
717 $this->error('Record ' . $table . ':' . $row['uid'] . ' already added.');
718 }
719 } else {
720 $this->error('Record ' . $table . ':' . $row['uid'] . ' was outside your DB mounts!');
721 }
722 }
723 }
724
725 /**
726 * This changes the file reference ID from a hash based on the absolute file path
727 * (coming from ReferenceIndex) to a hash based on the relative file path.
728 *
729 * @param array $relations
730 * @return array
731 */
732 protected function fixFileIDsInRelations(array $relations) {
733 foreach ($relations as $field => $relation) {
734 if (isset($relation['type']) && $relation['type'] === 'file') {
735 foreach ($relation['newValueFiles'] as $key => $fileRelationData) {
736 $absoluteFilePath = $fileRelationData['ID_absFile'];
737 if (GeneralUtility::isFirstPartOfStr($absoluteFilePath, PATH_site)) {
738 $relatedFilePath = PathUtility::stripPathSitePrefix($absoluteFilePath);
739 $relations[$field]['newValueFiles'][$key]['ID'] = md5($relatedFilePath);
740 }
741 }
742 }
743 if ($relation['type'] === 'flex') {
744 if (is_array($relation['flexFormRels']['file'])) {
745 foreach ($relation['flexFormRels']['file'] as $key => $subList) {
746 foreach ($subList as $subKey => $fileRelationData) {
747 $absoluteFilePath = $fileRelationData['ID_absFile'];
748 if (GeneralUtility::isFirstPartOfStr($absoluteFilePath, PATH_site)) {
749 $relatedFilePath = PathUtility::stripPathSitePrefix($absoluteFilePath);
750 $relations[$field]['flexFormRels']['file'][$key][$subKey]['ID'] = md5($relatedFilePath);
751 }
752 }
753 }
754 }
755 }
756 }
757 return $relations;
758 }
759
760 /**
761 * Relations could contain db relations to sys_file records. Some configuration combinations of TCA and
762 * SoftReferenceIndex create also softref relation entries for the identical file. This results
763 * in double included files, one in array "files" and one in array "file_fal".
764 * This function checks the relations for this double inclusions and removes the redundant softref relation.
765 *
766 * @param array $relations
767 * @return array
768 */
769 protected function removeSoftrefsHavingTheSameDatabaseRelation($relations) {
770 $fixedRelations = array();
771 foreach ($relations as $field => $relation) {
772 $newRelation = $relation;
773 if (isset($newRelation['type']) && $newRelation['type'] === 'db') {
774 foreach ($newRelation['itemArray'] as $key => $dbRelationData) {
775 if ($dbRelationData['table'] === 'sys_file') {
776 if (isset($newRelation['softrefs']['keys']['typolink'])) {
777 foreach ($newRelation['softrefs']['keys']['typolink'] as $softrefKey => $softRefData) {
778 if ($softRefData['subst']['type'] === 'file') {
779 $file = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->retrieveFileOrFolderObject($softRefData['subst']['relFileName']);
780 if ($file instanceof \TYPO3\CMS\Core\Resource\File) {
781 if ($file->getUid() == $dbRelationData['id']) {
782 unset($newRelation['softrefs']['keys']['typolink'][$softrefKey]);
783 }
784 }
785 }
786 }
787 if (empty($newRelation['softrefs']['keys']['typolink'])) {
788 unset($newRelation['softrefs']);
789 }
790 }
791 }
792 }
793 }
794 $fixedRelations[$field] = $newRelation;
795 }
796 return $fixedRelations;
797 }
798
799
800 /**
801 * This analyses the existing added records, finds all database relations to records and adds these records to the export file.
802 * This function can be called repeatedly until it returns an empty array.
803 * In principle it should not allow to infinite recursivity, but you better set a limit...
804 * Call this BEFORE the ext_addFilesFromRelations (so files from added relations are also included of course)
805 *
806 * @param int $relationLevel Recursion level
807 * @return array overview of relations found and added: Keys [table]:[uid], values array with table and id
808 * @see export_addFilesFromRelations()
809 */
810 public function export_addDBRelations($relationLevel = 0) {
811 // Traverse all "rels" registered for "records"
812 if (!is_array($this->dat['records'])) {
813 $this->error('There were no records available.');
814 return array();
815 }
816 $addR = array();
817 foreach ($this->dat['records'] as $k => $value) {
818 if (!is_array($this->dat['records'][$k])) {
819 continue;
820 }
821 foreach ($this->dat['records'][$k]['rels'] as $fieldname => $vR) {
822 // For all DB types of relations:
823 if ($vR['type'] == 'db') {
824 foreach ($vR['itemArray'] as $fI) {
825 $this->export_addDBRelations_registerRelation($fI, $addR);
826 }
827 }
828 // For all flex/db types of relations:
829 if ($vR['type'] == 'flex') {
830 // DB relations in flex form fields:
831 if (is_array($vR['flexFormRels']['db'])) {
832 foreach ($vR['flexFormRels']['db'] as $subList) {
833 foreach ($subList as $fI) {
834 $this->export_addDBRelations_registerRelation($fI, $addR);
835 }
836 }
837 }
838 // DB oriented soft references in flex form fields:
839 if (is_array($vR['flexFormRels']['softrefs'])) {
840 foreach ($vR['flexFormRels']['softrefs'] as $subList) {
841 foreach ($subList['keys'] as $spKey => $elements) {
842 foreach ($elements as $el) {
843 if ($el['subst']['type'] === 'db' && $this->includeSoftref($el['subst']['tokenID'])) {
844 list($tempTable, $tempUid) = explode(':', $el['subst']['recordRef']);
845 $fI = array(
846 'table' => $tempTable,
847 'id' => $tempUid
848 );
849 $this->export_addDBRelations_registerRelation($fI, $addR, $el['subst']['tokenID']);
850 }
851 }
852 }
853 }
854 }
855 }
856 // In any case, if there are soft refs:
857 if (is_array($vR['softrefs']['keys'])) {
858 foreach ($vR['softrefs']['keys'] as $spKey => $elements) {
859 foreach ($elements as $el) {
860 if ($el['subst']['type'] === 'db' && $this->includeSoftref($el['subst']['tokenID'])) {
861 list($tempTable, $tempUid) = explode(':', $el['subst']['recordRef']);
862 $fI = array(
863 'table' => $tempTable,
864 'id' => $tempUid
865 );
866 $this->export_addDBRelations_registerRelation($fI, $addR, $el['subst']['tokenID']);
867 }
868 }
869 }
870 }
871 }
872 }
873
874 // Now, if there were new records to add, do so:
875 if (!empty($addR)) {
876 foreach ($addR as $fI) {
877 // Get and set record:
878 $row = BackendUtility::getRecord($fI['table'], $fI['id']);
879 if (is_array($row)) {
880 $this->export_addRecord($fI['table'], $row, $relationLevel + 1);
881 }
882 // Set status message
883 // Relation pointers always larger than zero except certain "select" types with
884 // negative values pointing to uids - but that is not supported here.
885 if ($fI['id'] > 0) {
886 $rId = $fI['table'] . ':' . $fI['id'];
887 if (!isset($this->dat['records'][$rId])) {
888 $this->dat['records'][$rId] = 'NOT_FOUND';
889 $this->error('Relation record ' . $rId . ' was not found!');
890 }
891 }
892 }
893 }
894 // Return overview of relations found and added
895 return $addR;
896 }
897
898 /**
899 * Helper function for export_addDBRelations()
900 *
901 * @param array $fI Array with table/id keys to add
902 * @param array $addR Add array, passed by reference to be modified
903 * @param string $tokenID Softref Token ID, if applicable.
904 * @return void
905 * @see export_addDBRelations()
906 */
907 public function export_addDBRelations_registerRelation($fI, &$addR, $tokenID = '') {
908 $rId = $fI['table'] . ':' . $fI['id'];
909 if (
910 isset($GLOBALS['TCA'][$fI['table']]) && !$this->isTableStatic($fI['table']) && !$this->isExcluded($fI['table'], $fI['id'])
911 && (!$tokenID || $this->includeSoftref($tokenID)) && $this->inclRelation($fI['table'])
912 ) {
913 if (!isset($this->dat['records'][$rId])) {
914 // Set this record to be included since it is not already.
915 $addR[$rId] = $fI;
916 }
917 }
918 }
919
920 /**
921 * This adds all files in relations.
922 * Call this method AFTER adding all records including relations.
923 *
924 * @return void
925 * @see export_addDBRelations()
926 */
927 public function export_addFilesFromRelations() {
928 // Traverse all "rels" registered for "records"
929 if (!is_array($this->dat['records'])) {
930 $this->error('There were no records available.');
931 return;
932 }
933 foreach ($this->dat['records'] as $k => $value) {
934 if (!isset($this->dat['records'][$k]['rels']) || !is_array($this->dat['records'][$k]['rels'])) {
935 continue;
936 }
937 foreach ($this->dat['records'][$k]['rels'] as $fieldname => $vR) {
938 // For all file type relations:
939 if ($vR['type'] == 'file') {
940 foreach ($vR['newValueFiles'] as $key => $fI) {
941 $this->export_addFile($fI, $k, $fieldname);
942 // Remove the absolute reference to the file so it doesn't expose absolute paths from source server:
943 unset($this->dat['records'][$k]['rels'][$fieldname]['newValueFiles'][$key]['ID_absFile']);
944 }
945 }
946 // For all flex type relations:
947 if ($vR['type'] == 'flex') {
948 if (is_array($vR['flexFormRels']['file'])) {
949 foreach ($vR['flexFormRels']['file'] as $key => $subList) {
950 foreach ($subList as $subKey => $fI) {
951 $this->export_addFile($fI, $k, $fieldname);
952 // Remove the absolute reference to the file so it doesn't expose absolute paths from source server:
953 unset($this->dat['records'][$k]['rels'][$fieldname]['flexFormRels']['file'][$key][$subKey]['ID_absFile']);
954 }
955 }
956 }
957 // DB oriented soft references in flex form fields:
958 if (is_array($vR['flexFormRels']['softrefs'])) {
959 foreach ($vR['flexFormRels']['softrefs'] as $key => $subList) {
960 foreach ($subList['keys'] as $spKey => $elements) {
961 foreach ($elements as $subKey => $el) {
962 if ($el['subst']['type'] === 'file' && $this->includeSoftref($el['subst']['tokenID'])) {
963 // Create abs path and ID for file:
964 $ID_absFile = GeneralUtility::getFileAbsFileName(PATH_site . $el['subst']['relFileName']);
965 $ID = md5($el['subst']['relFileName']);
966 if ($ID_absFile) {
967 if (!$this->dat['files'][$ID]) {
968 $fI = array(
969 'filename' => PathUtility::basename($ID_absFile),
970 'ID_absFile' => $ID_absFile,
971 'ID' => $ID,
972 'relFileName' => $el['subst']['relFileName']
973 );
974 $this->export_addFile($fI, '_SOFTREF_');
975 }
976 $this->dat['records'][$k]['rels'][$fieldname]['flexFormRels']['softrefs'][$key]['keys'][$spKey][$subKey]['file_ID'] = $ID;
977 }
978 }
979 }
980 }
981 }
982 }
983 }
984 // In any case, if there are soft refs:
985 if (is_array($vR['softrefs']['keys'])) {
986 foreach ($vR['softrefs']['keys'] as $spKey => $elements) {
987 foreach ($elements as $subKey => $el) {
988 if ($el['subst']['type'] === 'file' && $this->includeSoftref($el['subst']['tokenID'])) {
989 // Create abs path and ID for file:
990 $ID_absFile = GeneralUtility::getFileAbsFileName(PATH_site . $el['subst']['relFileName']);
991 $ID = md5($el['subst']['relFileName']);
992 if ($ID_absFile) {
993 if (!$this->dat['files'][$ID]) {
994 $fI = array(
995 'filename' => PathUtility::basename($ID_absFile),
996 'ID_absFile' => $ID_absFile,
997 'ID' => $ID,
998 'relFileName' => $el['subst']['relFileName']
999 );
1000 $this->export_addFile($fI, '_SOFTREF_');
1001 }
1002 $this->dat['records'][$k]['rels'][$fieldname]['softrefs']['keys'][$spKey][$subKey]['file_ID'] = $ID;
1003 }
1004 }
1005 }
1006 }
1007 }
1008 }
1009 }
1010 }
1011
1012 /**
1013 * This adds all files from sys_file records
1014 *
1015 * @return void
1016 */
1017 public function export_addFilesFromSysFilesRecords() {
1018 if (!isset($this->dat['header']['records']['sys_file']) || !is_array($this->dat['header']['records']['sys_file'])) {
1019 return;
1020 }
1021 foreach ($this->dat['header']['records']['sys_file'] as $sysFileUid => $_) {
1022 $recordData = $this->dat['records']['sys_file:' . $sysFileUid]['data'];
1023 $file = ResourceFactory::getInstance()->createFileObject($recordData);
1024 $this->export_addSysFile($file);
1025 }
1026 }
1027
1028 /**
1029 * Adds a files content from a sys file record to the export memory
1030 *
1031 * @param \TYPO3\CMS\Core\Resource\File $file
1032 * @return void
1033 */
1034 public function export_addSysFile(\TYPO3\CMS\Core\Resource\File $file) {
1035 if ($file->getProperty('size') >= $this->maxFileSize) {
1036 $this->error('File ' . $file->getPublicUrl() . ' was larger (' . GeneralUtility::formatSize($file->getProperty('size')) . ') than the maxFileSize (' . GeneralUtility::formatSize($this->maxFileSize) . ')! Skipping.');
1037 return;
1038 }
1039 $fileContent = '';
1040 try {
1041 if (!$this->saveFilesOutsideExportFile) {
1042 $fileContent = $file->getContents();
1043 } else {
1044 $file->checkActionPermission('read');
1045 }
1046 } catch (\Exception $e) {
1047 $this->error('Error when trying to add file ' . $file->getCombinedIdentifier() . ': ' . $e->getMessage());
1048 return;
1049 }
1050 $fileUid = $file->getUid();
1051 $fileInfo = $file->getStorage()->getFileInfo($file);
1052 // we sadly have to cast it to string here, because the size property is also returning a string
1053 $fileSize = (string)$fileInfo['size'];
1054 if ($fileSize !== $file->getProperty('size')) {
1055 $this->error('File size of ' . $file->getCombinedIdentifier() . ' is not up-to-date in index! File added with current size.');
1056 $this->dat['records']['sys_file:' . $fileUid]['data']['size'] = $fileSize;
1057 }
1058 $fileSha1 = $file->getStorage()->hashFile($file, 'sha1');
1059 if ($fileSha1 !== $file->getProperty('sha1')) {
1060 $this->error('File sha1 hash of ' . $file->getCombinedIdentifier() . ' is not up-to-date in index! File added on current sha1.');
1061 $this->dat['records']['sys_file:' . $fileUid]['data']['sha1'] = $fileSha1;
1062 }
1063
1064 $fileRec = array();
1065 $fileRec['filesize'] = $fileSize;
1066 $fileRec['filename'] = $file->getProperty('name');
1067 $fileRec['filemtime'] = $file->getProperty('modification_date');
1068
1069 // build unique id based on the storage and the file identifier
1070 $fileId = md5($file->getStorage()->getUid() . ':' . $file->getProperty('identifier_hash'));
1071
1072 // Setting this data in the header
1073 $this->dat['header']['files_fal'][$fileId] = $fileRec;
1074
1075 if (!$this->saveFilesOutsideExportFile) {
1076 // ... and finally add the heavy stuff:
1077 $fileRec['content'] = $fileContent;
1078 } else {
1079 GeneralUtility::upload_copy_move($file->getForLocalProcessing(FALSE), $this->getTemporaryFilesPathForExport() . $file->getProperty('sha1'));
1080 }
1081 $fileRec['content_sha1'] = $fileSha1;
1082
1083 $this->dat['files_fal'][$fileId] = $fileRec;
1084 }
1085
1086
1087 /**
1088 * Adds a files content to the export memory
1089 *
1090 * @param array $fI File information with three keys: "filename" = filename without path, "ID_absFile" = absolute filepath to the file (including the filename), "ID" = md5 hash of "ID_absFile". "relFileName" is optional for files attached to records, but mandatory for soft referenced files (since the relFileName determines where such a file should be stored!)
1091 * @param string $recordRef If the file is related to a record, this is the id on the form [table]:[id]. Information purposes only.
1092 * @param string $fieldname If the file is related to a record, this is the field name it was related to. Information purposes only.
1093 * @return void
1094 */
1095 public function export_addFile($fI, $recordRef = '', $fieldname = '') {
1096 if (!@is_file($fI['ID_absFile'])) {
1097 $this->error($fI['ID_absFile'] . ' was not a file! Skipping.');
1098 return;
1099 }
1100 if (filesize($fI['ID_absFile']) >= $this->maxFileSize) {
1101 $this->error($fI['ID_absFile'] . ' was larger (' . GeneralUtility::formatSize(filesize($fI['ID_absFile'])) . ') than the maxFileSize (' . GeneralUtility::formatSize($this->maxFileSize) . ')! Skipping.');
1102 return;
1103 }
1104 $fileInfo = stat($fI['ID_absFile']);
1105 $fileRec = array();
1106 $fileRec['filesize'] = $fileInfo['size'];
1107 $fileRec['filename'] = PathUtility::basename($fI['ID_absFile']);
1108 $fileRec['filemtime'] = $fileInfo['mtime'];
1109 //for internal type file_reference
1110 $fileRec['relFileRef'] = PathUtility::stripPathSitePrefix($fI['ID_absFile']);
1111 if ($recordRef) {
1112 $fileRec['record_ref'] = $recordRef . '/' . $fieldname;
1113 }
1114 if ($fI['relFileName']) {
1115 $fileRec['relFileName'] = $fI['relFileName'];
1116 }
1117 // Setting this data in the header
1118 $this->dat['header']['files'][$fI['ID']] = $fileRec;
1119 // ... and for the recordlisting, why not let us know WHICH relations there was...
1120 if ($recordRef && $recordRef !== '_SOFTREF_') {
1121 $refParts = explode(':', $recordRef, 2);
1122 if (!is_array($this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'])) {
1123 $this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'] = array();
1124 }
1125 $this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'][] = $fI['ID'];
1126 }
1127 $fileMd5 = md5_file($fI['ID_absFile']);
1128 if (!$this->saveFilesOutsideExportFile) {
1129 // ... and finally add the heavy stuff:
1130 $fileRec['content'] = GeneralUtility::getUrl($fI['ID_absFile']);
1131 } else {
1132 GeneralUtility::upload_copy_move($fI['ID_absFile'], $this->getTemporaryFilesPathForExport() . $fileMd5);
1133 }
1134 $fileRec['content_md5'] = $fileMd5;
1135 $this->dat['files'][$fI['ID']] = $fileRec;
1136 // For soft references, do further processing:
1137 if ($recordRef === '_SOFTREF_') {
1138 // RTE files?
1139 if ($RTEoriginal = $this->getRTEoriginalFilename(PathUtility::basename($fI['ID_absFile']))) {
1140 $RTEoriginal_absPath = PathUtility::dirname($fI['ID_absFile']) . '/' . $RTEoriginal;
1141 if (@is_file($RTEoriginal_absPath)) {
1142 $RTEoriginal_ID = md5($RTEoriginal_absPath);
1143 $fileInfo = stat($RTEoriginal_absPath);
1144 $fileRec = array();
1145 $fileRec['filesize'] = $fileInfo['size'];
1146 $fileRec['filename'] = PathUtility::basename($RTEoriginal_absPath);
1147 $fileRec['filemtime'] = $fileInfo['mtime'];
1148 $fileRec['record_ref'] = '_RTE_COPY_ID:' . $fI['ID'];
1149 $this->dat['header']['files'][$fI['ID']]['RTE_ORIG_ID'] = $RTEoriginal_ID;
1150 // Setting this data in the header
1151 $this->dat['header']['files'][$RTEoriginal_ID] = $fileRec;
1152 $fileMd5 = md5_file($RTEoriginal_absPath);
1153 if (!$this->saveFilesOutsideExportFile) {
1154 // ... and finally add the heavy stuff:
1155 $fileRec['content'] = GeneralUtility::getUrl($RTEoriginal_absPath);
1156 } else {
1157 GeneralUtility::upload_copy_move($RTEoriginal_absPath, $this->getTemporaryFilesPathForExport() . $fileMd5);
1158 }
1159 $fileRec['content_md5'] = $fileMd5;
1160 $this->dat['files'][$RTEoriginal_ID] = $fileRec;
1161 } else {
1162 $this->error('RTE original file "' . PathUtility::stripPathSitePrefix($RTEoriginal_absPath) . '" was not found!');
1163 }
1164 }
1165 // Files with external media?
1166 // This is only done with files grabbed by a softreference parser since it is deemed improbable that hard-referenced files should undergo this treatment.
1167 $html_fI = pathinfo(PathUtility::basename($fI['ID_absFile']));
1168 if ($this->includeExtFileResources && GeneralUtility::inList($this->extFileResourceExtensions, strtolower($html_fI['extension']))) {
1169 $uniquePrefix = '###' . md5($GLOBALS['EXEC_TIME']) . '###';
1170 if (strtolower($html_fI['extension']) === 'css') {
1171 $prefixedMedias = explode($uniquePrefix, preg_replace('/(url[[:space:]]*\\([[:space:]]*["\']?)([^"\')]*)(["\']?[[:space:]]*\\))/i', '\\1' . $uniquePrefix . '\\2' . $uniquePrefix . '\\3', $fileRec['content']));
1172 } else {
1173 // html, htm:
1174 $htmlParser = GeneralUtility::makeInstance(\TYPO3\CMS\Core\Html\HtmlParser::class);
1175 $prefixedMedias = explode($uniquePrefix, $htmlParser->prefixResourcePath($uniquePrefix, $fileRec['content'], array(), $uniquePrefix));
1176 }
1177 $htmlResourceCaptured = FALSE;
1178 foreach ($prefixedMedias as $k => $v) {
1179 if ($k % 2) {
1180 $EXTres_absPath = GeneralUtility::resolveBackPath(PathUtility::dirname($fI['ID_absFile']) . '/' . $v);
1181 $EXTres_absPath = GeneralUtility::getFileAbsFileName($EXTres_absPath);
1182 if ($EXTres_absPath && GeneralUtility::isFirstPartOfStr($EXTres_absPath, PATH_site . $this->fileadminFolderName . '/') && @is_file($EXTres_absPath)) {
1183 $htmlResourceCaptured = TRUE;
1184 $EXTres_ID = md5($EXTres_absPath);
1185 $this->dat['header']['files'][$fI['ID']]['EXT_RES_ID'][] = $EXTres_ID;
1186 $prefixedMedias[$k] = '{EXT_RES_ID:' . $EXTres_ID . '}';
1187 // Add file to memory if it is not set already:
1188 if (!isset($this->dat['header']['files'][$EXTres_ID])) {
1189 $fileInfo = stat($EXTres_absPath);
1190 $fileRec = array();
1191 $fileRec['filesize'] = $fileInfo['size'];
1192 $fileRec['filename'] = PathUtility::basename($EXTres_absPath);
1193 $fileRec['filemtime'] = $fileInfo['mtime'];
1194 $fileRec['record_ref'] = '_EXT_PARENT_:' . $fI['ID'];
1195 // Media relative to the HTML file.
1196 $fileRec['parentRelFileName'] = $v;
1197 // Setting this data in the header
1198 $this->dat['header']['files'][$EXTres_ID] = $fileRec;
1199 // ... and finally add the heavy stuff:
1200 $fileRec['content'] = GeneralUtility::getUrl($EXTres_absPath);
1201 $fileRec['content_md5'] = md5($fileRec['content']);
1202 $this->dat['files'][$EXTres_ID] = $fileRec;
1203 }
1204 }
1205 }
1206 }
1207 if ($htmlResourceCaptured) {
1208 $this->dat['files'][$fI['ID']]['tokenizedContent'] = implode('', $prefixedMedias);
1209 }
1210 }
1211 }
1212 }
1213
1214 /**
1215 * If saveFilesOutsideExportFile is enabled, this function returns the path
1216 * where the files referenced in the export are copied to.
1217 *
1218 * @return string
1219 * @throws \RuntimeException
1220 * @see setSaveFilesOutsideExportFile()
1221 */
1222 public function getTemporaryFilesPathForExport() {
1223 if (!$this->saveFilesOutsideExportFile) {
1224 throw new \RuntimeException('You need to set saveFilesOutsideExportFile to TRUE before you want to get the temporary files path for export.', 1401205213);
1225 }
1226 if ($this->temporaryFilesPathForExport === NULL) {
1227 $temporaryFolderName = $this->getTemporaryFolderName();
1228 $this->temporaryFilesPathForExport = $temporaryFolderName . '/';
1229 }
1230 return $this->temporaryFilesPathForExport;
1231 }
1232
1233 /**
1234 *
1235 * @return string
1236 */
1237 protected function getTemporaryFolderName() {
1238 $temporaryPath = PATH_site . 'typo3temp/';
1239 do {
1240 $temporaryFolderName = $temporaryPath . 'export_temp_files_' . mt_rand(1, PHP_INT_MAX);
1241 } while (is_dir($temporaryFolderName));
1242 GeneralUtility::mkdir($temporaryFolderName);
1243 return $temporaryFolderName;
1244 }
1245
1246 /**
1247 * DB relations flattend to 1-dim array.
1248 * The list will be unique, no table/uid combination will appear twice.
1249 *
1250 * @param array $dbrels 2-dim Array of database relations organized by table key
1251 * @return array 1-dim array where entries are table:uid and keys are array with table/id
1252 */
1253 public function flatDBrels($dbrels) {
1254 $list = array();
1255 foreach ($dbrels as $dat) {
1256 if ($dat['type'] == 'db') {
1257 foreach ($dat['itemArray'] as $i) {
1258 $list[$i['table'] . ':' . $i['id']] = $i;
1259 }
1260 }
1261 if ($dat['type'] == 'flex' && is_array($dat['flexFormRels']['db'])) {
1262 foreach ($dat['flexFormRels']['db'] as $subList) {
1263 foreach ($subList as $i) {
1264 $list[$i['table'] . ':' . $i['id']] = $i;
1265 }
1266 }
1267 }
1268 }
1269 return $list;
1270 }
1271
1272 /**
1273 * Soft References flattend to 1-dim array.
1274 *
1275 * @param array $dbrels 2-dim Array of database relations organized by table key
1276 * @return array 1-dim array where entries are arrays with properties of the soft link found and keys are a unique combination of field, spKey, structure path if applicable and token ID
1277 */
1278 public function flatSoftRefs($dbrels) {
1279 $list = array();
1280 foreach ($dbrels as $field => $dat) {
1281 if (is_array($dat['softrefs']['keys'])) {
1282 foreach ($dat['softrefs']['keys'] as $spKey => $elements) {
1283 if (is_array($elements)) {
1284 foreach ($elements as $subKey => $el) {
1285 $lKey = $field . ':' . $spKey . ':' . $subKey;
1286 $list[$lKey] = array_merge(array('field' => $field, 'spKey' => $spKey), $el);
1287 // Add file_ID key to header - slightly "risky" way of doing this because if the calculation
1288 // changes for the same value in $this->records[...] this will not work anymore!
1289 if ($el['subst'] && $el['subst']['relFileName']) {
1290 $list[$lKey]['file_ID'] = md5(PATH_site . $el['subst']['relFileName']);
1291 }
1292 }
1293 }
1294 }
1295 }
1296 if ($dat['type'] == 'flex' && is_array($dat['flexFormRels']['softrefs'])) {
1297 foreach ($dat['flexFormRels']['softrefs'] as $structurePath => $subSoftrefs) {
1298 if (is_array($subSoftrefs['keys'])) {
1299 foreach ($subSoftrefs['keys'] as $spKey => $elements) {
1300 foreach ($elements as $subKey => $el) {
1301 $lKey = $field . ':' . $structurePath . ':' . $spKey . ':' . $subKey;
1302 $list[$lKey] = array_merge(array('field' => $field, 'spKey' => $spKey, 'structurePath' => $structurePath), $el);
1303 // Add file_ID key to header - slightly "risky" way of doing this because if the calculation
1304 // changes for the same value in $this->records[...] this will not work anymore!
1305 if ($el['subst'] && $el['subst']['relFileName']) {
1306 $list[$lKey]['file_ID'] = md5(PATH_site . $el['subst']['relFileName']);
1307 }
1308 }
1309 }
1310 }
1311 }
1312 }
1313 }
1314 return $list;
1315 }
1316
1317 /**
1318 * If include fields for a specific record type are set, the data
1319 * are filtered out with fields are not included in the fields.
1320 *
1321 * @param string $table The record type to be filtered
1322 * @param array $row The data to be filtered
1323 * @return array The filtered record row
1324 */
1325 protected function filterRecordFields($table, array $row) {
1326 if (isset($this->recordTypesIncludeFields[$table])) {
1327 $includeFields = array_unique(array_merge(
1328 $this->recordTypesIncludeFields[$table],
1329 $this->defaultRecordIncludeFields
1330 ));
1331 $newRow = array();
1332 foreach ($row as $key => $value) {
1333 if (in_array($key, $includeFields)) {
1334 $newRow[$key] = $value;
1335 }
1336 }
1337 } else {
1338 $newRow = $row;
1339 }
1340 return $newRow;
1341 }
1342
1343
1344 /**************************
1345 * File Output
1346 *************************/
1347
1348 /**
1349 * This compiles and returns the data content for an exported file
1350 *
1351 * @param string $type Type of output; "xml" gives xml, otherwise serialized array, possibly compressed.
1352 * @return string The output file stream
1353 */
1354 public function compileMemoryToFileContent($type = '') {
1355 if ($type == 'xml') {
1356 $out = $this->createXML();
1357 } else {
1358 $compress = $this->doOutputCompress();
1359 $out = '';
1360 // adding header:
1361 $out .= $this->addFilePart(serialize($this->dat['header']), $compress);
1362 // adding records:
1363 $out .= $this->addFilePart(serialize($this->dat['records']), $compress);
1364 // adding files:
1365 $out .= $this->addFilePart(serialize($this->dat['files']), $compress);
1366 // adding files_fal:
1367 $out .= $this->addFilePart(serialize($this->dat['files_fal']), $compress);
1368 }
1369 return $out;
1370 }
1371
1372 /**
1373 * Creates XML string from input array
1374 *
1375 * @return string XML content
1376 */
1377 public function createXML() {
1378 // Options:
1379 $options = array(
1380 'alt_options' => array(
1381 '/header' => array(
1382 'disableTypeAttrib' => TRUE,
1383 'clearStackPath' => TRUE,
1384 'parentTagMap' => array(
1385 'files' => 'file',
1386 'files_fal' => 'file',
1387 'records' => 'table',
1388 'table' => 'rec',
1389 'rec:rels' => 'relations',
1390 'relations' => 'element',
1391 'filerefs' => 'file',
1392 'pid_lookup' => 'page_contents',
1393 'header:relStaticTables' => 'static_tables',
1394 'static_tables' => 'tablename',
1395 'excludeMap' => 'item',
1396 'softrefCfg' => 'softrefExportMode',
1397 'extensionDependencies' => 'extkey',
1398 'softrefs' => 'softref_element'
1399 ),
1400 'alt_options' => array(
1401 '/pagetree' => array(
1402 'disableTypeAttrib' => TRUE,
1403 'useIndexTagForNum' => 'node',
1404 'parentTagMap' => array(
1405 'node:subrow' => 'node'
1406 )
1407 ),
1408 '/pid_lookup/page_contents' => array(
1409 'disableTypeAttrib' => TRUE,
1410 'parentTagMap' => array(
1411 'page_contents' => 'table'
1412 ),
1413 'grandParentTagMap' => array(
1414 'page_contents/table' => 'item'
1415 )
1416 )
1417 )
1418 ),
1419 '/records' => array(
1420 'disableTypeAttrib' => TRUE,
1421 'parentTagMap' => array(
1422 'records' => 'tablerow',
1423 'tablerow:data' => 'fieldlist',
1424 'tablerow:rels' => 'related',
1425 'related' => 'field',
1426 'field:itemArray' => 'relations',
1427 'field:newValueFiles' => 'filerefs',
1428 'field:flexFormRels' => 'flexform',
1429 'relations' => 'element',
1430 'filerefs' => 'file',
1431 'flexform:db' => 'db_relations',
1432 'flexform:file' => 'file_relations',
1433 'flexform:softrefs' => 'softref_relations',
1434 'softref_relations' => 'structurePath',
1435 'db_relations' => 'path',
1436 'file_relations' => 'path',
1437 'path' => 'element',
1438 'keys' => 'softref_key',
1439 'softref_key' => 'softref_element'
1440 ),
1441 'alt_options' => array(
1442 '/records/tablerow/fieldlist' => array(
1443 'useIndexTagForAssoc' => 'field'
1444 )
1445 )
1446 ),
1447 '/files' => array(
1448 'disableTypeAttrib' => TRUE,
1449 'parentTagMap' => array(
1450 'files' => 'file'
1451 )
1452 ),
1453 '/files_fal' => array(
1454 'disableTypeAttrib' => TRUE,
1455 'parentTagMap' => array(
1456 'files_fal' => 'file'
1457 )
1458 )
1459 )
1460 );
1461 // Creating XML file from $outputArray:
1462 $charset = $this->dat['header']['charset'] ?: 'utf-8';
1463 $XML = '<?xml version="1.0" encoding="' . $charset . '" standalone="yes" ?>' . LF;
1464 $XML .= GeneralUtility::array2xml($this->dat, '', 0, 'T3RecordDocument', 0, $options);
1465 return $XML;
1466 }
1467
1468 /**
1469 * Returns TRUE if the output should be compressed.
1470 *
1471 * @return bool TRUE if compression is possible AND requested.
1472 */
1473 public function doOutputCompress() {
1474 return $this->compress && !$this->dontCompress;
1475 }
1476
1477 /**
1478 * Returns a content part for a filename being build.
1479 *
1480 * @param array $data Data to store in part
1481 * @param bool $compress Compress file?
1482 * @return string Content stream.
1483 */
1484 public function addFilePart($data, $compress = FALSE) {
1485 if ($compress) {
1486 $data = gzcompress($data);
1487 }
1488 return md5($data) . ':' . ($compress ? '1' : '0') . ':' . str_pad(strlen($data), 10, '0', STR_PAD_LEFT) . ':' . $data . ':';
1489 }
1490
1491 /***********************
1492 * Import
1493 ***********************/
1494
1495 /**
1496 * Initialize all settings for the import
1497 *
1498 * @return void
1499 */
1500 protected function initializeImport() {
1501 // Set this flag to indicate that an import is being/has been done.
1502 $this->doesImport = 1;
1503 // Initialize:
1504 // These vars MUST last for the whole section not being cleared. They are used by the method setRelations() which are called at the end of the import session.
1505 $this->import_mapId = array();
1506 $this->import_newId = array();
1507 $this->import_newId_pids = array();
1508 // Temporary files stack initialized:
1509 $this->unlinkFiles = array();
1510 $this->alternativeFileName = array();
1511 $this->alternativeFilePath = array();
1512
1513 $this->initializeStorageObjects();
1514 }
1515
1516 /**
1517 * Initialize the all present storage objects
1518 *
1519 * @return void
1520 */
1521 protected function initializeStorageObjects() {
1522 /** @var $storageRepository \TYPO3\CMS\Core\Resource\StorageRepository */
1523 $storageRepository = GeneralUtility::makeInstance(\TYPO3\CMS\Core\Resource\StorageRepository::class);
1524 $this->storageObjects = $storageRepository->findAll();
1525 }
1526
1527 /**
1528 * Imports the internal data array to $pid.
1529 *
1530 * @param int $pid Page ID in which to import the content
1531 * @return void
1532 */
1533 public function importData($pid) {
1534
1535 $this->initializeImport();
1536
1537 // Write sys_file_storages first
1538 $this->writeSysFileStorageRecords();
1539 // Write sys_file records and write the binary file data
1540 $this->writeSysFileRecords();
1541 // Write records, first pages, then the rest
1542 // Fields with "hard" relations to database, files and flexform fields are kept empty during this run
1543 $this->writeRecords_pages($pid);
1544 $this->writeRecords_records($pid);
1545 // Finally all the file and DB record references must be fixed. This is done after all records have supposedly been written to database:
1546 // $this->import_mapId will indicate two things: 1) that a record WAS written to db and 2) that it has got a new id-number.
1547 $this->setRelations();
1548 // And when all DB relations are in place, we can fix file and DB relations in flexform fields (since data structures often depends on relations to a DS record):
1549 $this->setFlexFormRelations();
1550 // Unlink temporary files:
1551 $this->unlinkTempFiles();
1552 // Finally, traverse all records and process softreferences with substitution attributes.
1553 $this->processSoftReferences();
1554 // After all migrate records using sys_file_reference now
1555 if ($this->legacyImport) {
1556 $this->migrateLegacyImportRecords();
1557 }
1558 }
1559
1560 /**
1561 * Imports the sys_file_storage records from internal data array.
1562 *
1563 * @return void
1564 */
1565 protected function writeSysFileStorageRecords() {
1566 if (!isset($this->dat['header']['records']['sys_file_storage'])) {
1567 return;
1568 }
1569 $sysFileStorageUidsToBeResetToDefaultStorage = array();
1570 foreach ($this->dat['header']['records']['sys_file_storage'] as $sysFileStorageUid => $_) {
1571 $storageRecord = $this->dat['records']['sys_file_storage:' . $sysFileStorageUid]['data'];
1572 // continue with Local, writable and online storage only
1573 if ($storageRecord['driver'] === 'Local' && $storageRecord['is_writable'] && $storageRecord['is_online']) {
1574 $useThisStorageUidInsteadOfTheOneInImport = 0;
1575 /** @var $localStorage \TYPO3\CMS\Core\Resource\ResourceStorage */
1576 foreach ($this->storageObjects as $localStorage) {
1577 // check the available storage for Local, writable and online ones
1578 if ($localStorage->getDriverType() === 'Local' && $localStorage->isWritable() && $localStorage->isOnline()) {
1579 // check if there is already an identical storage present (same pathType and basePath)
1580 $storageRecordConfiguration = ResourceFactory::getInstance()->convertFlexFormDataToConfigurationArray($storageRecord['configuration']);
1581 $localStorageRecordConfiguration = $localStorage->getConfiguration();
1582 if (
1583 $storageRecordConfiguration['pathType'] === $localStorageRecordConfiguration['pathType']
1584 && $storageRecordConfiguration['basePath'] === $localStorageRecordConfiguration['basePath']
1585 ) {
1586 // same storage is already present
1587 $useThisStorageUidInsteadOfTheOneInImport = $localStorage->getUid();
1588 break;
1589 }
1590 }
1591 }
1592 if ($useThisStorageUidInsteadOfTheOneInImport > 0) {
1593 // same storage is already present; map the to be imported one to the present one
1594 $this->import_mapId['sys_file_storage'][$sysFileStorageUid] = $useThisStorageUidInsteadOfTheOneInImport;
1595 } else {
1596 // Local, writable and online storage. Is allowed to be used to later write files in.
1597 $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);
1598 }
1599 } else {
1600 // Storage with non Local drivers could be imported but must not be used to saves files in, because you
1601 // could not be sure, that this is supported. The default storage will be used in this case.
1602 // It could happen that non writable and non online storage will be created as dupes because you could not
1603 // check the detailed configuration options at this point
1604 $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);
1605 $sysFileStorageUidsToBeResetToDefaultStorage[] = $sysFileStorageUid;
1606 }
1607
1608 }
1609
1610 // Importing the added ones
1611 $tce = $this->getNewTCE();
1612 // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.
1613 $tce->reverseOrder = 1;
1614 $tce->isImporting = TRUE;
1615 $tce->start($this->import_data, array());
1616 $tce->process_datamap();
1617 $this->addToMapId($tce->substNEWwithIDs);
1618
1619 $defaultStorageUid = NULL;
1620 // get default storage
1621 $defaultStorage = ResourceFactory::getInstance()->getDefaultStorage();
1622 if ($defaultStorage !== NULL) {
1623 $defaultStorageUid = $defaultStorage->getUid();
1624 }
1625 foreach ($sysFileStorageUidsToBeResetToDefaultStorage as $sysFileStorageUidToBeResetToDefaultStorage) {
1626 $this->import_mapId['sys_file_storage'][$sysFileStorageUidToBeResetToDefaultStorage] = $defaultStorageUid;
1627 }
1628
1629 // unset the sys_file_storage records to prevent an import in writeRecords_records
1630 unset($this->dat['header']['records']['sys_file_storage']);
1631 }
1632
1633 /**
1634 * Imports the sys_file records and the binary files data from internal data array.
1635 *
1636 * @return void
1637 */
1638 protected function writeSysFileRecords() {
1639 if (!isset($this->dat['header']['records']['sys_file'])) {
1640 return;
1641 }
1642 $this->addGeneralErrorsByTable('sys_file');
1643
1644 // fetch fresh storage records from database
1645 $storageRecords = $this->fetchStorageRecords();
1646
1647 $defaultStorage = ResourceFactory::getInstance()->getDefaultStorage();
1648
1649 $sanitizedFolderMappings = array();
1650
1651 foreach ($this->dat['header']['records']['sys_file'] as $sysFileUid => $_) {
1652 $fileRecord = $this->dat['records']['sys_file:' . $sysFileUid]['data'];
1653
1654 $temporaryFile = NULL;
1655 // check if there is the right file already in the local folder
1656 if ($this->filesPathForImport !== NULL) {
1657 if (is_file($this->filesPathForImport . '/' . $fileRecord['sha1']) && sha1_file($this->filesPathForImport . '/' . $fileRecord['sha1']) === $fileRecord['sha1']) {
1658 $temporaryFile = $this->filesPathForImport . '/' . $fileRecord['sha1'];
1659 }
1660 }
1661
1662 // save file to disk
1663 if ($temporaryFile === NULL) {
1664 $fileId = md5($fileRecord['storage'] . ':' . $fileRecord['identifier_hash']);
1665 $temporaryFile = $this->writeTemporaryFileFromData($fileId);
1666 if ($temporaryFile === NULL) {
1667 // error on writing the file. Error message was already added
1668 continue;
1669 }
1670 }
1671
1672 $originalStorageUid = $fileRecord['storage'];
1673 $useStorageFromStorageRecords = FALSE;
1674
1675 // replace storage id, if an alternative one was registered
1676 if (isset($this->import_mapId['sys_file_storage'][$fileRecord['storage']])) {
1677 $fileRecord['storage'] = $this->import_mapId['sys_file_storage'][$fileRecord['storage']];
1678 $useStorageFromStorageRecords = TRUE;
1679 }
1680
1681 if (empty($fileRecord['storage']) && !$this->isFallbackStorage($fileRecord['storage'])) {
1682 // no storage for the file is defined, mostly because of a missing default storage.
1683 $this->error('Error: No storage for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $originalStorageUid . '"');
1684 continue;
1685 }
1686
1687 // using a storage from the local storage is only allowed, if the uid is present in the
1688 // mapping. Only in this case we could be sure, that it's a local, online and writable storage.
1689 if ($useStorageFromStorageRecords && isset($storageRecords[$fileRecord['storage']])) {
1690 /** @var $storage \TYPO3\CMS\Core\Resource\ResourceStorage */
1691 $storage = ResourceFactory::getInstance()->getStorageObject($fileRecord['storage'], $storageRecords[$fileRecord['storage']]);
1692 } elseif ($this->isFallbackStorage($fileRecord['storage'])) {
1693 $storage = ResourceFactory::getInstance()->getStorageObject(0);
1694 } elseif ($defaultStorage !== NULL) {
1695 $storage = $defaultStorage;
1696 } else {
1697 $this->error('Error: No storage available for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1698 continue;
1699 }
1700
1701 $newFile = NULL;
1702
1703 // check, if there is an identical file
1704 try {
1705 if ($storage->hasFile($fileRecord['identifier'])) {
1706 $file = $storage->getFile($fileRecord['identifier']);
1707 if ($file->getSha1() === $fileRecord['sha1']) {
1708 $newFile = $file;
1709 }
1710 }
1711 } catch (Exception $e) {}
1712
1713 if ($newFile === NULL) {
1714
1715 $folderName = PathUtility::dirname(ltrim($fileRecord['identifier'], '/'));
1716 if (in_array($folderName, $sanitizedFolderMappings)) {
1717 $folderName = $sanitizedFolderMappings[$folderName];
1718 }
1719 if (!$storage->hasFolder($folderName)) {
1720 try {
1721 $importFolder = $storage->createFolder($folderName);
1722 if ($importFolder->getIdentifier() !== $folderName && !in_array($folderName, $sanitizedFolderMappings)) {
1723 $sanitizedFolderMappings[$folderName] = $importFolder->getIdentifier();
1724 }
1725 } catch (Exception $e) {
1726 $this->error('Error: Folder could not be created for file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1727 continue;
1728 }
1729 } else {
1730 $importFolder = $storage->getFolder($folderName);
1731 }
1732
1733 try {
1734 /** @var $newFile \TYPO3\CMS\Core\Resource\File */
1735 $newFile = $storage->addFile($temporaryFile, $importFolder, $fileRecord['name']);
1736 } catch (Exception $e) {
1737 $this->error('Error: File could not be added to the storage: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1738 continue;
1739 }
1740
1741 if ($newFile->getSha1() !== $fileRecord['sha1']) {
1742 $this->error('Error: The hash of the written file is not identical to the import data! File could be corrupted! File: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1743 }
1744 }
1745
1746 // save the new uid in the import id map
1747 $this->import_mapId['sys_file'][$fileRecord['uid']] = $newFile->getUid();
1748 $this->fixUidLocalInSysFileReferenceRecords($fileRecord['uid'], $newFile->getUid());
1749
1750 }
1751
1752 // unset the sys_file records to prevent an import in writeRecords_records
1753 unset($this->dat['header']['records']['sys_file']);
1754 }
1755
1756 /**
1757 * Checks if the $storageId is the id of the fallback storage
1758 *
1759 * @param int|string $storageId
1760 * @return bool
1761 */
1762 protected function isFallbackStorage($storageId) {
1763 return $storageId === 0 || $storageId === '0';
1764 }
1765
1766 /**
1767 * Normally the importer works like the following:
1768 * Step 1: import the records with cleared field values of relation fields (see addSingle())
1769 * Step 2: update the records with the right relation ids (see setRelations())
1770 *
1771 * In step 2 the saving fields of type "relation to sys_file_reference" checks the related sys_file_reference
1772 * record (created in step 1) with the FileExtensionFilter for matching file extensions of the related file.
1773 * To make this work correct, the uid_local of sys_file_reference records has to be not empty AND has to
1774 * relate to the correct (imported) sys_file record uid!!!
1775 *
1776 * This is fixed here.
1777 *
1778 * @param int $oldFileUid
1779 * @param int $newFileUid
1780 * @return void
1781 */
1782 protected function fixUidLocalInSysFileReferenceRecords($oldFileUid, $newFileUid) {
1783 if (!isset($this->dat['header']['records']['sys_file_reference'])) {
1784 return;
1785 }
1786
1787 foreach ($this->dat['header']['records']['sys_file_reference'] as $sysFileReferenceUid => $_) {
1788 $fileReferenceRecord = $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'];
1789 if ($fileReferenceRecord['uid_local'] == $oldFileUid) {
1790 $fileReferenceRecord['uid_local'] = $newFileUid;
1791 $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'] = $fileReferenceRecord;
1792 }
1793 }
1794 }
1795
1796 /**
1797 * Initializes the folder for legacy imports as subfolder of backend users default upload folder
1798 *
1799 * @return void
1800 */
1801 protected function initializeLegacyImportFolder() {
1802 /** @var \TYPO3\CMS\Core\Resource\Folder $folder */
1803 $folder = $GLOBALS['BE_USER']->getDefaultUploadFolder();
1804 if ($folder === FALSE) {
1805 $this->error('Error: the backend users default upload folder is missing! No files will be imported!');
1806 }
1807 if (!$folder->hasFolder($this->legacyImportTargetPath)) {
1808 try {
1809 $this->legacyImportFolder = $folder->createFolder($this->legacyImportTargetPath);
1810 } catch (\TYPO3\CMS\Core\Exception $e) {
1811 $this->error('Error: the import folder in the default upload folder could not be created! No files will be imported!');
1812 }
1813 } else {
1814 $this->legacyImportFolder = $folder->getSubFolder($this->legacyImportTargetPath);
1815 }
1816
1817 }
1818
1819 /**
1820 * Fetched fresh storage records from database because the new imported
1821 * ones are not in cached data of the StorageRepository
1822 *
1823 * @return bool|array
1824 */
1825 protected function fetchStorageRecords() {
1826 $whereClause = BackendUtility::BEenableFields('sys_file_storage');
1827 $whereClause .= BackendUtility::deleteClause('sys_file_storage');
1828
1829 $rows = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows(
1830 '*',
1831 'sys_file_storage',
1832 '1=1' . $whereClause,
1833 '',
1834 '',
1835 '',
1836 'uid'
1837 );
1838
1839 return $rows;
1840 }
1841
1842 /**
1843 * Writes the file from import array to temp dir and returns the filename of it.
1844 *
1845 * @param string $fileId
1846 * @param string $dataKey
1847 * @return string Absolute filename of the temporary filename of the file
1848 */
1849 protected function writeTemporaryFileFromData($fileId, $dataKey = 'files_fal') {
1850 $temporaryFilePath = NULL;
1851 if (is_array($this->dat[$dataKey][$fileId])) {
1852 $temporaryFilePathInternal = GeneralUtility::tempnam('import_temp_');
1853 GeneralUtility::writeFile($temporaryFilePathInternal, $this->dat[$dataKey][$fileId]['content']);
1854 clearstatcache();
1855 if (@is_file($temporaryFilePathInternal)) {
1856 $this->unlinkFiles[] = $temporaryFilePathInternal;
1857 if (filesize($temporaryFilePathInternal) == $this->dat[$dataKey][$fileId]['filesize']) {
1858 $temporaryFilePath = $temporaryFilePathInternal;
1859 } else {
1860 $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' had a size (' . filesize($temporaryFilePathInternal) . ') different from the original (' . $this->dat[$dataKey][$fileId]['filesize'] . ')', 1);
1861 }
1862 } else {
1863 $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' was not written as it should have been!', 1);
1864 }
1865 } else {
1866 $this->error('Error: No file found for ID ' . $fileId, 1);
1867 }
1868 return $temporaryFilePath;
1869 }
1870
1871 /**
1872 * Writing pagetree/pages to database:
1873 *
1874 * @param int $pid PID in which to import. If the operation is an update operation, the root of the page tree inside will be moved to this PID unless it is the same as the root page from the import
1875 * @return void
1876 * @see writeRecords_records()
1877 */
1878 public function writeRecords_pages($pid) {
1879 // First, write page structure if any:
1880 if (is_array($this->dat['header']['records']['pages'])) {
1881 $this->addGeneralErrorsByTable('pages');
1882 // $pageRecords is a copy of the pages array in the imported file. Records here are unset one by one when the addSingle function is called.
1883 $pageRecords = $this->dat['header']['records']['pages'];
1884 $this->import_data = array();
1885 // First add page tree if any
1886 if (is_array($this->dat['header']['pagetree'])) {
1887 $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);
1888 foreach ($pagesFromTree as $uid) {
1889 $thisRec = $this->dat['header']['records']['pages'][$uid];
1890 // PID: Set the main $pid, unless a NEW-id is found
1891 $setPid = isset($this->import_newId_pids[$thisRec['pid']]) ? $this->import_newId_pids[$thisRec['pid']] : $pid;
1892 $this->addSingle('pages', $uid, $setPid);
1893 unset($pageRecords[$uid]);
1894 }
1895 }
1896 // Then add all remaining pages not in tree on root level:
1897 if (!empty($pageRecords)) {
1898 $remainingPageUids = array_keys($pageRecords);
1899 foreach ($remainingPageUids as $pUid) {
1900 $this->addSingle('pages', $pUid, $pid);
1901 }
1902 }
1903 // Now write to database:
1904 $tce = $this->getNewTCE();
1905 $tce->isImporting = TRUE;
1906 $this->callHook('before_writeRecordsPages', array(
1907 'tce' => &$tce,
1908 'data' => &$this->import_data
1909 ));
1910 $tce->suggestedInsertUids = $this->suggestedInsertUids;
1911 $tce->start($this->import_data, array());
1912 $tce->process_datamap();
1913 $this->callHook('after_writeRecordsPages', array(
1914 'tce' => &$tce
1915 ));
1916 // post-processing: Registering new ids (end all tcemain sessions with this)
1917 $this->addToMapId($tce->substNEWwithIDs);
1918 // In case of an update, order pages from the page tree correctly:
1919 if ($this->update && is_array($this->dat['header']['pagetree'])) {
1920 $this->writeRecords_pages_order($pid);
1921 }
1922 }
1923 }
1924
1925 /**
1926 * Organize all updated pages in page tree so they are related like in the import file
1927 * Only used for updates and when $this->dat['header']['pagetree'] is an array.
1928 *
1929 * @param int $pid Page id in which to import
1930 * @return void
1931 * @access private
1932 * @see writeRecords_pages(), writeRecords_records_order()
1933 */
1934 public function writeRecords_pages_order($pid) {
1935 $cmd_data = array();
1936 // Get uid-pid relations and traverse them in order to map to possible new IDs
1937 $pidsFromTree = $this->flatInversePageTree_pid($this->dat['header']['pagetree']);
1938 foreach ($pidsFromTree as $origPid => $newPid) {
1939 if ($newPid >= 0 && $this->dontIgnorePid('pages', $origPid)) {
1940 // If the page had a new id (because it was created) use that instead!
1941 if (substr($this->import_newId_pids[$origPid], 0, 3) === 'NEW') {
1942 if ($this->import_mapId['pages'][$origPid]) {
1943 $mappedPid = $this->import_mapId['pages'][$origPid];
1944 $cmd_data['pages'][$mappedPid]['move'] = $newPid;
1945 }
1946 } else {
1947 $cmd_data['pages'][$origPid]['move'] = $newPid;
1948 }
1949 }
1950 }
1951 // Execute the move commands if any:
1952 if (!empty($cmd_data)) {
1953 $tce = $this->getNewTCE();
1954 $this->callHook('before_writeRecordsPagesOrder', array(
1955 'tce' => &$tce,
1956 'data' => &$cmd_data
1957 ));
1958 $tce->start(array(), $cmd_data);
1959 $tce->process_cmdmap();
1960 $this->callHook('after_writeRecordsPagesOrder', array(
1961 'tce' => &$tce
1962 ));
1963 }
1964 }
1965
1966 /**
1967 * Write all database records except pages (writtein in writeRecords_pages())
1968 *
1969 * @param int $pid Page id in which to import
1970 * @return void
1971 * @see writeRecords_pages()
1972 */
1973 public function writeRecords_records($pid) {
1974 // Write the rest of the records
1975 $this->import_data = array();
1976 if (is_array($this->dat['header']['records'])) {
1977 foreach ($this->dat['header']['records'] as $table => $recs) {
1978 $this->addGeneralErrorsByTable($table);
1979 if ($table != 'pages') {
1980 foreach ($recs as $uid => $thisRec) {
1981 // PID: Set the main $pid, unless a NEW-id is found
1982 $setPid = isset($this->import_mapId['pages'][$thisRec['pid']])
1983 ? (int)$this->import_mapId['pages'][$thisRec['pid']]
1984 : (int)$pid;
1985 if (is_array($GLOBALS['TCA'][$table]) && isset($GLOBALS['TCA'][$table]['ctrl']['rootLevel'])) {
1986 $rootLevelSetting = (int)$GLOBALS['TCA'][$table]['ctrl']['rootLevel'];
1987 if ($rootLevelSetting === 1) {
1988 $setPid = 0;
1989 } elseif ($rootLevelSetting === 0 && $setPid === 0) {
1990 $this->error('Error: Record type ' . $table . ' is not allowed on pid 0');
1991 continue;
1992 }
1993 }
1994 // Add record:
1995 $this->addSingle($table, $uid, $setPid);
1996 }
1997 }
1998 }
1999 } else {
2000 $this->error('Error: No records defined in internal data array.');
2001 }
2002 // Now write to database:
2003 $tce = $this->getNewTCE();
2004 $this->callHook('before_writeRecordsRecords', array(
2005 'tce' => &$tce,
2006 'data' => &$this->import_data
2007 ));
2008 $tce->suggestedInsertUids = $this->suggestedInsertUids;
2009 // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.
2010 $tce->reverseOrder = 1;
2011 $tce->isImporting = TRUE;
2012 $tce->start($this->import_data, array());
2013 $tce->process_datamap();
2014 $this->callHook('after_writeRecordsRecords', array(
2015 'tce' => &$tce
2016 ));
2017 // post-processing: Removing files and registering new ids (end all tcemain sessions with this)
2018 $this->addToMapId($tce->substNEWwithIDs);
2019 // In case of an update, order pages from the page tree correctly:
2020 if ($this->update) {
2021 $this->writeRecords_records_order($pid);
2022 }
2023 }
2024
2025 /**
2026 * Organize all updated record to their new positions.
2027 * Only used for updates
2028 *
2029 * @param int $mainPid Main PID into which we import.
2030 * @return void
2031 * @access private
2032 * @see writeRecords_records(), writeRecords_pages_order()
2033 */
2034 public function writeRecords_records_order($mainPid) {
2035 $cmd_data = array();
2036 if (is_array($this->dat['header']['pagetree'])) {
2037 $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);
2038 } else {
2039 $pagesFromTree = array();
2040 }
2041 if (is_array($this->dat['header']['pid_lookup'])) {
2042 foreach ($this->dat['header']['pid_lookup'] as $pid => $recList) {
2043 $newPid = isset($this->import_mapId['pages'][$pid]) ? $this->import_mapId['pages'][$pid] : $mainPid;
2044 if (\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($newPid)) {
2045 foreach ($recList as $tableName => $uidList) {
2046 // If $mainPid===$newPid then we are on root level and we can consider to move pages as well!
2047 // (they will not be in the page tree!)
2048 if (($tableName != 'pages' || !$pagesFromTree[$pid]) && is_array($uidList)) {
2049 $uidList = array_reverse(array_keys($uidList));
2050 foreach ($uidList as $uid) {
2051 if ($this->dontIgnorePid($tableName, $uid)) {
2052 $cmd_data[$tableName][$uid]['move'] = $newPid;
2053 } else {
2054
2055 }
2056 }
2057 }
2058 }
2059 }
2060 }
2061 }
2062 // Execute the move commands if any:
2063 if (!empty($cmd_data)) {
2064 $tce = $this->getNewTCE();
2065 $this->callHook('before_writeRecordsRecordsOrder', array(
2066 'tce' => &$tce,
2067 'data' => &$cmd_data
2068 ));
2069 $tce->start(array(), $cmd_data);
2070 $tce->process_cmdmap();
2071 $this->callHook('after_writeRecordsRecordsOrder', array(
2072 'tce' => &$tce
2073 ));
2074 }
2075 }
2076
2077 /**
2078 * Adds a single record to the $importData array. Also copies files to tempfolder.
2079 * However all File/DB-references and flexform field contents are set to blank for now!
2080 * That is done with setRelations() later
2081 *
2082 * @param string $table Table name (from import memory)
2083 * @param int $uid Record UID (from import memory)
2084 * @param int $pid Page id
2085 * @return void
2086 * @see writeRecords()
2087 */
2088 public function addSingle($table, $uid, $pid) {
2089 if ($this->import_mode[$table . ':' . $uid] === 'exclude') {
2090 return;
2091 }
2092 $record = $this->dat['records'][$table . ':' . $uid]['data'];
2093 if (is_array($record)) {
2094 if ($this->update && $this->doesRecordExist($table, $uid) && $this->import_mode[$table . ':' . $uid] !== 'as_new') {
2095 $ID = $uid;
2096 } elseif ($table === 'sys_file_metadata' && $record['sys_language_uid'] == '0' && $this->import_mapId['sys_file'][$record['file']]) {
2097 // on adding sys_file records the belonging sys_file_metadata record was also created
2098 // if there is one the record need to be overwritten instead of creating a new one.
2099 $recordInDatabase = $GLOBALS['TYPO3_DB']->exec_SELECTgetSingleRow(
2100 'uid',
2101 'sys_file_metadata',
2102 'file = ' . $this->import_mapId['sys_file'][$record['file']] . ' AND sys_language_uid = 0 AND pid = 0'
2103 );
2104 // if no record could be found, $this->import_mapId['sys_file'][$record['file']] is pointing
2105 // to a file, that was already there, thus a new metadata record should be created
2106 if (is_array($recordInDatabase)) {
2107 $this->import_mapId['sys_file_metadata'][$record['uid']] = $recordInDatabase['uid'];
2108 $ID = $recordInDatabase['uid'];
2109 } else {
2110 $ID = uniqid('NEW', TRUE);
2111 }
2112
2113 } else {
2114 $ID = uniqid('NEW', TRUE);
2115 }
2116 $this->import_newId[$table . ':' . $ID] = array('table' => $table, 'uid' => $uid);
2117 if ($table == 'pages') {
2118 $this->import_newId_pids[$uid] = $ID;
2119 }
2120 // Set main record data:
2121 $this->import_data[$table][$ID] = $record;
2122 $this->import_data[$table][$ID]['tx_impexp_origuid'] = $this->import_data[$table][$ID]['uid'];
2123 // Reset permission data:
2124 if ($table === 'pages') {
2125 // Have to reset the user/group IDs so pages are owned by importing user. Otherwise strange things may happen for non-admins!
2126 unset($this->import_data[$table][$ID]['perms_userid']);
2127 unset($this->import_data[$table][$ID]['perms_groupid']);
2128 }
2129 // PID and UID:
2130 unset($this->import_data[$table][$ID]['uid']);
2131 // Updates:
2132 if (\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($ID)) {
2133 unset($this->import_data[$table][$ID]['pid']);
2134 } else {
2135 // Inserts:
2136 $this->import_data[$table][$ID]['pid'] = $pid;
2137 if (($this->import_mode[$table . ':' . $uid] === 'force_uid' && $this->update || $this->force_all_UIDS) && $GLOBALS['BE_USER']->isAdmin()) {
2138 $this->import_data[$table][$ID]['uid'] = $uid;
2139 $this->suggestedInsertUids[$table . ':' . $uid] = 'DELETE';
2140 }
2141 }
2142 // Setting db/file blank:
2143 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
2144 switch ((string)$config['type']) {
2145 case 'db':
2146
2147 case 'file':
2148 // Fixed later in ->setRelations() [because we need to know ALL newly created IDs before we can map relations!]
2149 // In the meantime we set NO values for relations.
2150 //
2151 // BUT for field uid_local of table sys_file_reference the relation MUST not be cleared here,
2152 // because the value is already the uid of the right imported sys_file record.
2153 // @see fixUidLocalInSysFileReferenceRecords()
2154 // If it's empty or a uid to another record the FileExtensionFilter will throw an exception or
2155 // delete the reference record if the file extension of the related record doesn't match.
2156 if ($table !== 'sys_file_reference' && $field !== 'uid_local') {
2157 $this->import_data[$table][$ID][$field] = '';
2158 }
2159 break;
2160 case 'flex':
2161 // Fixed later in setFlexFormRelations()
2162 // In the meantime we set NO value for flexforms - this is mainly because file references
2163 // inside will not be processed properly; In fact references will point to no file
2164 // or existing files (in which case there will be double-references which is a big problem of course!)
2165 $this->import_data[$table][$ID][$field] = '';
2166 break;
2167 }
2168 }
2169 } elseif ($table . ':' . $uid != 'pages:0') {
2170 // On root level we don't want this error message.
2171 $this->error('Error: no record was found in data array!', 1);
2172 }
2173 }
2174
2175 /**
2176 * Registers the substNEWids in memory.
2177 *
2178 * @param array $substNEWwithIDs From tcemain to be merged into internal mapping variable in this object
2179 * @return void
2180 * @see writeRecords()
2181 */
2182 public function addToMapId($substNEWwithIDs) {
2183 foreach ($this->import_data as $table => $recs) {
2184 foreach ($recs as $id => $value) {
2185 $old_uid = $this->import_newId[$table . ':' . $id]['uid'];
2186 if (isset($substNEWwithIDs[$id])) {
2187 $this->import_mapId[$table][$old_uid] = $substNEWwithIDs[$id];
2188 } elseif ($this->update) {
2189 // Map same ID to same ID....
2190 $this->import_mapId[$table][$old_uid] = $id;
2191 } else {
2192 // if $this->import_mapId contains already the right mapping, skip the error msg.
2193 // See special handling of sys_file_metadata in addSingle() => nothing to do
2194 if (!($table === 'sys_file_metadata' && isset($this->import_mapId[$table][$old_uid]) && $this->import_mapId[$table][$old_uid] == $id)) {
2195 $this->error('Possible error: ' . $table . ':' . $old_uid . ' had no new id assigned to it. This indicates that the record was not added to database during import. Please check changelog!', 1);
2196 }
2197
2198 }
2199 }
2200 }
2201 }
2202
2203 /**
2204 * Returns a new $TCE object
2205 *
2206 * @return DataHandler $TCE object
2207 */
2208 public function getNewTCE() {
2209 $tce = GeneralUtility::makeInstance(\TYPO3\CMS\Core\DataHandling\DataHandler::class);
2210 $tce->stripslashes_values = 0;
2211 $tce->dontProcessTransformations = 1;
2212 $tce->enableLogging = $this->enableLogging;
2213 $tce->alternativeFileName = $this->alternativeFileName;
2214 $tce->alternativeFilePath = $this->alternativeFilePath;
2215 return $tce;
2216 }
2217
2218 /**
2219 * Cleaning up all the temporary files stored in typo3temp/ folder
2220 *
2221 * @return void
2222 */
2223 public function unlinkTempFiles() {
2224 foreach ($this->unlinkFiles as $fileName) {
2225 if (GeneralUtility::isFirstPartOfStr($fileName, PATH_site . 'typo3temp/')) {
2226 GeneralUtility::unlink_tempfile($fileName);
2227 clearstatcache();
2228 if (is_file($fileName)) {
2229 $this->error('Error: ' . $fileName . ' was NOT unlinked as it should have been!', 1);
2230 }
2231 } else {
2232 $this->error('Error: ' . $fileName . ' was not in temp-path. Not removed!', 1);
2233 }
2234 }
2235 $this->unlinkFiles = array();
2236 }
2237
2238 /***************************
2239 * Import / Relations setting
2240 ***************************/
2241
2242 /**
2243 * At the end of the import process all file and DB relations should be set properly (that is relations
2244 * to imported records are all re-created so imported records are correctly related again)
2245 * Relations in flexform fields are processed in setFlexFormRelations() after this function
2246 *
2247 * @return void
2248 * @see setFlexFormRelations()
2249 */
2250 public function setRelations() {
2251 $updateData = array();
2252 // import_newId contains a register of all records that was in the import memorys "records" key
2253 foreach ($this->import_newId as $nId => $dat) {
2254 $table = $dat['table'];
2255 $uid = $dat['uid'];
2256 // original UID - NOT the new one!
2257 // If the record has been written and received a new id, then proceed:
2258 if (is_array($this->import_mapId[$table]) && isset($this->import_mapId[$table][$uid])) {
2259 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
2260 if (is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {
2261 $thisNewPageUid = 0;
2262 if ($this->legacyImport) {
2263 if ($table != 'pages') {
2264 $oldPid = $this->dat['records'][$table . ':' . $uid]['data']['pid'];
2265 $thisNewPageUid = BackendUtility::wsMapId($table, $this->import_mapId['pages'][$oldPid]);
2266 } else {
2267 $thisNewPageUid = $thisNewUid;
2268 }
2269 }
2270 // Traverse relation fields of each record
2271 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
2272 // uid_local of sys_file_reference needs no update because the correct reference uid was already written
2273 // @see ImportExport::fixUidLocalInSysFileReferenceRecords()
2274 if ($table === 'sys_file_reference' && $field === 'uid_local') {
2275 continue;
2276 }
2277 switch ((string)$config['type']) {
2278 case 'db':
2279 if (is_array($config['itemArray']) && !empty($config['itemArray'])) {
2280 $itemConfig = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
2281 $valArray = $this->setRelations_db($config['itemArray'], $itemConfig);
2282 $updateData[$table][$thisNewUid][$field] = implode(',', $valArray);
2283 }
2284 break;
2285 case 'file':
2286 if (is_array($config['newValueFiles']) && !empty($config['newValueFiles'])) {
2287 $valArr = array();
2288 foreach ($config['newValueFiles'] as $fI) {
2289 $valArr[] = $this->import_addFileNameToBeCopied($fI);
2290 }
2291 if ($this->legacyImport && $this->legacyImportFolder === NULL && isset($this->legacyImportMigrationTables[$table][$field])) {
2292 // Do nothing - the legacy import folder is missing
2293 } elseif ($this->legacyImport && $this->legacyImportFolder !== NULL && isset($this->legacyImportMigrationTables[$table][$field])) {
2294 $refIds = array();
2295 foreach ($valArr as $tempFile) {
2296 $fileName = $this->alternativeFileName[$tempFile];
2297 $fileObject = NULL;
2298
2299 try {
2300 // check, if there is alreay the same file in the folder
2301 if ($this->legacyImportFolder->hasFile($fileName)) {
2302 $fileStorage = $this->legacyImportFolder->getStorage();
2303 $file = $fileStorage->getFile($this->legacyImportFolder->getIdentifier() . $fileName);
2304 if ($file->getSha1() === sha1_file($tempFile)) {
2305 $fileObject = $file;
2306 }
2307 }
2308 } catch (Exception $e) {}
2309
2310 if ($fileObject === NULL) {
2311 try {
2312 $fileObject = $this->legacyImportFolder->addFile($tempFile, $fileName, DuplicationBehavior::RENAME);
2313 } catch (\TYPO3\CMS\Core\Exception $e) {
2314 $this->error('Error: no file could be added to the storage for file name' . $this->alternativeFileName[$tempFile]);
2315 }
2316 }
2317 if ($fileObject !== NULL) {
2318 $refId = uniqid('NEW', TRUE);
2319 $refIds[] = $refId;
2320 $updateData['sys_file_reference'][$refId] = array(
2321 'uid_local' => $fileObject->getUid(),
2322 'uid_foreign' => $thisNewUid, // uid of your content record
2323 'tablenames' => $table,
2324 'fieldname' => $field,
2325 'pid' => $thisNewPageUid, // parent id of the parent page
2326 'table_local' => 'sys_file',
2327 );
2328 }
2329 }
2330 $updateData[$table][$thisNewUid][$field] = implode(',', $refIds);
2331 if (!empty($this->legacyImportMigrationTables[$table][$field])) {
2332 $this->legacyImportMigrationRecords[$table][$thisNewUid][$field] = $refIds;
2333 }
2334 } else {
2335 $updateData[$table][$thisNewUid][$field] = implode(',', $valArr);
2336 }
2337 }
2338 break;
2339 }
2340 }
2341 } else {
2342 $this->error('Error: no record was found in data array!', 1);
2343 }
2344 } else {
2345 $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')', 1);
2346 }
2347 }
2348 if (!empty($updateData)) {
2349 $tce = $this->getNewTCE();
2350 $tce->isImporting = TRUE;
2351 $this->callHook('before_setRelation', array(
2352 'tce' => &$tce,
2353 'data' => &$updateData
2354 ));
2355 $tce->start($updateData, array());
2356 $tce->process_datamap();
2357 // Replace the temporary "NEW" ids with the final ones.
2358 foreach ($this->legacyImportMigrationRecords as $table => $records) {
2359 foreach ($records as $uid => $fields) {
2360 foreach ($fields as $field => $referenceIds) {
2361 foreach ($referenceIds as $key => $referenceId) {
2362 $this->legacyImportMigrationRecords[$table][$uid][$field][$key] = $tce->substNEWwithIDs[$referenceId];
2363 }
2364 }
2365 }
2366 }
2367 $this->callHook('after_setRelations', array(
2368 'tce' => &$tce
2369 ));
2370 }
2371 }
2372
2373 /**
2374 * Maps relations for database
2375 *
2376 * @param array $itemArray Array of item sets (table/uid) from a dbAnalysis object
2377 * @param array $itemConfig Array of TCA config of the field the relation to be set on
2378 * @return array Array with values [table]_[uid] or [uid] for field of type group / internal_type file_reference. These values have the regular tcemain-input group/select type which means they will automatically be processed into a uid-list or MM relations.
2379 */
2380 public function setRelations_db($itemArray, $itemConfig) {
2381 $valArray = array();
2382 foreach ($itemArray as $relDat) {
2383 if (is_array($this->import_mapId[$relDat['table']]) && isset($this->import_mapId[$relDat['table']][$relDat['id']])) {
2384 // Since non FAL file relation type group internal_type file_reference are handled as reference to
2385 // sys_file records Datahandler requires the value as uid of the the related sys_file record only
2386 if ($itemConfig['type'] === 'group' && $itemConfig['internal_type'] === 'file_reference') {
2387 $value = $this->import_mapId[$relDat['table']][$relDat['id']];
2388 } elseif ($itemConfig['type'] === 'input' && isset($itemConfig['wizards']['link'])) {
2389 // If an input field has a relation to a sys_file record this need to be converted back to
2390 // the public path. But use getPublicUrl here, because could normally only be a local file path.
2391 $fileUid = $this->import_mapId[$relDat['table']][$relDat['id']];
2392 // Fallback value
2393 $value = 'file:' . $fileUid;
2394 try {
2395 $file = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->retrieveFileOrFolderObject($fileUid);
2396 } catch (\Exception $e) {}
2397 if ($file instanceof \TYPO3\CMS\Core\Resource\FileInterface) {
2398 $value = $file->getPublicUrl();
2399 }
2400 } else {
2401 $value = $relDat['table'] . '_' . $this->import_mapId[$relDat['table']][$relDat['id']];
2402 }
2403 $valArray[] = $value;
2404 } elseif ($this->isTableStatic($relDat['table']) || $this->isExcluded($relDat['table'], $relDat['id']) || $relDat['id'] < 0) {
2405 // Checking for less than zero because some select types could contain negative values,
2406 // eg. fe_groups (-1, -2) and sys_language (-1 = ALL languages). This must be handled on both export and import.
2407 $valArray[] = $relDat['table'] . '_' . $relDat['id'];
2408 } else {
2409 $this->error('Lost relation: ' . $relDat['table'] . ':' . $relDat['id'], 1);
2410 }
2411 }
2412 return $valArray;
2413 }
2414
2415 /**
2416 * Writes the file from import array to temp dir and returns the filename of it.
2417 *
2418 * @param array $fI File information with three keys: "filename" = filename without path, "ID_absFile" = absolute filepath to the file (including the filename), "ID" = md5 hash of "ID_absFile
2419 * @return string|NULL Absolute filename of the temporary filename of the file. In ->alternativeFileName the original name is set.
2420 */
2421 public function import_addFileNameToBeCopied($fI) {
2422 if (is_array($this->dat['files'][$fI['ID']])) {
2423 $tmpFile = NULL;
2424 // check if there is the right file already in the local folder
2425 if ($this->filesPathForImport !== NULL) {
2426 if (is_file($this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5']) &&
2427 md5_file($this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5']) === $this->dat['files'][$fI['ID']]['content_md5']) {
2428 $tmpFile = $this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5'];
2429 }
2430 }
2431 if ($tmpFile === NULL) {
2432 $tmpFile = GeneralUtility::tempnam('import_temp_');
2433 GeneralUtility::writeFile($tmpFile, $this->dat['files'][$fI['ID']]['content']);
2434 }
2435 clearstatcache();
2436 if (@is_file($tmpFile)) {
2437 $this->unlinkFiles[] = $tmpFile;
2438 if (filesize($tmpFile) == $this->dat['files'][$fI['ID']]['filesize']) {
2439 $this->alternativeFileName[$tmpFile] = $fI['filename'];
2440 $this->alternativeFilePath[$tmpFile] = $this->dat['files'][$fI['ID']]['relFileRef'];
2441 return $tmpFile;
2442 } else {
2443 $this->error('Error: temporary file ' . $tmpFile . ' had a size (' . filesize($tmpFile) . ') different from the original (' . $this->dat['files'][$fI['ID']]['filesize'] . ')', 1);
2444 }
2445 } else {
2446 $this->error('Error: temporary file ' . $tmpFile . ' was not written as it should have been!', 1);
2447 }
2448 } else {
2449 $this->error('Error: No file found for ID ' . $fI['ID'], 1);
2450 }
2451 return NULL;
2452 }
2453
2454 /**
2455 * After all DB relations has been set in the end of the import (see setRelations()) then it is time to correct all relations inside of FlexForm fields.
2456 * The reason for doing this after is that the setting of relations may affect (quite often!) which data structure is used for the flexforms field!
2457 *
2458 * @return void
2459 * @see setRelations()
2460 */
2461 public function setFlexFormRelations() {
2462 $updateData = array();
2463 // import_newId contains a register of all records that was in the import memorys "records" key
2464 foreach ($this->import_newId as $nId => $dat) {
2465 $table = $dat['table'];
2466 $uid = $dat['uid'];
2467 // original UID - NOT the new one!
2468 // If the record has been written and received a new id, then proceed:
2469 if (!isset($this->import_mapId[$table][$uid])) {
2470 $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')', 1);
2471 continue;
2472 }
2473
2474 if (!is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {
2475 $this->error('Error: no record was found in data array!', 1);
2476 continue;
2477 }
2478 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
2479 // Traverse relation fields of each record
2480 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
2481 switch ((string)$config['type']) {
2482 case 'flex':
2483 // Get XML content and set as default value (string, non-processed):
2484 $updateData[$table][$thisNewUid][$field] = $this->dat['records'][$table . ':' . $uid]['data'][$field];
2485 // If there has been registered relations inside the flex form field, run processing on the content:
2486 if (!empty($config['flexFormRels']['db']) || !empty($config['flexFormRels']['file'])) {
2487 $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');
2488 // This will fetch the new row for the element (which should be updated with any references to data structures etc.)
2489 $conf = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
2490 if (is_array($origRecordRow) && is_array($conf) && $conf['type'] === 'flex') {
2491 // Get current data structure and value array:
2492 $dataStructArray = BackendUtility::getFlexFormDS($conf, $origRecordRow, $table, $field);
2493 $currentValueArray = GeneralUtility::xml2array($updateData[$table][$thisNewUid][$field]);
2494 // Do recursive processing of the XML data:
2495 $iteratorObj = GeneralUtility::makeInstance(\TYPO3\CMS\Core\DataHandling\DataHandler::class);
2496 $iteratorObj->callBackObj = $this;
2497 $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData(
2498 $currentValueArray['data'],
2499 array(),
2500 array(),
2501 $dataStructArray,
2502 array($table, $thisNewUid, $field, $config),
2503 'remapListedDBRecords_flexFormCallBack'
2504 );
2505 // The return value is set as an array which means it will be processed by tcemain for file and DB references!
2506 if (is_array($currentValueArray['data'])) {
2507 $updateData[$table][$thisNewUid][$field] = $currentValueArray;
2508 }
2509 }
2510 }
2511 break;
2512 }
2513 }
2514 }
2515 if (!empty($updateData)) {
2516 $tce = $this->getNewTCE();
2517 $tce->isImporting = TRUE;
2518 $this->callHook('before_setFlexFormRelations', array(
2519 'tce' => &$tce,
2520 'data' => &$updateData
2521 ));
2522 $tce->start($updateData, array());
2523 $tce->process_datamap();
2524 $this->callHook('after_setFlexFormRelations', array(
2525 'tce' => &$tce
2526 ));
2527 }
2528 }
2529
2530 /**
2531 * Callback function for traversing the FlexForm structure in relation to remapping database relations
2532 *
2533 * @param array $pParams Set of parameters in numeric array: table, uid, field
2534 * @param array $dsConf TCA config for field (from Data Structure of course)
2535 * @param string $dataValue Field value (from FlexForm XML)
2536 * @param string $dataValue_ext1 Not used
2537 * @param string $dataValue_ext2 Not used
2538 * @param string $path Path of where the data structure of the element is found
2539 * @return array Array where the "value" key carries the value.
2540 * @see setFlexFormRelations()
2541 */
2542 public function remapListedDBRecords_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path) {
2543 // Extract parameters:
2544 list($table, $uid, $field, $config) = $pParams;
2545 // In case the $path is used as index without a trailing slash we will remove that
2546 if (!is_array($config['flexFormRels']['db'][$path]) && is_array($config['flexFormRels']['db'][rtrim($path, '/')])) {
2547 $path = rtrim($path, '/');
2548 }
2549 if (is_array($config['flexFormRels']['db'][$path])) {
2550 $valArray = $this->setRelations_db($config['flexFormRels']['db'][$path], $dsConf);
2551 $dataValue = implode(',', $valArray);
2552 }
2553 if (is_array($config['flexFormRels']['file'][$path])) {
2554 $valArr = array();
2555 foreach ($config['flexFormRels']['file'][$path] as $fI) {
2556 $valArr[] = $this->import_addFileNameToBeCopied($fI);
2557 }
2558 $dataValue = implode(',', $valArr);
2559 }
2560 return array('value' => $dataValue);
2561 }
2562
2563 /**************************
2564 * Import / Soft References
2565 *************************/
2566
2567 /**
2568 * Processing of soft references
2569 *
2570 * @return void
2571 */
2572 public function processSoftReferences() {
2573 // Initialize:
2574 $inData = array();
2575 // Traverse records:
2576 if (is_array($this->dat['header']['records'])) {
2577 foreach ($this->dat['header']['records'] as $table => $recs) {
2578 foreach ($recs as $uid => $thisRec) {
2579 // If there are soft references defined, traverse those:
2580 if (isset($GLOBALS['TCA'][$table]) && is_array($thisRec['softrefs'])) {
2581 // First traversal is to collect softref configuration and split them up based on fields.
2582 // This could probably also have been done with the "records" key instead of the header.
2583 $fieldsIndex = array();
2584 foreach ($thisRec['softrefs'] as $softrefDef) {
2585 // If a substitution token is set:
2586 if ($softrefDef['field'] && is_array($softrefDef['subst']) && $softrefDef['subst']['tokenID']) {
2587 $fieldsIndex[$softrefDef['field']][$softrefDef['subst']['tokenID']] = $softrefDef;
2588 }
2589 }
2590 // The new id:
2591 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
2592 // Now, if there are any fields that require substitution to be done, lets go for that:
2593 foreach ($fieldsIndex as $field => $softRefCfgs) {
2594 if (is_array($GLOBALS['TCA'][$table]['columns'][$field])) {
2595 $conf = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
2596 if ($conf['type'] === 'flex') {
2597 // This will fetch the new row for the element (which should be updated with any references to data structures etc.)
2598 $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');
2599 if (is_array($origRecordRow)) {
2600 // Get current data structure and value array:
2601 $dataStructArray = BackendUtility::getFlexFormDS($conf, $origRecordRow, $table, $field);
2602 $currentValueArray = GeneralUtility::xml2array($origRecordRow[$field]);
2603 // Do recursive processing of the XML data:
2604 /** @var $iteratorObj \TYPO3\CMS\Core\DataHandling\DataHandler */
2605 $iteratorObj = GeneralUtility::makeInstance(\TYPO3\CMS\Core\DataHandling\DataHandler::class);
2606 $iteratorObj->callBackObj = $this;
2607 $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData($currentValueArray['data'], array(), array(), $dataStructArray, array($table, $uid, $field, $softRefCfgs), 'processSoftReferences_flexFormCallBack');
2608 // The return value is set as an array which means it will be processed by tcemain for file and DB references!
2609 if (is_array($currentValueArray['data'])) {
2610 $inData[$table][$thisNewUid][$field] = $currentValueArray;
2611 }
2612 }
2613 } else {
2614 // Get tokenizedContent string and proceed only if that is not blank:
2615 $tokenizedContent = $this->dat['records'][$table . ':' . $uid]['rels'][$field]['softrefs']['tokenizedContent'];
2616 if (strlen($tokenizedContent) && is_array($softRefCfgs)) {
2617 $inData[$table][$thisNewUid][$field] = $this->processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid);
2618 }
2619 }
2620 }
2621 }
2622 }
2623 }
2624 }
2625 }
2626 // Now write to database:
2627 $tce = $this->getNewTCE();
2628 $tce->isImporting = TRUE;
2629 $this->callHook('before_processSoftReferences', array(
2630 'tce' => $tce,
2631 'data' => &$inData
2632 ));
2633 $tce->enableLogging = TRUE;
2634 $tce->start($inData, array());
2635 $tce->process_datamap();
2636 $this->callHook('after_processSoftReferences', array(
2637 'tce' => $tce
2638 ));
2639 }
2640
2641 /**
2642 * Callback function for traversing the FlexForm structure in relation to remapping softreference relations
2643 *
2644 * @param array $pParams Set of parameters in numeric array: table, uid, field
2645 * @param array $dsConf TCA config for field (from Data Structure of course)
2646 * @param string $dataValue Field value (from FlexForm XML)
2647 * @param string $dataValue_ext1 Not used
2648 * @param string $dataValue_ext2 Not used
2649 * @param string $path Path of where the data structure where the element is found
2650 * @return array Array where the "value" key carries the value.
2651 * @see setFlexFormRelations()
2652 */
2653 public function processSoftReferences_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path) {
2654 // Extract parameters:
2655 list($table, $origUid, $field, $softRefCfgs) = $pParams;
2656 if (is_array($softRefCfgs)) {
2657 // First, find all soft reference configurations for this structure path (they are listed flat in the header):
2658 $thisSoftRefCfgList = array();
2659 foreach ($softRefCfgs as $sK => $sV) {
2660 if ($sV['structurePath'] === $path) {
2661 $thisSoftRefCfgList[$sK] = $sV;
2662 }
2663 }
2664 // If any was found, do processing:
2665 if (!empty($thisSoftRefCfgList)) {
2666 // Get tokenizedContent string and proceed only if that is not blank:
2667 $tokenizedContent = $this->dat['records'][$table . ':' . $origUid]['rels'][$field]['flexFormRels']['softrefs'][$path]['tokenizedContent'];
2668 if (strlen($tokenizedContent)) {
2669 $dataValue = $this->processSoftReferences_substTokens($tokenizedContent, $thisSoftRefCfgList, $table, $origUid);
2670 }
2671 }
2672 }
2673 // Return
2674 return array('value' => $dataValue);
2675 }
2676
2677 /**
2678 * Substition of softreference tokens
2679 *
2680 * @param string $tokenizedContent Content of field with soft reference tokens in.
2681 * @param array $softRefCfgs Soft reference configurations
2682 * @param string $table Table for which the processing occurs
2683 * @param string $uid UID of record from table
2684 * @return string The input content with tokens substituted according to entries in softRefCfgs
2685 */
2686 public function processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid) {
2687 // traverse each softref type for this field:
2688 foreach ($softRefCfgs as $cfg) {
2689 // Get token ID:
2690 $tokenID = $cfg['subst']['tokenID'];
2691 // Default is current token value:
2692 $insertValue = $cfg['subst']['tokenValue'];
2693 // Based on mode:
2694 switch ((string)$this->softrefCfg[$tokenID]['mode']) {
2695 case 'exclude':
2696 // Exclude is a simple passthrough of the value
2697 break;
2698 case 'editable':
2699 // Editable always picks up the value from this input array:
2700 $insertValue = $this->softrefInputValues[$tokenID];
2701 break;
2702 default:
2703 // Mapping IDs/creating files: Based on type, look up new value:
2704 switch ((string)$cfg['subst']['type']) {
2705 case 'file':
2706 // Create / Overwrite file:
2707 $insertValue = $this->processSoftReferences_saveFile($cfg['subst']['relFileName'], $cfg, $table, $uid);
2708 break;
2709 case 'db':
2710 default:
2711 // Trying to map database element if found in the mapID array:
2712 list($tempTable, $tempUid) = explode(':', $cfg['subst']['recordRef']);
2713 if (isset($this->import_mapId[$tempTable][$tempUid])) {
2714 $insertValue = BackendUtility::wsMapId($tempTable, $this->import_mapId[$tempTable][$tempUid]);
2715 // Look if reference is to a page and the original token value was NOT an integer - then we assume is was an alias and try to look up the new one!
2716 if ($tempTable === 'pages' && !\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($cfg['subst']['tokenValue'])) {
2717 $recWithUniqueValue = BackendUtility::getRecord($tempTable, $insertValue, 'alias');
2718 if ($recWithUniqueValue['alias']) {
2719 $insertValue = $recWithUniqueValue['alias'];
2720 }
2721 } elseif (strpos($cfg['subst']['tokenValue'], ':') !== FALSE) {
2722 list($tokenKey, $tokenId) = explode(':', $cfg['subst']['tokenValue']);
2723 $insertValue = $tokenKey . ':' . $insertValue;
2724 }
2725 }
2726 }
2727 }
2728 // Finally, swap the soft reference token in tokenized content with the insert value:
2729 $tokenizedContent = str_replace('{softref:' . $tokenID . '}', $insertValue, $tokenizedContent);
2730 }
2731 return $tokenizedContent;
2732 }
2733
2734 /**
2735 * Process a soft reference file
2736 *
2737 * @param string $relFileName Old Relative filename
2738 * @param array $cfg soft reference configuration array
2739 * @param string $table Table for which the processing occurs
2740 * @param string $uid UID of record from table
2741 * @return string New relative filename (value to insert instead of the softref token)
2742 */
2743 public function processSoftReferences_saveFile($relFileName, $cfg, $table, $uid) {
2744 if ($fileHeaderInfo = $this->dat['header']['files'][$cfg['file_ID']]) {
2745 // Initialize; Get directory prefix for file and find possible RTE filename
2746 $dirPrefix = PathUtility::dirname($relFileName) . '/';
2747 $rteOrigName = $this->getRTEoriginalFilename(PathUtility::basename($relFileName));
2748 // If filename looks like an RTE file, and the directory is in "uploads/", then process as a RTE file!
2749 if ($rteOrigName && GeneralUtility::isFirstPartOfStr($dirPrefix, 'uploads/')) {
2750 // RTE:
2751 // First, find unique RTE file name:
2752 if (@is_dir((PATH_site . $dirPrefix))) {
2753 // From the "original" RTE filename, produce a new "original" destination filename which is unused.
2754 // Even if updated, the image should be unique. Currently the problem with this is that it leaves a lot of unused RTE images...
2755 $fileProcObj = $this->getFileProcObj();
2756 $origDestName = $fileProcObj->getUniqueName($rteOrigName, PATH_site . $dirPrefix);
2757 // Create copy file name:
2758 $pI = pathinfo($relFileName);
2759 $copyDestName = PathUtility::dirname($origDestName) . '/RTEmagicC_' . substr(PathUtility::basename($origDestName), 10) . '.' . $pI['extension'];
2760 if (
2761 !@is_file($copyDestName) && !@is_file($origDestName)
2762 && $origDestName === GeneralUtility::getFileAbsFileName($origDestName)
2763 && $copyDestName === GeneralUtility::getFileAbsFileName($copyDestName)
2764 ) {
2765 if ($this->dat['header']['files'][$fileHeaderInfo['RTE_ORIG_ID']]) {
2766 if ($this->legacyImport) {
2767 $fileName = PathUtility::basename($copyDestName);
2768 $this->writeSysFileResourceForLegacyImport($fileName, $cfg['file_ID']);
2769 $relFileName = $this->filePathMap[$cfg['file_ID']] . '" data-htmlarea-file-uid="' . $fileName . '" data-htmlarea-file-table="sys_file';
2770 // Also save the original file
2771 $originalFileName = PathUtility::basename($origDestName);
2772 $this->writeSysFileResourceForLegacyImport($originalFileName, $fileHeaderInfo['RTE_ORIG_ID']);
2773 } else {
2774 // Write the copy and original RTE file to the respective filenames:
2775 $this->writeFileVerify($copyDestName, $cfg['file_ID'], TRUE);
2776 $this->writeFileVerify($origDestName, $fileHeaderInfo['RTE_ORIG_ID'], TRUE);
2777 // Return the relative path of the copy file name:
2778 return PathUtility::stripPathSitePrefix($copyDestName);
2779 }
2780 } else {
2781 $this->error('ERROR: Could not find original file ID');
2782 }
2783 } else {
2784 $this->error('ERROR: The destination filenames "' . $copyDestName . '" and "' . $origDestName . '" either existed or have non-valid names');
2785 }
2786 } else {
2787 $this->error('ERROR: "' . PATH_site . $dirPrefix . '" was not a directory, so could not process file "' . $relFileName . '"');
2788 }
2789 } elseif (GeneralUtility::isFirstPartOfStr($dirPrefix, $this->fileadminFolderName . '/')) {
2790 // File in fileadmin/ folder:
2791 // Create file (and possible resources)
2792 $newFileName = $this->processSoftReferences_saveFile_createRelFile($dirPrefix, PathUtility::basename($relFileName), $cfg['file_ID'], $table, $uid);
2793 if (strlen($newFileName)) {
2794 $relFileName = $newFileName;
2795 } else {
2796 $this->error('ERROR: No new file created for "' . $relFileName . '"');
2797 }
2798 } else {
2799 $this->error('ERROR: Sorry, cannot operate on non-RTE files which are outside the fileadmin folder.');
2800 }
2801 } else {
2802 $this->error('ERROR: Could not find file ID in header.');
2803 }
2804 // Return (new) filename relative to PATH_site:
2805 return $relFileName;
2806 }
2807
2808 /**
2809 * Create file in directory and return the new (unique) filename
2810 *
2811 * @param string $origDirPrefix Directory prefix, relative, with trailing slash
2812 * @param string $fileName Filename (without path)
2813 * @param string $fileID File ID from import memory
2814 * @param string $table Table for which the processing occurs
2815 * @param string $uid UID of record from table
2816 * @return string|NULL New relative filename, if any
2817 */
2818 public function processSoftReferences_saveFile_createRelFile($origDirPrefix, $fileName, $fileID, $table, $uid) {
2819 // If the fileID map contains an entry for this fileID then just return the relative filename of that entry;
2820 // we don't want to write another unique filename for this one!
2821 if (isset($this->fileIDMap[$fileID])) {
2822 return PathUtility::stripPathSitePrefix($this->fileIDMap[$fileID]);
2823 }
2824 if ($this->legacyImport) {
2825 // set dirPrefix to fileadmin because the right target folder is set and checked for permissions later
2826 $dirPrefix = $this->fileadminFolderName . '/';
2827 } else {
2828 // Verify FileMount access to dir-prefix. Returns the best alternative relative path if any
2829 $dirPrefix = $this->verifyFolderAccess($origDirPrefix);
2830 }
2831 if ($dirPrefix && (!$this->update || $origDirPrefix === $dirPrefix) && $this->checkOrCreateDir($dirPrefix)) {
2832 $fileHeaderInfo = $this->dat['header']['files'][$fileID];
2833 $updMode = $this->update && $this->import_mapId[$table][$uid] === $uid && $this->import_mode[$table . ':' . $uid] !== 'as_new';
2834 // Create new name for file:
2835 // Must have same ID in map array (just for security, is not really needed) and NOT be set "as_new".
2836
2837 // Write main file:
2838 if ($this->legacyImport) {
2839 $fileWritten = $this->writeSysFileResourceForLegacyImport($fileName, $fileID);
2840 if ($fileWritten) {
2841 $newName = 'file:' . $fileName;
2842 return $newName;
2843 // no support for HTML/CSS file resources attached ATM - see below
2844 }
2845 } else {
2846