71f62aaf773b370983c64f617bee5c7ca02e2a92
[Packages/TYPO3.CMS.git] / typo3 / sysext / impexp / Classes / ImportExport.php
1 <?php
2 namespace TYPO3\CMS\Impexp;
3
4 /*
5 * This file is part of the TYPO3 CMS project.
6 *
7 * It is free software; you can redistribute it and/or modify it under
8 * the terms of the GNU General Public License, either version 2
9 * of the License, or any later version.
10 *
11 * For the full copyright and license information, please read the
12 * LICENSE.txt file that was distributed with this source code.
13 *
14 * The TYPO3 project - inspiring people to share!
15 */
16
17 use TYPO3\CMS\Backend\Utility\BackendUtility;
18 use TYPO3\CMS\Core\DataHandling\DataHandler;
19 use TYPO3\CMS\Core\Exception;
20 use TYPO3\CMS\Core\Resource\ResourceFactory;
21 use TYPO3\CMS\Core\Utility\File\ExtendedFileUtility;
22 use TYPO3\CMS\Core\Utility\GeneralUtility;
23 use TYPO3\CMS\Core\Utility\PathUtility;
24
25 /**
26 * EXAMPLE for using the impexp-class for exporting stuff:
27 *
28 * Create and initialize:
29 * $this->export = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Impexp\ImportExport::class);
30 * $this->export->init();
31 * Set which tables relations we will allow:
32 * $this->export->relOnlyTables[]="tt_news"; // exclusively includes. See comment in the class
33 *
34 * Adding records:
35 * $this->export->export_addRecord("pages", $this->pageinfo);
36 * $this->export->export_addRecord("pages", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("pages", 38));
37 * $this->export->export_addRecord("pages", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("pages", 39));
38 * $this->export->export_addRecord("tt_content", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("tt_content", 12));
39 * $this->export->export_addRecord("tt_content", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("tt_content", 74));
40 * $this->export->export_addRecord("sys_template", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("sys_template", 20));
41 *
42 * Adding all the relations (recursively in 5 levels so relations has THEIR relations registered as well)
43 * for($a=0;$a<5;$a++) {
44 * $addR = $this->export->export_addDBRelations($a);
45 * if (!count($addR)) break;
46 * }
47 *
48 * Finally load all the files.
49 * $this->export->export_addFilesFromRelations(); // MUST be after the DBrelations are set so that file from ALL added records are included!
50 *
51 * Write export
52 * $out = $this->export->compileMemoryToFileContent();
53 */
54
55 /**
56 * T3D file Import/Export library (TYPO3 Record Document)
57 *
58 * @author Kasper Skårhøj <kasperYYYY@typo3.com>
59 */
60 class ImportExport {
61
62 /**
63 * If set, static relations (not exported) will be shown in overview as well
64 *
65 * @var bool
66 */
67 public $showStaticRelations = FALSE;
68
69 /**
70 * Name of the "fileadmin" folder where files for export/import should be located
71 *
72 * @var string
73 */
74 public $fileadminFolderName = '';
75
76 /**
77 * Whether "import" or "export" mode of object. Set through init() function
78 *
79 * @var string
80 */
81 public $mode = '';
82
83 /**
84 * Updates all records that has same UID instead of creating new!
85 *
86 * @var bool
87 */
88 public $update = FALSE;
89
90 /**
91 * Is set by importData() when an import has been done.
92 *
93 * @var bool
94 */
95 public $doesImport = FALSE;
96
97 /**
98 * If set to a page-record, then the preview display of the content will expect this page-record to be the target
99 * for the import and accordingly display validation information. This triggers the visual view of the
100 * import/export memory to validate if import is possible
101 *
102 * @var array
103 */
104 public $display_import_pid_record = array();
105
106 /**
107 * Used to register the forged UID values for imported records that we want
108 * to create with the same UIDs as in the import file. Admin-only feature.
109 *
110 * @var array
111 */
112 public $suggestedInsertUids = array();
113
114 /**
115 * Setting import modes during update state: as_new, exclude, force_uid
116 *
117 * @var array
118 */
119 public $import_mode = array();
120
121 /**
122 * If set, PID correct is ignored globally
123 *
124 * @var bool
125 */
126 public $global_ignore_pid = FALSE;
127
128 /**
129 * If set, all UID values are forced! (update or import)
130 *
131 * @var bool
132 */
133 public $force_all_UIDS = FALSE;
134
135 /**
136 * If set, a diff-view column is added to the overview.
137 *
138 * @var bool
139 */
140 public $showDiff = FALSE;
141
142 /**
143 * If set, and if the user is admin, allow the writing of PHP scripts to fileadmin/ area.
144 *
145 * @var bool
146 */
147 public $allowPHPScripts = FALSE;
148
149 /**
150 * Disable logging when importing
151 *
152 * @var bool
153 */
154 public $enableLogging = FALSE;
155
156 /**
157 * Array of values to substitute in editable softreferences.
158 *
159 * @var array
160 */
161 public $softrefInputValues = array();
162
163 /**
164 * Mapping between the fileID from import memory and the final filenames they are written to.
165 *
166 * @var array
167 */
168 public $fileIDMap = array();
169
170 /**
171 * 1MB max file size
172 *
173 * @var int
174 */
175 public $maxFileSize = 1000000;
176
177 /**
178 * 1MB max record size
179 *
180 * @var int
181 */
182 public $maxRecordSize = 1000000;
183
184 /**
185 * 10MB max export size
186 *
187 * @var int
188 */
189 public $maxExportSize = 10000000;
190
191 /**
192 * Add table names here which are THE ONLY ones which will be included
193 * into export if found as relations. '_ALL' will allow all tables.
194 *
195 * @var array
196 */
197 public $relOnlyTables = array();
198
199 /**
200 * Add tables names here which should not be exported with the file.
201 * (Where relations should be mapped to same UIDs in target system).
202 *
203 * @var array
204 */
205 public $relStaticTables = array();
206
207 /**
208 * Exclude map. Keys are table:uid pairs and if set, records are not added to the export.
209 *
210 * @var array
211 */
212 public $excludeMap = array();
213
214 /**
215 * Soft Reference Token ID modes.
216 *
217 * @var array
218 */
219 public $softrefCfg = array();
220
221 /**
222 * Listing extension dependencies.
223 *
224 * @var array
225 */
226 public $extensionDependencies = array();
227
228 /**
229 * Set by user: If set, compression in t3d files is disabled
230 *
231 * @var bool
232 */
233 public $dontCompress = FALSE;
234
235 /**
236 * If set, HTML file resources are included.
237 *
238 * @var bool
239 */
240 public $includeExtFileResources = FALSE;
241
242 /**
243 * Files with external media (HTML/css style references inside)
244 *
245 * @var string
246 */
247 public $extFileResourceExtensions = 'html,htm,css';
248
249 /**
250 * After records are written this array is filled with [table][original_uid] = [new_uid]
251 *
252 * @var array
253 */
254 public $import_mapId = array();
255
256 /**
257 * Keys are [tablename]:[new NEWxxx ids (or when updating it is uids)]
258 * while values are arrays with table/uid of the original record it is based on.
259 * With the array keys the new ids can be looked up inside tcemain
260 *
261 * @var array
262 */
263 public $import_newId = array();
264
265 /**
266 * Page id map for page tree (import)
267 *
268 * @var array
269 */
270 public $import_newId_pids = array();
271
272 /**
273 * Internal data accumulation for writing records during import
274 *
275 * @var array
276 */
277 public $import_data = array();
278
279 /**
280 * Error log.
281 *
282 * @var array
283 */
284 public $errorLog = array();
285
286 /**
287 * Cache for record paths
288 *
289 * @var array
290 */
291 public $cache_getRecordPath = array();
292
293 /**
294 * Cache of checkPID values.
295 *
296 * @var array
297 */
298 public $checkPID_cache = array();
299
300 /**
301 * Set internally if the gzcompress function exists
302 * Used by ImportExportController
303 *
304 * @var bool
305 */
306 public $compress = FALSE;
307
308 /**
309 * Internal import/export memory
310 *
311 * @var array
312 */
313 public $dat = array();
314
315 /**
316 * File processing object
317 *
318 * @var \TYPO3\CMS\Core\Utility\File\ExtendedFileUtility
319 */
320 protected $fileProcObj = NULL;
321
322 /**
323 * Keys are [recordname], values are an array of fields to be included
324 * in the export
325 *
326 * @var array
327 */
328 protected $recordTypesIncludeFields = array();
329
330 /**
331 * Default array of fields to be included in the export
332 *
333 * @var array
334 */
335 protected $defaultRecordIncludeFields = array('uid', 'pid');
336
337 /**
338 * Array of current registered storage objects
339 *
340 * @var array
341 */
342 protected $storageObjects = array();
343
344 /**
345 * Is set, if the import file has a TYPO3 version below 6.0
346 *
347 * @var bool
348 */
349 protected $legacyImport = FALSE;
350
351 /**
352 * @var \TYPO3\CMS\Core\Resource\Folder
353 */
354 protected $legacyImportFolder = NULL;
355
356 /**
357 * Related to the default storage root
358 *
359 * @var string
360 */
361 protected $legacyImportTargetPath = '_imported/';
362
363 /**
364 * Table fields to migrate
365 *
366 * @var array
367 */
368 protected $legacyImportMigrationTables = array(
369 'tt_content' => array(
370 'image' => array(
371 'titleTexts' => 'titleText',
372 'description' => 'imagecaption',
373 'links' => 'image_link',
374 'alternativeTexts' => 'altText'
375 ),
376 'media' => array(
377 'description' => 'imagecaption',
378 )
379 ),
380 'pages' => array(
381 'media' => array()
382 ),
383 'pages_language_overlay' => array(
384 'media' => array()
385 )
386 );
387
388
389 /**
390 * Records to be migrated after all
391 * Multidimensional array [table][uid][field] = array([related sys_file_reference uids])
392 *
393 * @var array
394 */
395 protected $legacyImportMigrationRecords = array();
396
397 /**
398 * @var bool
399 */
400 protected $saveFilesOutsideExportFile = FALSE;
401
402 /**
403 * @var NULL|string
404 */
405 protected $temporaryFilesPathForExport = NULL;
406
407 /**
408 * @var NULL|string
409 */
410 protected $filesPathForImport = NULL;
411
412 /**
413 * @var array
414 */
415 protected $unlinkFiles = array();
416
417 /**
418 * @var array
419 */
420 protected $alternativeFileName = array();
421
422 /**
423 * @var array
424 */
425 protected $alternativeFilePath = array();
426
427 /**
428 * @var array
429 */
430 protected $filePathMap = array();
431
432 /**
433 * @var array
434 */
435 protected $remainHeader = array();
436
437 /**************************
438 * Initialize
439 *************************/
440
441 /**
442 * Init the object, both import and export
443 *
444 * @param bool $dontCompress If set, compression of t3d files is disabled
445 * @param string $mode Mode of usage, either "import" or "export
446 * @return void
447 */
448 public function init($dontCompress = FALSE, $mode = '') {
449 $this->compress = function_exists('gzcompress');
450 $this->dontCompress = $dontCompress;
451 $this->mode = $mode;
452 $this->fileadminFolderName = !empty($GLOBALS['TYPO3_CONF_VARS']['BE']['fileadminDir']) ? rtrim($GLOBALS['TYPO3_CONF_VARS']['BE']['fileadminDir'], '/') : 'fileadmin';
453 }
454
455 /**************************
456 * Export / Init + Meta Data
457 *************************/
458
459 /**
460 * Set header basics
461 *
462 * @return void
463 */
464 public function setHeaderBasics() {
465 // Initializing:
466 if (is_array($this->softrefCfg)) {
467 foreach ($this->softrefCfg as $key => $value) {
468 if (!strlen($value['mode'])) {
469 unset($this->softrefCfg[$key]);
470 }
471 }
472 }
473 // Setting in header memory:
474 // Version of file format
475 $this->dat['header']['XMLversion'] = '1.0';
476 // Initialize meta data array (to put it in top of file)
477 $this->dat['header']['meta'] = array();
478 // Add list of tables to consider static
479 $this->dat['header']['relStaticTables'] = $this->relStaticTables;
480 // The list of excluded records
481 $this->dat['header']['excludeMap'] = $this->excludeMap;
482 // Soft Reference mode for elements
483 $this->dat['header']['softrefCfg'] = $this->softrefCfg;
484 // List of extensions the import depends on.
485 $this->dat['header']['extensionDependencies'] = $this->extensionDependencies;
486 }
487
488 /**
489 * Set charset
490 *
491 * @param string $charset Charset for the content in the export. During import the character set will be converted if the target system uses another charset.
492 * @return void
493 */
494 public function setCharset($charset) {
495 $this->dat['header']['charset'] = $charset;
496 }
497
498 /**
499 * Sets meta data
500 *
501 * @param string $title Title of the export
502 * @param string $description Description of the export
503 * @param string $notes Notes about the contents
504 * @param string $packager_username Backend Username of the packager (the guy making the export)
505 * @param string $packager_name Real name of the packager
506 * @param string $packager_email Email of the packager
507 * @return void
508 */
509 public function setMetaData($title, $description, $notes, $packager_username, $packager_name, $packager_email) {
510 $this->dat['header']['meta'] = array(
511 'title' => $title,
512 'description' => $description,
513 'notes' => $notes,
514 'packager_username' => $packager_username,
515 'packager_name' => $packager_name,
516 'packager_email' => $packager_email,
517 'TYPO3_version' => TYPO3_version,
518 'created' => strftime('%A %e. %B %Y', $GLOBALS['EXEC_TIME'])
519 );
520 }
521
522 /**
523 * Option to enable having the files not included in the export file.
524 * The files are saved to a temporary folder instead.
525 *
526 * @param bool $saveFilesOutsideExportFile
527 * @see getTemporaryFilesPathForExport()
528 */
529 public function setSaveFilesOutsideExportFile($saveFilesOutsideExportFile) {
530 $this->saveFilesOutsideExportFile = $saveFilesOutsideExportFile;
531 }
532
533 /**
534 * Sets a thumbnail image to the exported file
535 *
536 * @param string $imgFilepath Filename reference, gif, jpg, png. Absolute path.
537 * @return void
538 */
539 public function addThumbnail($imgFilepath) {
540 if (@is_file($imgFilepath)) {
541 $imgInfo = @getimagesize($imgFilepath);
542 if (is_array($imgInfo)) {
543 $fileContent = GeneralUtility::getUrl($imgFilepath);
544 $this->dat['header']['thumbnail'] = array(
545 'imgInfo' => $imgInfo,
546 'content' => $fileContent,
547 'filesize' => strlen($fileContent),
548 'filemtime' => filemtime($imgFilepath),
549 'filename' => PathUtility::basename($imgFilepath)
550 );
551 }
552 }
553 }
554
555 /**************************
556 * Export / Init Page tree
557 *************************/
558
559 /**
560 * Sets the page-tree array in the export header and returns the array in a flattened version
561 *
562 * @param array $idH Hierarchy of ids, the page tree: array([uid] => array("uid" => [uid], "subrow" => array(.....)), [uid] => ....)
563 * @return array The hierarchical page tree converted to a one-dimensional list of pages
564 */
565 public function setPageTree($idH) {
566 $this->dat['header']['pagetree'] = $this->unsetExcludedSections($idH);
567 return $this->flatInversePageTree($this->dat['header']['pagetree']);
568 }
569
570 /**
571 * Removes entries in the page tree which are found in ->excludeMap[]
572 *
573 * @param array $idH Page uid hierarchy
574 * @return array Modified input array
575 * @access private
576 * @see setPageTree()
577 */
578 public function unsetExcludedSections($idH) {
579 if (is_array($idH)) {
580 foreach ($idH as $k => $v) {
581 if ($this->excludeMap['pages:' . $idH[$k]['uid']]) {
582 unset($idH[$k]);
583 } elseif (is_array($idH[$k]['subrow'])) {
584 $idH[$k]['subrow'] = $this->unsetExcludedSections($idH[$k]['subrow']);
585 }
586 }
587 }
588 return $idH;
589 }
590
591 /**
592 * Recursively flattening the idH array (for setPageTree() function)
593 *
594 * @param array $idH Page uid hierarchy
595 * @param array $a Accumulation array of pages (internal, don't set from outside)
596 * @return array Array with uid-uid pairs for all pages in the page tree.
597 * @see flatInversePageTree_pid()
598 */
599 public function flatInversePageTree($idH, $a = array()) {
600 if (is_array($idH)) {
601 $idH = array_reverse($idH);
602 foreach ($idH as $k => $v) {
603 $a[$v['uid']] = $v['uid'];
604 if (is_array($v['subrow'])) {
605 $a = $this->flatInversePageTree($v['subrow'], $a);
606 }
607 }
608 }
609 return $a;
610 }
611
612 /**
613 * Recursively flattening the idH array (for setPageTree() function), setting PIDs as values
614 *
615 * @param array $idH Page uid hierarchy
616 * @param array $a Accumulation array of pages (internal, don't set from outside)
617 * @param int $pid PID value (internal)
618 * @return array Array with uid-pid pairs for all pages in the page tree.
619 * @see flatInversePageTree()
620 */
621 public function flatInversePageTree_pid($idH, $a = array(), $pid = -1) {
622 if (is_array($idH)) {
623 $idH = array_reverse($idH);
624 foreach ($idH as $v) {
625 $a[$v['uid']] = $pid;
626 if (is_array($v['subrow'])) {
627 $a = $this->flatInversePageTree_pid($v['subrow'], $a, $v['uid']);
628 }
629 }
630 }
631 return $a;
632 }
633
634 /**************************
635 * Export
636 *************************/
637
638 /**
639 * Sets the fields of record types to be included in the export
640 *
641 * @param array $recordTypesIncludeFields Keys are [recordname], values are an array of fields to be included in the export
642 * @throws \TYPO3\CMS\Core\Exception if an array value is not type of array
643 * @return void
644 */
645 public function setRecordTypesIncludeFields(array $recordTypesIncludeFields) {
646 foreach ($recordTypesIncludeFields as $table => $fields) {
647 if (!is_array($fields)) {
648 throw new \TYPO3\CMS\Core\Exception('The include fields for record type ' . htmlspecialchars($table) . ' are not defined by an array.', 1391440658);
649 }
650 $this->setRecordTypeIncludeFields($table, $fields);
651 }
652 }
653
654 /**
655 * Sets the fields of a record type to be included in the export
656 *
657 * @param string $table The record type
658 * @param array $fields The fields to be included
659 * @return void
660 */
661 public function setRecordTypeIncludeFields($table, array $fields) {
662 $this->recordTypesIncludeFields[$table] = $fields;
663 }
664
665 /**
666 * Adds the record $row from $table.
667 * No checking for relations done here. Pure data.
668 *
669 * @param string $table Table name
670 * @param array $row Record row.
671 * @param int $relationLevel (Internal) if the record is added as a relation, this is set to the "level" it was on.
672 * @return void
673 */
674 public function export_addRecord($table, $row, $relationLevel = 0) {
675 BackendUtility::workspaceOL($table, $row);
676 if ((string)$table !== '' && is_array($row) && $row['uid'] > 0 && !$this->excludeMap[($table . ':' . $row['uid'])]) {
677 if ($this->checkPID($table === 'pages' ? $row['uid'] : $row['pid'])) {
678 if (!isset($this->dat['records'][($table . ':' . $row['uid'])])) {
679 // Prepare header info:
680 $row = $this->filterRecordFields($table, $row);
681 $headerInfo = array();
682 $headerInfo['uid'] = $row['uid'];
683 $headerInfo['pid'] = $row['pid'];
684 $headerInfo['title'] = GeneralUtility::fixed_lgd_cs(BackendUtility::getRecordTitle($table, $row), 40);
685 $headerInfo['size'] = strlen(serialize($row));
686 if ($relationLevel) {
687 $headerInfo['relationLevel'] = $relationLevel;
688 }
689 // If record content is not too large in size, set the header content and add the rest:
690 if ($headerInfo['size'] < $this->maxRecordSize) {
691 // Set the header summary:
692 $this->dat['header']['records'][$table][$row['uid']] = $headerInfo;
693 // Create entry in the PID lookup:
694 $this->dat['header']['pid_lookup'][$row['pid']][$table][$row['uid']] = 1;
695 // Initialize reference index object:
696 $refIndexObj = GeneralUtility::makeInstance(\TYPO3\CMS\Core\Database\ReferenceIndex::class);
697 // Yes to workspace overlays for exporting....
698 $refIndexObj->WSOL = TRUE;
699 $relations = $refIndexObj->getRelations($table, $row);
700 $relations = $this->fixFileIDsInRelations($relations);
701 $relations = $this->removeSoftrefsHavingTheSameDatabaseRelation($relations);
702 // Data:
703 $this->dat['records'][$table . ':' . $row['uid']] = array();
704 $this->dat['records'][$table . ':' . $row['uid']]['data'] = $row;
705 $this->dat['records'][$table . ':' . $row['uid']]['rels'] = $relations;
706 $this->errorLog = array_merge($this->errorLog, $refIndexObj->errorLog);
707 // Merge error logs.
708 // Add information about the relations in the record in the header:
709 $this->dat['header']['records'][$table][$row['uid']]['rels'] = $this->flatDBrels($this->dat['records'][$table . ':' . $row['uid']]['rels']);
710 // Add information about the softrefs to header:
711 $this->dat['header']['records'][$table][$row['uid']]['softrefs'] = $this->flatSoftRefs($this->dat['records'][$table . ':' . $row['uid']]['rels']);
712 } else {
713 $this->error('Record ' . $table . ':' . $row['uid'] . ' was larger than maxRecordSize (' . GeneralUtility::formatSize($this->maxRecordSize) . ')');
714 }
715 } else {
716 $this->error('Record ' . $table . ':' . $row['uid'] . ' already added.');
717 }
718 } else {
719 $this->error('Record ' . $table . ':' . $row['uid'] . ' was outside your DB mounts!');
720 }
721 }
722 }
723
724 /**
725 * This changes the file reference ID from a hash based on the absolute file path
726 * (coming from ReferenceIndex) to a hash based on the relative file path.
727 *
728 * @param array $relations
729 * @return array
730 */
731 protected function fixFileIDsInRelations(array $relations) {
732 foreach ($relations as $field => $relation) {
733 if (isset($relation['type']) && $relation['type'] === 'file') {
734 foreach ($relation['newValueFiles'] as $key => $fileRelationData) {
735 $absoluteFilePath = $fileRelationData['ID_absFile'];
736 if (GeneralUtility::isFirstPartOfStr($absoluteFilePath, PATH_site)) {
737 $relatedFilePath = PathUtility::stripPathSitePrefix($absoluteFilePath);
738 $relations[$field]['newValueFiles'][$key]['ID'] = md5($relatedFilePath);
739 }
740 }
741 }
742 if ($relation['type'] === 'flex') {
743 if (is_array($relation['flexFormRels']['file'])) {
744 foreach ($relation['flexFormRels']['file'] as $key => $subList) {
745 foreach ($subList as $subKey => $fileRelationData) {
746 $absoluteFilePath = $fileRelationData['ID_absFile'];
747 if (GeneralUtility::isFirstPartOfStr($absoluteFilePath, PATH_site)) {
748 $relatedFilePath = PathUtility::stripPathSitePrefix($absoluteFilePath);
749 $relations[$field]['flexFormRels']['file'][$key][$subKey]['ID'] = md5($relatedFilePath);
750 }
751 }
752 }
753 }
754 }
755 }
756 return $relations;
757 }
758
759 /**
760 * Relations could contain db relations to sys_file records. Some configuration combinations of TCA and
761 * SoftReferenceIndex create also softref relation entries for the identical file. This results
762 * in double included files, one in array "files" and one in array "file_fal".
763 * This function checks the relations for this double inclusions and removes the redundant softref relation.
764 *
765 * @param array $relations
766 * @return array
767 */
768 protected function removeSoftrefsHavingTheSameDatabaseRelation($relations) {
769 $fixedRelations = array();
770 foreach ($relations as $field => $relation) {
771 $newRelation = $relation;
772 if (isset($newRelation['type']) && $newRelation['type'] === 'db') {
773 foreach ($newRelation['itemArray'] as $key => $dbRelationData) {
774 if ($dbRelationData['table'] === 'sys_file') {
775 if (isset($newRelation['softrefs']['keys']['typolink'])) {
776 foreach ($newRelation['softrefs']['keys']['typolink'] as $softrefKey => $softRefData) {
777 if ($softRefData['subst']['type'] === 'file') {
778 $file = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->retrieveFileOrFolderObject($softRefData['subst']['relFileName']);
779 if ($file instanceof \TYPO3\CMS\Core\Resource\File) {
780 if ($file->getUid() == $dbRelationData['id']) {
781 unset($newRelation['softrefs']['keys']['typolink'][$softrefKey]);
782 }
783 }
784 }
785 }
786 if (empty($newRelation['softrefs']['keys']['typolink'])) {
787 unset($newRelation['softrefs']);
788 }
789 }
790 }
791 }
792 }
793 $fixedRelations[$field] = $newRelation;
794 }
795 return $fixedRelations;
796 }
797
798
799 /**
800 * This analyses the existing added records, finds all database relations to records and adds these records to the export file.
801 * This function can be called repeatedly until it returns an empty array.
802 * In principle it should not allow to infinite recursivity, but you better set a limit...
803 * Call this BEFORE the ext_addFilesFromRelations (so files from added relations are also included of course)
804 *
805 * @param int $relationLevel Recursion level
806 * @return array overview of relations found and added: Keys [table]:[uid], values array with table and id
807 * @see export_addFilesFromRelations()
808 */
809 public function export_addDBRelations($relationLevel = 0) {
810 // Traverse all "rels" registered for "records"
811 if (!is_array($this->dat['records'])) {
812 $this->error('There were no records available.');
813 return array();
814 }
815 $addR = array();
816 foreach ($this->dat['records'] as $k => $value) {
817 if (!is_array($this->dat['records'][$k])) {
818 continue;
819 }
820 foreach ($this->dat['records'][$k]['rels'] as $fieldname => $vR) {
821 // For all DB types of relations:
822 if ($vR['type'] == 'db') {
823 foreach ($vR['itemArray'] as $fI) {
824 $this->export_addDBRelations_registerRelation($fI, $addR);
825 }
826 }
827 // For all flex/db types of relations:
828 if ($vR['type'] == 'flex') {
829 // DB relations in flex form fields:
830 if (is_array($vR['flexFormRels']['db'])) {
831 foreach ($vR['flexFormRels']['db'] as $subList) {
832 foreach ($subList as $fI) {
833 $this->export_addDBRelations_registerRelation($fI, $addR);
834 }
835 }
836 }
837 // DB oriented soft references in flex form fields:
838 if (is_array($vR['flexFormRels']['softrefs'])) {
839 foreach ($vR['flexFormRels']['softrefs'] as $subList) {
840 foreach ($subList['keys'] as $spKey => $elements) {
841 foreach ($elements as $el) {
842 if ($el['subst']['type'] === 'db' && $this->includeSoftref($el['subst']['tokenID'])) {
843 list($tempTable, $tempUid) = explode(':', $el['subst']['recordRef']);
844 $fI = array(
845 'table' => $tempTable,
846 'id' => $tempUid
847 );
848 $this->export_addDBRelations_registerRelation($fI, $addR, $el['subst']['tokenID']);
849 }
850 }
851 }
852 }
853 }
854 }
855 // In any case, if there are soft refs:
856 if (is_array($vR['softrefs']['keys'])) {
857 foreach ($vR['softrefs']['keys'] as $spKey => $elements) {
858 foreach ($elements as $el) {
859 if ($el['subst']['type'] === 'db' && $this->includeSoftref($el['subst']['tokenID'])) {
860 list($tempTable, $tempUid) = explode(':', $el['subst']['recordRef']);
861 $fI = array(
862 'table' => $tempTable,
863 'id' => $tempUid
864 );
865 $this->export_addDBRelations_registerRelation($fI, $addR, $el['subst']['tokenID']);
866 }
867 }
868 }
869 }
870 }
871 }
872
873 // Now, if there were new records to add, do so:
874 if (!empty($addR)) {
875 foreach ($addR as $fI) {
876 // Get and set record:
877 $row = BackendUtility::getRecord($fI['table'], $fI['id']);
878 if (is_array($row)) {
879 $this->export_addRecord($fI['table'], $row, $relationLevel + 1);
880 }
881 // Set status message
882 // Relation pointers always larger than zero except certain "select" types with
883 // negative values pointing to uids - but that is not supported here.
884 if ($fI['id'] > 0) {
885 $rId = $fI['table'] . ':' . $fI['id'];
886 if (!isset($this->dat['records'][$rId])) {
887 $this->dat['records'][$rId] = 'NOT_FOUND';
888 $this->error('Relation record ' . $rId . ' was not found!');
889 }
890 }
891 }
892 }
893 // Return overview of relations found and added
894 return $addR;
895 }
896
897 /**
898 * Helper function for export_addDBRelations()
899 *
900 * @param array $fI Array with table/id keys to add
901 * @param array $addR Add array, passed by reference to be modified
902 * @param string $tokenID Softref Token ID, if applicable.
903 * @return void
904 * @see export_addDBRelations()
905 */
906 public function export_addDBRelations_registerRelation($fI, &$addR, $tokenID = '') {
907 $rId = $fI['table'] . ':' . $fI['id'];
908 if (
909 isset($GLOBALS['TCA'][$fI['table']]) && !$this->isTableStatic($fI['table']) && !$this->isExcluded($fI['table'], $fI['id'])
910 && (!$tokenID || $this->includeSoftref($tokenID)) && $this->inclRelation($fI['table'])
911 ) {
912 if (!isset($this->dat['records'][$rId])) {
913 // Set this record to be included since it is not already.
914 $addR[$rId] = $fI;
915 }
916 }
917 }
918
919 /**
920 * This adds all files in relations.
921 * Call this method AFTER adding all records including relations.
922 *
923 * @return void
924 * @see export_addDBRelations()
925 */
926 public function export_addFilesFromRelations() {
927 // Traverse all "rels" registered for "records"
928 if (!is_array($this->dat['records'])) {
929 $this->error('There were no records available.');
930 return;
931 }
932 foreach ($this->dat['records'] as $k => $value) {
933 if (!isset($this->dat['records'][$k]['rels']) || !is_array($this->dat['records'][$k]['rels'])) {
934 continue;
935 }
936 foreach ($this->dat['records'][$k]['rels'] as $fieldname => $vR) {
937 // For all file type relations:
938 if ($vR['type'] == 'file') {
939 foreach ($vR['newValueFiles'] as $key => $fI) {
940 $this->export_addFile($fI, $k, $fieldname);
941 // Remove the absolute reference to the file so it doesn't expose absolute paths from source server:
942 unset($this->dat['records'][$k]['rels'][$fieldname]['newValueFiles'][$key]['ID_absFile']);
943 }
944 }
945 // For all flex type relations:
946 if ($vR['type'] == 'flex') {
947 if (is_array($vR['flexFormRels']['file'])) {
948 foreach ($vR['flexFormRels']['file'] as $key => $subList) {
949 foreach ($subList as $subKey => $fI) {
950 $this->export_addFile($fI, $k, $fieldname);
951 // Remove the absolute reference to the file so it doesn't expose absolute paths from source server:
952 unset($this->dat['records'][$k]['rels'][$fieldname]['flexFormRels']['file'][$key][$subKey]['ID_absFile']);
953 }
954 }
955 }
956 // DB oriented soft references in flex form fields:
957 if (is_array($vR['flexFormRels']['softrefs'])) {
958 foreach ($vR['flexFormRels']['softrefs'] as $key => $subList) {
959 foreach ($subList['keys'] as $spKey => $elements) {
960 foreach ($elements as $subKey => $el) {
961 if ($el['subst']['type'] === 'file' && $this->includeSoftref($el['subst']['tokenID'])) {
962 // Create abs path and ID for file:
963 $ID_absFile = GeneralUtility::getFileAbsFileName(PATH_site . $el['subst']['relFileName']);
964 $ID = md5($el['subst']['relFileName']);
965 if ($ID_absFile) {
966 if (!$this->dat['files'][$ID]) {
967 $fI = array(
968 'filename' => PathUtility::basename($ID_absFile),
969 'ID_absFile' => $ID_absFile,
970 'ID' => $ID,
971 'relFileName' => $el['subst']['relFileName']
972 );
973 $this->export_addFile($fI, '_SOFTREF_');
974 }
975 $this->dat['records'][$k]['rels'][$fieldname]['flexFormRels']['softrefs'][$key]['keys'][$spKey][$subKey]['file_ID'] = $ID;
976 }
977 }
978 }
979 }
980 }
981 }
982 }
983 // In any case, if there are soft refs:
984 if (is_array($vR['softrefs']['keys'])) {
985 foreach ($vR['softrefs']['keys'] as $spKey => $elements) {
986 foreach ($elements as $subKey => $el) {
987 if ($el['subst']['type'] === 'file' && $this->includeSoftref($el['subst']['tokenID'])) {
988 // Create abs path and ID for file:
989 $ID_absFile = GeneralUtility::getFileAbsFileName(PATH_site . $el['subst']['relFileName']);
990 $ID = md5($el['subst']['relFileName']);
991 if ($ID_absFile) {
992 if (!$this->dat['files'][$ID]) {
993 $fI = array(
994 'filename' => PathUtility::basename($ID_absFile),
995 'ID_absFile' => $ID_absFile,
996 'ID' => $ID,
997 'relFileName' => $el['subst']['relFileName']
998 );
999 $this->export_addFile($fI, '_SOFTREF_');
1000 }
1001 $this->dat['records'][$k]['rels'][$fieldname]['softrefs']['keys'][$spKey][$subKey]['file_ID'] = $ID;
1002 }
1003 }
1004 }
1005 }
1006 }
1007 }
1008 }
1009 }
1010
1011 /**
1012 * This adds all files from sys_file records
1013 *
1014 * @return void
1015 */
1016 public function export_addFilesFromSysFilesRecords() {
1017 if (!isset($this->dat['header']['records']['sys_file']) || !is_array($this->dat['header']['records']['sys_file'])) {
1018 return;
1019 }
1020 foreach ($this->dat['header']['records']['sys_file'] as $sysFileUid => $_) {
1021 $recordData = $this->dat['records']['sys_file:' . $sysFileUid]['data'];
1022 $file = ResourceFactory::getInstance()->createFileObject($recordData);
1023 $this->export_addSysFile($file);
1024 }
1025 }
1026
1027 /**
1028 * Adds a files content from a sys file record to the export memory
1029 *
1030 * @param \TYPO3\CMS\Core\Resource\File $file
1031 * @return void
1032 */
1033 public function export_addSysFile(\TYPO3\CMS\Core\Resource\File $file) {
1034 if ($file->getProperty('size') >= $this->maxFileSize) {
1035 $this->error('File ' . $file->getPublicUrl() . ' was larger (' . GeneralUtility::formatSize($file->getProperty('size')) . ') than the maxFileSize (' . GeneralUtility::formatSize($this->maxFileSize) . ')! Skipping.');
1036 return;
1037 }
1038 $fileContent = '';
1039 try {
1040 if (!$this->saveFilesOutsideExportFile) {
1041 $fileContent = $file->getContents();
1042 } else {
1043 $file->checkActionPermission('read');
1044 }
1045 } catch (\Exception $e) {
1046 $this->error('Error when trying to add file ' . $file->getCombinedIdentifier() . ': ' . $e->getMessage());
1047 return;
1048 }
1049 $fileUid = $file->getUid();
1050 $fileInfo = $file->getStorage()->getFileInfo($file);
1051 // we sadly have to cast it to string here, because the size property is also returning a string
1052 $fileSize = (string)$fileInfo['size'];
1053 if ($fileSize !== $file->getProperty('size')) {
1054 $this->error('File size of ' . $file->getCombinedIdentifier() . ' is not up-to-date in index! File added with current size.');
1055 $this->dat['records']['sys_file:' . $fileUid]['data']['size'] = $fileSize;
1056 }
1057 $fileSha1 = $file->getStorage()->hashFile($file, 'sha1');
1058 if ($fileSha1 !== $file->getProperty('sha1')) {
1059 $this->error('File sha1 hash of ' . $file->getCombinedIdentifier() . ' is not up-to-date in index! File added on current sha1.');
1060 $this->dat['records']['sys_file:' . $fileUid]['data']['sha1'] = $fileSha1;
1061 }
1062
1063 $fileRec = array();
1064 $fileRec['filesize'] = $fileSize;
1065 $fileRec['filename'] = $file->getProperty('name');
1066 $fileRec['filemtime'] = $file->getProperty('modification_date');
1067
1068 // build unique id based on the storage and the file identifier
1069 $fileId = md5($file->getStorage()->getUid() . ':' . $file->getProperty('identifier_hash'));
1070
1071 // Setting this data in the header
1072 $this->dat['header']['files_fal'][$fileId] = $fileRec;
1073
1074 if (!$this->saveFilesOutsideExportFile) {
1075 // ... and finally add the heavy stuff:
1076 $fileRec['content'] = $fileContent;
1077 } else {
1078 GeneralUtility::upload_copy_move($file->getForLocalProcessing(FALSE), $this->getTemporaryFilesPathForExport() . $file->getProperty('sha1'));
1079 }
1080 $fileRec['content_sha1'] = $fileSha1;
1081
1082 $this->dat['files_fal'][$fileId] = $fileRec;
1083 }
1084
1085
1086 /**
1087 * Adds a files content to the export memory
1088 *
1089 * @param array $fI File information with three keys: "filename" = filename without path, "ID_absFile" = absolute filepath to the file (including the filename), "ID" = md5 hash of "ID_absFile". "relFileName" is optional for files attached to records, but mandatory for soft referenced files (since the relFileName determines where such a file should be stored!)
1090 * @param string $recordRef If the file is related to a record, this is the id on the form [table]:[id]. Information purposes only.
1091 * @param string $fieldname If the file is related to a record, this is the field name it was related to. Information purposes only.
1092 * @return void
1093 */
1094 public function export_addFile($fI, $recordRef = '', $fieldname = '') {
1095 if (!@is_file($fI['ID_absFile'])) {
1096 $this->error($fI['ID_absFile'] . ' was not a file! Skipping.');
1097 return;
1098 }
1099 if (filesize($fI['ID_absFile']) >= $this->maxFileSize) {
1100 $this->error($fI['ID_absFile'] . ' was larger (' . GeneralUtility::formatSize(filesize($fI['ID_absFile'])) . ') than the maxFileSize (' . GeneralUtility::formatSize($this->maxFileSize) . ')! Skipping.');
1101 return;
1102 }
1103 $fileInfo = stat($fI['ID_absFile']);
1104 $fileRec = array();
1105 $fileRec['filesize'] = $fileInfo['size'];
1106 $fileRec['filename'] = PathUtility::basename($fI['ID_absFile']);
1107 $fileRec['filemtime'] = $fileInfo['mtime'];
1108 //for internal type file_reference
1109 $fileRec['relFileRef'] = PathUtility::stripPathSitePrefix($fI['ID_absFile']);
1110 if ($recordRef) {
1111 $fileRec['record_ref'] = $recordRef . '/' . $fieldname;
1112 }
1113 if ($fI['relFileName']) {
1114 $fileRec['relFileName'] = $fI['relFileName'];
1115 }
1116 // Setting this data in the header
1117 $this->dat['header']['files'][$fI['ID']] = $fileRec;
1118 // ... and for the recordlisting, why not let us know WHICH relations there was...
1119 if ($recordRef && $recordRef !== '_SOFTREF_') {
1120 $refParts = explode(':', $recordRef, 2);
1121 if (!is_array($this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'])) {
1122 $this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'] = array();
1123 }
1124 $this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'][] = $fI['ID'];
1125 }
1126 $fileMd5 = md5_file($fI['ID_absFile']);
1127 if (!$this->saveFilesOutsideExportFile) {
1128 // ... and finally add the heavy stuff:
1129 $fileRec['content'] = GeneralUtility::getUrl($fI['ID_absFile']);
1130 } else {
1131 GeneralUtility::upload_copy_move($fI['ID_absFile'], $this->getTemporaryFilesPathForExport() . $fileMd5);
1132 }
1133 $fileRec['content_md5'] = $fileMd5;
1134 $this->dat['files'][$fI['ID']] = $fileRec;
1135 // For soft references, do further processing:
1136 if ($recordRef === '_SOFTREF_') {
1137 // RTE files?
1138 if ($RTEoriginal = $this->getRTEoriginalFilename(PathUtility::basename($fI['ID_absFile']))) {
1139 $RTEoriginal_absPath = PathUtility::dirname($fI['ID_absFile']) . '/' . $RTEoriginal;
1140 if (@is_file($RTEoriginal_absPath)) {
1141 $RTEoriginal_ID = md5($RTEoriginal_absPath);
1142 $fileInfo = stat($RTEoriginal_absPath);
1143 $fileRec = array();
1144 $fileRec['filesize'] = $fileInfo['size'];
1145 $fileRec['filename'] = PathUtility::basename($RTEoriginal_absPath);
1146 $fileRec['filemtime'] = $fileInfo['mtime'];
1147 $fileRec['record_ref'] = '_RTE_COPY_ID:' . $fI['ID'];
1148 $this->dat['header']['files'][$fI['ID']]['RTE_ORIG_ID'] = $RTEoriginal_ID;
1149 // Setting this data in the header
1150 $this->dat['header']['files'][$RTEoriginal_ID] = $fileRec;
1151 $fileMd5 = md5_file($RTEoriginal_absPath);
1152 if (!$this->saveFilesOutsideExportFile) {
1153 // ... and finally add the heavy stuff:
1154 $fileRec['content'] = GeneralUtility::getUrl($RTEoriginal_absPath);
1155 } else {
1156 GeneralUtility::upload_copy_move($RTEoriginal_absPath, $this->getTemporaryFilesPathForExport() . $fileMd5);
1157 }
1158 $fileRec['content_md5'] = $fileMd5;
1159 $this->dat['files'][$RTEoriginal_ID] = $fileRec;
1160 } else {
1161 $this->error('RTE original file "' . PathUtility::stripPathSitePrefix($RTEoriginal_absPath) . '" was not found!');
1162 }
1163 }
1164 // Files with external media?
1165 // This is only done with files grabbed by a softreference parser since it is deemed improbable that hard-referenced files should undergo this treatment.
1166 $html_fI = pathinfo(PathUtility::basename($fI['ID_absFile']));
1167 if ($this->includeExtFileResources && GeneralUtility::inList($this->extFileResourceExtensions, strtolower($html_fI['extension']))) {
1168 $uniquePrefix = '###' . md5($GLOBALS['EXEC_TIME']) . '###';
1169 if (strtolower($html_fI['extension']) === 'css') {
1170 $prefixedMedias = explode($uniquePrefix, preg_replace('/(url[[:space:]]*\\([[:space:]]*["\']?)([^"\')]*)(["\']?[[:space:]]*\\))/i', '\\1' . $uniquePrefix . '\\2' . $uniquePrefix . '\\3', $fileRec['content']));
1171 } else {
1172 // html, htm:
1173 $htmlParser = GeneralUtility::makeInstance(\TYPO3\CMS\Core\Html\HtmlParser::class);
1174 $prefixedMedias = explode($uniquePrefix, $htmlParser->prefixResourcePath($uniquePrefix, $fileRec['content'], array(), $uniquePrefix));
1175 }
1176 $htmlResourceCaptured = FALSE;
1177 foreach ($prefixedMedias as $k => $v) {
1178 if ($k % 2) {
1179 $EXTres_absPath = GeneralUtility::resolveBackPath(PathUtility::dirname($fI['ID_absFile']) . '/' . $v);
1180 $EXTres_absPath = GeneralUtility::getFileAbsFileName($EXTres_absPath);
1181 if ($EXTres_absPath && GeneralUtility::isFirstPartOfStr($EXTres_absPath, PATH_site . $this->fileadminFolderName . '/') && @is_file($EXTres_absPath)) {
1182 $htmlResourceCaptured = TRUE;
1183 $EXTres_ID = md5($EXTres_absPath);
1184 $this->dat['header']['files'][$fI['ID']]['EXT_RES_ID'][] = $EXTres_ID;
1185 $prefixedMedias[$k] = '{EXT_RES_ID:' . $EXTres_ID . '}';
1186 // Add file to memory if it is not set already:
1187 if (!isset($this->dat['header']['files'][$EXTres_ID])) {
1188 $fileInfo = stat($EXTres_absPath);
1189 $fileRec = array();
1190 $fileRec['filesize'] = $fileInfo['size'];
1191 $fileRec['filename'] = PathUtility::basename($EXTres_absPath);
1192 $fileRec['filemtime'] = $fileInfo['mtime'];
1193 $fileRec['record_ref'] = '_EXT_PARENT_:' . $fI['ID'];
1194 // Media relative to the HTML file.
1195 $fileRec['parentRelFileName'] = $v;
1196 // Setting this data in the header
1197 $this->dat['header']['files'][$EXTres_ID] = $fileRec;
1198 // ... and finally add the heavy stuff:
1199 $fileRec['content'] = GeneralUtility::getUrl($EXTres_absPath);
1200 $fileRec['content_md5'] = md5($fileRec['content']);
1201 $this->dat['files'][$EXTres_ID] = $fileRec;
1202 }
1203 }
1204 }
1205 }
1206 if ($htmlResourceCaptured) {
1207 $this->dat['files'][$fI['ID']]['tokenizedContent'] = implode('', $prefixedMedias);
1208 }
1209 }
1210 }
1211 }
1212
1213 /**
1214 * If saveFilesOutsideExportFile is enabled, this function returns the path
1215 * where the files referenced in the export are copied to.
1216 *
1217 * @return string
1218 * @throws \RuntimeException
1219 * @see setSaveFilesOutsideExportFile()
1220 */
1221 public function getTemporaryFilesPathForExport() {
1222 if (!$this->saveFilesOutsideExportFile) {
1223 throw new \RuntimeException('You need to set saveFilesOutsideExportFile to TRUE before you want to get the temporary files path for export.', 1401205213);
1224 }
1225 if ($this->temporaryFilesPathForExport === NULL) {
1226 $temporaryFolderName = $this->getTemporaryFolderName();
1227 $this->temporaryFilesPathForExport = $temporaryFolderName . '/';
1228 }
1229 return $this->temporaryFilesPathForExport;
1230 }
1231
1232 /**
1233 *
1234 * @return string
1235 */
1236 protected function getTemporaryFolderName() {
1237 $temporaryPath = PATH_site . 'typo3temp/';
1238 do {
1239 $temporaryFolderName = $temporaryPath . 'export_temp_files_' . mt_rand(1, PHP_INT_MAX);
1240 } while (is_dir($temporaryFolderName));
1241 GeneralUtility::mkdir($temporaryFolderName);
1242 return $temporaryFolderName;
1243 }
1244
1245 /**
1246 * DB relations flattend to 1-dim array.
1247 * The list will be unique, no table/uid combination will appear twice.
1248 *
1249 * @param array $dbrels 2-dim Array of database relations organized by table key
1250 * @return array 1-dim array where entries are table:uid and keys are array with table/id
1251 */
1252 public function flatDBrels($dbrels) {
1253 $list = array();
1254 foreach ($dbrels as $dat) {
1255 if ($dat['type'] == 'db') {
1256 foreach ($dat['itemArray'] as $i) {
1257 $list[$i['table'] . ':' . $i['id']] = $i;
1258 }
1259 }
1260 if ($dat['type'] == 'flex' && is_array($dat['flexFormRels']['db'])) {
1261 foreach ($dat['flexFormRels']['db'] as $subList) {
1262 foreach ($subList as $i) {
1263 $list[$i['table'] . ':' . $i['id']] = $i;
1264 }
1265 }
1266 }
1267 }
1268 return $list;
1269 }
1270
1271 /**
1272 * Soft References flattend to 1-dim array.
1273 *
1274 * @param array $dbrels 2-dim Array of database relations organized by table key
1275 * @return array 1-dim array where entries are arrays with properties of the soft link found and keys are a unique combination of field, spKey, structure path if applicable and token ID
1276 */
1277 public function flatSoftRefs($dbrels) {
1278 $list = array();
1279 foreach ($dbrels as $field => $dat) {
1280 if (is_array($dat['softrefs']['keys'])) {
1281 foreach ($dat['softrefs']['keys'] as $spKey => $elements) {
1282 if (is_array($elements)) {
1283 foreach ($elements as $subKey => $el) {
1284 $lKey = $field . ':' . $spKey . ':' . $subKey;
1285 $list[$lKey] = array_merge(array('field' => $field, 'spKey' => $spKey), $el);
1286 // Add file_ID key to header - slightly "risky" way of doing this because if the calculation
1287 // changes for the same value in $this->records[...] this will not work anymore!
1288 if ($el['subst'] && $el['subst']['relFileName']) {
1289 $list[$lKey]['file_ID'] = md5(PATH_site . $el['subst']['relFileName']);
1290 }
1291 }
1292 }
1293 }
1294 }
1295 if ($dat['type'] == 'flex' && is_array($dat['flexFormRels']['softrefs'])) {
1296 foreach ($dat['flexFormRels']['softrefs'] as $structurePath => $subSoftrefs) {
1297 if (is_array($subSoftrefs['keys'])) {
1298 foreach ($subSoftrefs['keys'] as $spKey => $elements) {
1299 foreach ($elements as $subKey => $el) {
1300 $lKey = $field . ':' . $structurePath . ':' . $spKey . ':' . $subKey;
1301 $list[$lKey] = array_merge(array('field' => $field, 'spKey' => $spKey, 'structurePath' => $structurePath), $el);
1302 // Add file_ID key to header - slightly "risky" way of doing this because if the calculation
1303 // changes for the same value in $this->records[...] this will not work anymore!
1304 if ($el['subst'] && $el['subst']['relFileName']) {
1305 $list[$lKey]['file_ID'] = md5(PATH_site . $el['subst']['relFileName']);
1306 }
1307 }
1308 }
1309 }
1310 }
1311 }
1312 }
1313 return $list;
1314 }
1315
1316 /**
1317 * If include fields for a specific record type are set, the data
1318 * are filtered out with fields are not included in the fields.
1319 *
1320 * @param string $table The record type to be filtered
1321 * @param array $row The data to be filtered
1322 * @return array The filtered record row
1323 */
1324 protected function filterRecordFields($table, array $row) {
1325 if (isset($this->recordTypesIncludeFields[$table])) {
1326 $includeFields = array_unique(array_merge(
1327 $this->recordTypesIncludeFields[$table],
1328 $this->defaultRecordIncludeFields
1329 ));
1330 $newRow = array();
1331 foreach ($row as $key => $value) {
1332 if (in_array($key, $includeFields)) {
1333 $newRow[$key] = $value;
1334 }
1335 }
1336 } else {
1337 $newRow = $row;
1338 }
1339 return $newRow;
1340 }
1341
1342
1343 /**************************
1344 * File Output
1345 *************************/
1346
1347 /**
1348 * This compiles and returns the data content for an exported file
1349 *
1350 * @param string $type Type of output; "xml" gives xml, otherwise serialized array, possibly compressed.
1351 * @return string The output file stream
1352 */
1353 public function compileMemoryToFileContent($type = '') {
1354 if ($type == 'xml') {
1355 $out = $this->createXML();
1356 } else {
1357 $compress = $this->doOutputCompress();
1358 $out = '';
1359 // adding header:
1360 $out .= $this->addFilePart(serialize($this->dat['header']), $compress);
1361 // adding records:
1362 $out .= $this->addFilePart(serialize($this->dat['records']), $compress);
1363 // adding files:
1364 $out .= $this->addFilePart(serialize($this->dat['files']), $compress);
1365 // adding files_fal:
1366 $out .= $this->addFilePart(serialize($this->dat['files_fal']), $compress);
1367 }
1368 return $out;
1369 }
1370
1371 /**
1372 * Creates XML string from input array
1373 *
1374 * @return string XML content
1375 */
1376 public function createXML() {
1377 // Options:
1378 $options = array(
1379 'alt_options' => array(
1380 '/header' => array(
1381 'disableTypeAttrib' => TRUE,
1382 'clearStackPath' => TRUE,
1383 'parentTagMap' => array(
1384 'files' => 'file',
1385 'files_fal' => 'file',
1386 'records' => 'table',
1387 'table' => 'rec',
1388 'rec:rels' => 'relations',
1389 'relations' => 'element',
1390 'filerefs' => 'file',
1391 'pid_lookup' => 'page_contents',
1392 'header:relStaticTables' => 'static_tables',
1393 'static_tables' => 'tablename',
1394 'excludeMap' => 'item',
1395 'softrefCfg' => 'softrefExportMode',
1396 'extensionDependencies' => 'extkey',
1397 'softrefs' => 'softref_element'
1398 ),
1399 'alt_options' => array(
1400 '/pagetree' => array(
1401 'disableTypeAttrib' => TRUE,
1402 'useIndexTagForNum' => 'node',
1403 'parentTagMap' => array(
1404 'node:subrow' => 'node'
1405 )
1406 ),
1407 '/pid_lookup/page_contents' => array(
1408 'disableTypeAttrib' => TRUE,
1409 'parentTagMap' => array(
1410 'page_contents' => 'table'
1411 ),
1412 'grandParentTagMap' => array(
1413 'page_contents/table' => 'item'
1414 )
1415 )
1416 )
1417 ),
1418 '/records' => array(
1419 'disableTypeAttrib' => TRUE,
1420 'parentTagMap' => array(
1421 'records' => 'tablerow',
1422 'tablerow:data' => 'fieldlist',
1423 'tablerow:rels' => 'related',
1424 'related' => 'field',
1425 'field:itemArray' => 'relations',
1426 'field:newValueFiles' => 'filerefs',
1427 'field:flexFormRels' => 'flexform',
1428 'relations' => 'element',
1429 'filerefs' => 'file',
1430 'flexform:db' => 'db_relations',
1431 'flexform:file' => 'file_relations',
1432 'flexform:softrefs' => 'softref_relations',
1433 'softref_relations' => 'structurePath',
1434 'db_relations' => 'path',
1435 'file_relations' => 'path',
1436 'path' => 'element',
1437 'keys' => 'softref_key',
1438 'softref_key' => 'softref_element'
1439 ),
1440 'alt_options' => array(
1441 '/records/tablerow/fieldlist' => array(
1442 'useIndexTagForAssoc' => 'field'
1443 )
1444 )
1445 ),
1446 '/files' => array(
1447 'disableTypeAttrib' => TRUE,
1448 'parentTagMap' => array(
1449 'files' => 'file'
1450 )
1451 ),
1452 '/files_fal' => array(
1453 'disableTypeAttrib' => TRUE,
1454 'parentTagMap' => array(
1455 'files_fal' => 'file'
1456 )
1457 )
1458 )
1459 );
1460 // Creating XML file from $outputArray:
1461 $charset = $this->dat['header']['charset'] ?: 'utf-8';
1462 $XML = '<?xml version="1.0" encoding="' . $charset . '" standalone="yes" ?>' . LF;
1463 $XML .= GeneralUtility::array2xml($this->dat, '', 0, 'T3RecordDocument', 0, $options);
1464 return $XML;
1465 }
1466
1467 /**
1468 * Returns TRUE if the output should be compressed.
1469 *
1470 * @return bool TRUE if compression is possible AND requested.
1471 */
1472 public function doOutputCompress() {
1473 return $this->compress && !$this->dontCompress;
1474 }
1475
1476 /**
1477 * Returns a content part for a filename being build.
1478 *
1479 * @param array $data Data to store in part
1480 * @param bool $compress Compress file?
1481 * @return string Content stream.
1482 */
1483 public function addFilePart($data, $compress = FALSE) {
1484 if ($compress) {
1485 $data = gzcompress($data);
1486 }
1487 return md5($data) . ':' . ($compress ? '1' : '0') . ':' . str_pad(strlen($data), 10, '0', STR_PAD_LEFT) . ':' . $data . ':';
1488 }
1489
1490 /***********************
1491 * Import
1492 ***********************/
1493
1494 /**
1495 * Initialize all settings for the import
1496 *
1497 * @return void
1498 */
1499 protected function initializeImport() {
1500 // Set this flag to indicate that an import is being/has been done.
1501 $this->doesImport = 1;
1502 // Initialize:
1503 // These vars MUST last for the whole section not being cleared. They are used by the method setRelations() which are called at the end of the import session.
1504 $this->import_mapId = array();
1505 $this->import_newId = array();
1506 $this->import_newId_pids = array();
1507 // Temporary files stack initialized:
1508 $this->unlinkFiles = array();
1509 $this->alternativeFileName = array();
1510 $this->alternativeFilePath = array();
1511
1512 $this->initializeStorageObjects();
1513 }
1514
1515 /**
1516 * Initialize the all present storage objects
1517 *
1518 * @return void
1519 */
1520 protected function initializeStorageObjects() {
1521 /** @var $storageRepository \TYPO3\CMS\Core\Resource\StorageRepository */
1522 $storageRepository = GeneralUtility::makeInstance(\TYPO3\CMS\Core\Resource\StorageRepository::class);
1523 $this->storageObjects = $storageRepository->findAll();
1524 }
1525
1526 /**
1527 * Imports the internal data array to $pid.
1528 *
1529 * @param int $pid Page ID in which to import the content
1530 * @return void
1531 */
1532 public function importData($pid) {
1533
1534 $this->initializeImport();
1535
1536 // Write sys_file_storages first
1537 $this->writeSysFileStorageRecords();
1538 // Write sys_file records and write the binary file data
1539 $this->writeSysFileRecords();
1540 // Write records, first pages, then the rest
1541 // Fields with "hard" relations to database, files and flexform fields are kept empty during this run
1542 $this->writeRecords_pages($pid);
1543 $this->writeRecords_records($pid);
1544 // Finally all the file and DB record references must be fixed. This is done after all records have supposedly been written to database:
1545 // $this->import_mapId will indicate two things: 1) that a record WAS written to db and 2) that it has got a new id-number.
1546 $this->setRelations();
1547 // And when all DB relations are in place, we can fix file and DB relations in flexform fields (since data structures often depends on relations to a DS record):
1548 $this->setFlexFormRelations();
1549 // Unlink temporary files:
1550 $this->unlinkTempFiles();
1551 // Finally, traverse all records and process softreferences with substitution attributes.
1552 $this->processSoftReferences();
1553 // After all migrate records using sys_file_reference now
1554 if ($this->legacyImport) {
1555 $this->migrateLegacyImportRecords();
1556 }
1557 }
1558
1559 /**
1560 * Imports the sys_file_storage records from internal data array.
1561 *
1562 * @return void
1563 */
1564 protected function writeSysFileStorageRecords() {
1565 if (!isset($this->dat['header']['records']['sys_file_storage'])) {
1566 return;
1567 }
1568 $sysFileStorageUidsToBeResetToDefaultStorage = array();
1569 foreach ($this->dat['header']['records']['sys_file_storage'] as $sysFileStorageUid => $_) {
1570 $storageRecord = $this->dat['records']['sys_file_storage:' . $sysFileStorageUid]['data'];
1571 // continue with Local, writable and online storage only
1572 if ($storageRecord['driver'] === 'Local' && $storageRecord['is_writable'] && $storageRecord['is_online']) {
1573 $useThisStorageUidInsteadOfTheOneInImport = 0;
1574 /** @var $localStorage \TYPO3\CMS\Core\Resource\ResourceStorage */
1575 foreach ($this->storageObjects as $localStorage) {
1576 // check the available storage for Local, writable and online ones
1577 if ($localStorage->getDriverType() === 'Local' && $localStorage->isWritable() && $localStorage->isOnline()) {
1578 // check if there is already a identical storage present (same pathType and basePath)
1579 $storageRecordConfiguration = ResourceFactory::getInstance()->convertFlexFormDataToConfigurationArray($storageRecord['configuration']);
1580 $localStorageRecordConfiguration = $localStorage->getConfiguration();
1581 if (
1582 $storageRecordConfiguration['pathType'] === $localStorageRecordConfiguration['pathType']
1583 && $storageRecordConfiguration['basePath'] === $localStorageRecordConfiguration['basePath']
1584 ) {
1585 // same storage is already present
1586 $useThisStorageUidInsteadOfTheOneInImport = $localStorage->getUid();
1587 break;
1588 }
1589 }
1590 }
1591 if ($useThisStorageUidInsteadOfTheOneInImport > 0) {
1592 // same storage is already present; map the to be imported one to the present one
1593 $this->import_mapId['sys_file_storage'][$sysFileStorageUid] = $useThisStorageUidInsteadOfTheOneInImport;
1594 } else {
1595 // Local, writable and online storage. Is allowed to be used to later write files in.
1596 $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);
1597 }
1598 } else {
1599 // Storage with non Local drivers could be imported but must not be used to saves files in, because you
1600 // could not be sure, that this is supported. The default storage will be used in this case.
1601 // It could happen that non writable and non online storage will be created as dupes because you could not
1602 // check the detailed configuration options at this point
1603 $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);
1604 $sysFileStorageUidsToBeResetToDefaultStorage[] = $sysFileStorageUid;
1605 }
1606
1607 }
1608
1609 // Importing the added ones
1610 $tce = $this->getNewTCE();
1611 // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.
1612 $tce->reverseOrder = 1;
1613 $tce->isImporting = TRUE;
1614 $tce->start($this->import_data, array());
1615 $tce->process_datamap();
1616 $this->addToMapId($tce->substNEWwithIDs);
1617
1618 $defaultStorageUid = NULL;
1619 // get default storage
1620 $defaultStorage = ResourceFactory::getInstance()->getDefaultStorage();
1621 if ($defaultStorage !== NULL) {
1622 $defaultStorageUid = $defaultStorage->getUid();
1623 }
1624 foreach ($sysFileStorageUidsToBeResetToDefaultStorage as $sysFileStorageUidToBeResetToDefaultStorage) {
1625 $this->import_mapId['sys_file_storage'][$sysFileStorageUidToBeResetToDefaultStorage] = $defaultStorageUid;
1626 }
1627
1628 // unset the sys_file_storage records to prevent a import in writeRecords_records
1629 unset($this->dat['header']['records']['sys_file_storage']);
1630 }
1631
1632 /**
1633 * Imports the sys_file records and the binary files data from internal data array.
1634 *
1635 * @return void
1636 */
1637 protected function writeSysFileRecords() {
1638 if (!isset($this->dat['header']['records']['sys_file'])) {
1639 return;
1640 }
1641 $this->addGeneralErrorsByTable('sys_file');
1642
1643 // fetch fresh storage records from database
1644 $storageRecords = $this->fetchStorageRecords();
1645
1646 $defaultStorage = ResourceFactory::getInstance()->getDefaultStorage();
1647
1648 $sanitizedFolderMappings = array();
1649
1650 foreach ($this->dat['header']['records']['sys_file'] as $sysFileUid => $_) {
1651 $fileRecord = $this->dat['records']['sys_file:' . $sysFileUid]['data'];
1652
1653 $temporaryFile = NULL;
1654 // check if there is the right file already in the local folder
1655 if ($this->filesPathForImport !== NULL) {
1656 if (is_file($this->filesPathForImport . '/' . $fileRecord['sha1']) && sha1_file($this->filesPathForImport . '/' . $fileRecord['sha1']) === $fileRecord['sha1']) {
1657 $temporaryFile = $this->filesPathForImport . '/' . $fileRecord['sha1'];
1658 }
1659 }
1660
1661 // save file to disk
1662 if ($temporaryFile === NULL) {
1663 $fileId = md5($fileRecord['storage'] . ':' . $fileRecord['identifier_hash']);
1664 $temporaryFile = $this->writeTemporaryFileFromData($fileId);
1665 if ($temporaryFile === NULL) {
1666 // error on writing the file. Error message was already added
1667 continue;
1668 }
1669 }
1670
1671 $originalStorageUid = $fileRecord['storage'];
1672 $useStorageFromStorageRecords = FALSE;
1673
1674 // replace storage id, if a alternative one was registered
1675 if (isset($this->import_mapId['sys_file_storage'][$fileRecord['storage']])) {
1676 $fileRecord['storage'] = $this->import_mapId['sys_file_storage'][$fileRecord['storage']];
1677 $useStorageFromStorageRecords = TRUE;
1678 }
1679
1680 if (empty($fileRecord['storage']) && !$this->isFallbackStorage($fileRecord['storage'])) {
1681 // no storage for the file is defined, mostly because of a missing default storage.
1682 $this->error('Error: No storage for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $originalStorageUid . '"');
1683 continue;
1684 }
1685
1686 // using a storage from the local storage is only allowed, if the uid is present in the
1687 // mapping. Only in this case we could be sure, that it's a local, online and writable storage.
1688 if ($useStorageFromStorageRecords && isset($storageRecords[$fileRecord['storage']])) {
1689 /** @var $storage \TYPO3\CMS\Core\Resource\ResourceStorage */
1690 $storage = ResourceFactory::getInstance()->getStorageObject($fileRecord['storage'], $storageRecords[$fileRecord['storage']]);
1691 } elseif ($this->isFallbackStorage($fileRecord['storage'])) {
1692 $storage = ResourceFactory::getInstance()->getStorageObject(0);
1693 } elseif ($defaultStorage !== NULL) {
1694 $storage = $defaultStorage;
1695 } else {
1696 $this->error('Error: No storage available for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1697 continue;
1698 }
1699
1700 $newFile = NULL;
1701
1702 // check, if there is a identical file
1703 try {
1704 if ($storage->hasFile($fileRecord['identifier'])) {
1705 $file = $storage->getFile($fileRecord['identifier']);
1706 if ($file->getSha1() === $fileRecord['sha1']) {
1707 $newFile = $file;
1708 }
1709 }
1710 } catch (Exception $e) {}
1711
1712 if ($newFile === NULL) {
1713
1714 $folderName = PathUtility::dirname(ltrim($fileRecord['identifier'], '/'));
1715 if (in_array($folderName, $sanitizedFolderMappings)) {
1716 $folderName = $sanitizedFolderMappings[$folderName];
1717 }
1718 if (!$storage->hasFolder($folderName)) {
1719 try {
1720 $importFolder = $storage->createFolder($folderName);
1721 if ($importFolder->getIdentifier() !== $folderName && !in_array($folderName, $sanitizedFolderMappings)) {
1722 $sanitizedFolderMappings[$folderName] = $importFolder->getIdentifier();
1723 }
1724 } catch (Exception $e) {
1725 $this->error('Error: Folder could not be created for file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1726 continue;
1727 }
1728 } else {
1729 $importFolder = $storage->getFolder($folderName);
1730 }
1731
1732 try {
1733 /** @var $newFile \TYPO3\CMS\Core\Resource\File */
1734 $newFile = $storage->addFile($temporaryFile, $importFolder, $fileRecord['name']);
1735 } catch (Exception $e) {
1736 $this->error('Error: File could not be added to the storage: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1737 continue;
1738 }
1739
1740 if ($newFile->getSha1() !== $fileRecord['sha1']) {
1741 $this->error('Error: The hash of the written file is not identical to the import data! File could be corrupted! File: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1742 }
1743 }
1744
1745 // save the new uid in the import id map
1746 $this->import_mapId['sys_file'][$fileRecord['uid']] = $newFile->getUid();
1747 $this->fixUidLocalInSysFileReferenceRecords($fileRecord['uid'], $newFile->getUid());
1748
1749 }
1750
1751 // unset the sys_file records to prevent a import in writeRecords_records
1752 unset($this->dat['header']['records']['sys_file']);
1753 }
1754
1755 /**
1756 * Checks if the $storageId is the id of the fallback storage
1757 *
1758 * @param int|string $storageId
1759 * @return bool
1760 */
1761 protected function isFallbackStorage($storageId) {
1762 return $storageId === 0 || $storageId === '0';
1763 }
1764
1765 /**
1766 * Normally the importer works like the following:
1767 * Step 1: import the records with cleared field values of relation fields (see addSingle())
1768 * Step 2: update the records with the right relation ids (see setRelations())
1769 *
1770 * In step 2 the saving fields of type "relation to sys_file_reference" checks the related sys_file_reference
1771 * record (created in step 1) with the FileExtensionFilter for matching file extensions of the related file.
1772 * To make this work correct, the uid_local of sys_file_reference records has to be not empty AND has to
1773 * relate to the correct (imported) sys_file record uid!!!
1774 *
1775 * This is fixed here.
1776 *
1777 * @param int $oldFileUid
1778 * @param int $newFileUid
1779 * @return void
1780 */
1781 protected function fixUidLocalInSysFileReferenceRecords($oldFileUid, $newFileUid) {
1782 if (!isset($this->dat['header']['records']['sys_file_reference'])) {
1783 return;
1784 }
1785
1786 foreach ($this->dat['header']['records']['sys_file_reference'] as $sysFileReferenceUid => $_) {
1787 $fileReferenceRecord = $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'];
1788 if ($fileReferenceRecord['uid_local'] == $oldFileUid) {
1789 $fileReferenceRecord['uid_local'] = $newFileUid;
1790 $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'] = $fileReferenceRecord;
1791 }
1792 }
1793 }
1794
1795 /**
1796 * Initializes the folder for legacy imports as subfolder of backend users default upload folder
1797 *
1798 * @return void
1799 */
1800 protected function initializeLegacyImportFolder() {
1801 /** @var \TYPO3\CMS\Core\Resource\Folder $folder */
1802 $folder = $GLOBALS['BE_USER']->getDefaultUploadFolder();
1803 if ($folder === FALSE) {
1804 $this->error('Error: the backend users default upload folder is missing! No files will be imported!');
1805 }
1806 if (!$folder->hasFolder($this->legacyImportTargetPath)) {
1807 try {
1808 $this->legacyImportFolder = $folder->createFolder($this->legacyImportTargetPath);
1809 } catch (\TYPO3\CMS\Core\Exception $e) {
1810 $this->error('Error: the import folder in the default upload folder could not be created! No files will be imported!');
1811 }
1812 } else {
1813 $this->legacyImportFolder = $folder->getSubFolder($this->legacyImportTargetPath);
1814 }
1815
1816 }
1817
1818 /**
1819 * Fetched fresh storage records from database because the new imported
1820 * ones are not in cached data of the StorageRepository
1821 *
1822 * @return bool|array
1823 */
1824 protected function fetchStorageRecords() {
1825 $whereClause = BackendUtility::BEenableFields('sys_file_storage');
1826 $whereClause .= BackendUtility::deleteClause('sys_file_storage');
1827
1828 $rows = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows(
1829 '*',
1830 'sys_file_storage',
1831 '1=1' . $whereClause,
1832 '',
1833 '',
1834 '',
1835 'uid'
1836 );
1837
1838 return $rows;
1839 }
1840
1841 /**
1842 * Writes the file from import array to temp dir and returns the filename of it.
1843 *
1844 * @param string $fileId
1845 * @param string $dataKey
1846 * @return string Absolute filename of the temporary filename of the file
1847 */
1848 protected function writeTemporaryFileFromData($fileId, $dataKey = 'files_fal') {
1849 $temporaryFilePath = NULL;
1850 if (is_array($this->dat[$dataKey][$fileId])) {
1851 $temporaryFilePathInternal = GeneralUtility::tempnam('import_temp_');
1852 GeneralUtility::writeFile($temporaryFilePathInternal, $this->dat[$dataKey][$fileId]['content']);
1853 clearstatcache();
1854 if (@is_file($temporaryFilePathInternal)) {
1855 $this->unlinkFiles[] = $temporaryFilePathInternal;
1856 if (filesize($temporaryFilePathInternal) == $this->dat[$dataKey][$fileId]['filesize']) {
1857 $temporaryFilePath = $temporaryFilePathInternal;
1858 } else {
1859 $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' had a size (' . filesize($temporaryFilePathInternal) . ') different from the original (' . $this->dat[$dataKey][$fileId]['filesize'] . ')', 1);
1860 }
1861 } else {
1862 $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' was not written as it should have been!', 1);
1863 }
1864 } else {
1865 $this->error('Error: No file found for ID ' . $fileId, 1);
1866 }
1867 return $temporaryFilePath;
1868 }
1869
1870 /**
1871 * Writing pagetree/pages to database:
1872 *
1873 * @param int $pid PID in which to import. If the operation is an update operation, the root of the page tree inside will be moved to this PID unless it is the same as the root page from the import
1874 * @return void
1875 * @see writeRecords_records()
1876 */
1877 public function writeRecords_pages($pid) {
1878 // First, write page structure if any:
1879 if (is_array($this->dat['header']['records']['pages'])) {
1880 $this->addGeneralErrorsByTable('pages');
1881 // $pageRecords is a copy of the pages array in the imported file. Records here are unset one by one when the addSingle function is called.
1882 $pageRecords = $this->dat['header']['records']['pages'];
1883 $this->import_data = array();
1884 // First add page tree if any
1885 if (is_array($this->dat['header']['pagetree'])) {
1886 $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);
1887 foreach ($pagesFromTree as $uid) {
1888 $thisRec = $this->dat['header']['records']['pages'][$uid];
1889 // PID: Set the main $pid, unless a NEW-id is found
1890 $setPid = isset($this->import_newId_pids[$thisRec['pid']]) ? $this->import_newId_pids[$thisRec['pid']] : $pid;
1891 $this->addSingle('pages', $uid, $setPid);
1892 unset($pageRecords[$uid]);
1893 }
1894 }
1895 // Then add all remaining pages not in tree on root level:
1896 if (count($pageRecords)) {
1897 $remainingPageUids = array_keys($pageRecords);
1898 foreach ($remainingPageUids as $pUid) {
1899 $this->addSingle('pages', $pUid, $pid);
1900 }
1901 }
1902 // Now write to database:
1903 $tce = $this->getNewTCE();
1904 $tce->isImporting = TRUE;
1905 $this->callHook('before_writeRecordsPages', array(
1906 'tce' => &$tce,
1907 'data' => &$this->import_data
1908 ));
1909 $tce->suggestedInsertUids = $this->suggestedInsertUids;
1910 $tce->start($this->import_data, array());
1911 $tce->process_datamap();
1912 $this->callHook('after_writeRecordsPages', array(
1913 'tce' => &$tce
1914 ));
1915 // post-processing: Registering new ids (end all tcemain sessions with this)
1916 $this->addToMapId($tce->substNEWwithIDs);
1917 // In case of an update, order pages from the page tree correctly:
1918 if ($this->update && is_array($this->dat['header']['pagetree'])) {
1919 $this->writeRecords_pages_order($pid);
1920 }
1921 }
1922 }
1923
1924 /**
1925 * Organize all updated pages in page tree so they are related like in the import file
1926 * Only used for updates and when $this->dat['header']['pagetree'] is an array.
1927 *
1928 * @param int $pid Page id in which to import
1929 * @return void
1930 * @access private
1931 * @see writeRecords_pages(), writeRecords_records_order()
1932 */
1933 public function writeRecords_pages_order($pid) {
1934 $cmd_data = array();
1935 // Get uid-pid relations and traverse them in order to map to possible new IDs
1936 $pidsFromTree = $this->flatInversePageTree_pid($this->dat['header']['pagetree']);
1937 foreach ($pidsFromTree as $origPid => $newPid) {
1938 if ($newPid >= 0 && $this->dontIgnorePid('pages', $origPid)) {
1939 // If the page had a new id (because it was created) use that instead!
1940 if (substr($this->import_newId_pids[$origPid], 0, 3) === 'NEW') {
1941 if ($this->import_mapId['pages'][$origPid]) {
1942 $mappedPid = $this->import_mapId['pages'][$origPid];
1943 $cmd_data['pages'][$mappedPid]['move'] = $newPid;
1944 }
1945 } else {
1946 $cmd_data['pages'][$origPid]['move'] = $newPid;
1947 }
1948 }
1949 }
1950 // Execute the move commands if any:
1951 if (count($cmd_data)) {
1952 $tce = $this->getNewTCE();
1953 $this->callHook('before_writeRecordsPagesOrder', array(
1954 'tce' => &$tce,
1955 'data' => &$cmd_data
1956 ));
1957 $tce->start(array(), $cmd_data);
1958 $tce->process_cmdmap();
1959 $this->callHook('after_writeRecordsPagesOrder', array(
1960 'tce' => &$tce
1961 ));
1962 }
1963 }
1964
1965 /**
1966 * Write all database records except pages (writtein in writeRecords_pages())
1967 *
1968 * @param int $pid Page id in which to import
1969 * @return void
1970 * @see writeRecords_pages()
1971 */
1972 public function writeRecords_records($pid) {
1973 // Write the rest of the records
1974 $this->import_data = array();
1975 if (is_array($this->dat['header']['records'])) {
1976 foreach ($this->dat['header']['records'] as $table => $recs) {
1977 $this->addGeneralErrorsByTable($table);
1978 if ($table != 'pages') {
1979 foreach ($recs as $uid => $thisRec) {
1980 // PID: Set the main $pid, unless a NEW-id is found
1981 $setPid = isset($this->import_mapId['pages'][$thisRec['pid']]) ? $this->import_mapId['pages'][$thisRec['pid']] : $pid;
1982 if (is_array($GLOBALS['TCA'][$table]) && $GLOBALS['TCA'][$table]['ctrl']['rootLevel']) {
1983 $setPid = 0;
1984 }
1985 // Add record:
1986 $this->addSingle($table, $uid, $setPid);
1987 }
1988 }
1989 }
1990 } else {
1991 $this->error('Error: No records defined in internal data array.');
1992 }
1993 // Now write to database:
1994 $tce = $this->getNewTCE();
1995 $this->callHook('before_writeRecordsRecords', array(
1996 'tce' => &$tce,
1997 'data' => &$this->import_data
1998 ));
1999 $tce->suggestedInsertUids = $this->suggestedInsertUids;
2000 // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.
2001 $tce->reverseOrder = 1;
2002 $tce->isImporting = TRUE;
2003 $tce->start($this->import_data, array());
2004 $tce->process_datamap();
2005 $this->callHook('after_writeRecordsRecords', array(
2006 'tce' => &$tce
2007 ));
2008 // post-processing: Removing files and registering new ids (end all tcemain sessions with this)
2009 $this->addToMapId($tce->substNEWwithIDs);
2010 // In case of an update, order pages from the page tree correctly:
2011 if ($this->update) {
2012 $this->writeRecords_records_order($pid);
2013 }
2014 }
2015
2016 /**
2017 * Organize all updated record to their new positions.
2018 * Only used for updates
2019 *
2020 * @param int $mainPid Main PID into which we import.
2021 * @return void
2022 * @access private
2023 * @see writeRecords_records(), writeRecords_pages_order()
2024 */
2025 public function writeRecords_records_order($mainPid) {
2026 $cmd_data = array();
2027 if (is_array($this->dat['header']['pagetree'])) {
2028 $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);
2029 } else {
2030 $pagesFromTree = array();
2031 }
2032 if (is_array($this->dat['header']['pid_lookup'])) {
2033 foreach ($this->dat['header']['pid_lookup'] as $pid => $recList) {
2034 $newPid = isset($this->import_mapId['pages'][$pid]) ? $this->import_mapId['pages'][$pid] : $mainPid;
2035 if (\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($newPid)) {
2036 foreach ($recList as $tableName => $uidList) {
2037 // If $mainPid===$newPid then we are on root level and we can consider to move pages as well!
2038 // (they will not be in the page tree!)
2039 if (($tableName != 'pages' || !$pagesFromTree[$pid]) && is_array($uidList)) {
2040 $uidList = array_reverse(array_keys($uidList));
2041 foreach ($uidList as $uid) {
2042 if ($this->dontIgnorePid($tableName, $uid)) {
2043 $cmd_data[$tableName][$uid]['move'] = $newPid;
2044 } else {
2045
2046 }
2047 }
2048 }
2049 }
2050 }
2051 }
2052 }
2053 // Execute the move commands if any:
2054 if (count($cmd_data)) {
2055 $tce = $this->getNewTCE();
2056 $this->callHook('before_writeRecordsRecordsOrder', array(
2057 'tce' => &$tce,
2058 'data' => &$cmd_data
2059 ));
2060 $tce->start(array(), $cmd_data);
2061 $tce->process_cmdmap();
2062 $this->callHook('after_writeRecordsRecordsOrder', array(
2063 'tce' => &$tce
2064 ));
2065 }
2066 }
2067
2068 /**
2069 * Adds a single record to the $importData array. Also copies files to tempfolder.
2070 * However all File/DB-references and flexform field contents are set to blank for now!
2071 * That is done with setRelations() later
2072 *
2073 * @param string $table Table name (from import memory)
2074 * @param int $uid Record UID (from import memory)
2075 * @param int $pid Page id
2076 * @return void
2077 * @see writeRecords()
2078 */
2079 public function addSingle($table, $uid, $pid) {
2080 if ($this->import_mode[$table . ':' . $uid] === 'exclude') {
2081 return;
2082 }
2083 $record = $this->dat['records'][$table . ':' . $uid]['data'];
2084 if (is_array($record)) {
2085 if ($this->update && $this->doesRecordExist($table, $uid) && $this->import_mode[$table . ':' . $uid] !== 'as_new') {
2086 $ID = $uid;
2087 } elseif ($table === 'sys_file_metadata' && $record['sys_language_uid'] == '0' && $this->import_mapId['sys_file'][$record['file']]) {
2088 // on adding sys_file records the belonging sys_file_metadata record was also created
2089 // if there is one the record need to be overwritten instead of creating a new one.
2090 $recordInDatabase = $GLOBALS['TYPO3_DB']->exec_SELECTgetSingleRow(
2091 'uid',
2092 'sys_file_metadata',
2093 'file = ' . $this->import_mapId['sys_file'][$record['file']] . ' AND sys_language_uid = 0 AND pid = 0'
2094 );
2095 // if no record could be found, $this->import_mapId['sys_file'][$record['file']] is pointing
2096 // to a file, that was already there, thus a new metadata record should be created
2097 if (is_array($recordInDatabase)) {
2098 $this->import_mapId['sys_file_metadata'][$record['uid']] = $recordInDatabase['uid'];
2099 $ID = $recordInDatabase['uid'];
2100 } else {
2101 $ID = uniqid('NEW', TRUE);
2102 }
2103
2104 } else {
2105 $ID = uniqid('NEW', TRUE);
2106 }
2107 $this->import_newId[$table . ':' . $ID] = array('table' => $table, 'uid' => $uid);
2108 if ($table == 'pages') {
2109 $this->import_newId_pids[$uid] = $ID;
2110 }
2111 // Set main record data:
2112 $this->import_data[$table][$ID] = $record;
2113 $this->import_data[$table][$ID]['tx_impexp_origuid'] = $this->import_data[$table][$ID]['uid'];
2114 // Reset permission data:
2115 if ($table === 'pages') {
2116 // Have to reset the user/group IDs so pages are owned by importing user. Otherwise strange things may happen for non-admins!
2117 unset($this->import_data[$table][$ID]['perms_userid']);
2118 unset($this->import_data[$table][$ID]['perms_groupid']);
2119 }
2120 // PID and UID:
2121 unset($this->import_data[$table][$ID]['uid']);
2122 // Updates:
2123 if (\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($ID)) {
2124 unset($this->import_data[$table][$ID]['pid']);
2125 } else {
2126 // Inserts:
2127 $this->import_data[$table][$ID]['pid'] = $pid;
2128 if (($this->import_mode[$table . ':' . $uid] === 'force_uid' && $this->update || $this->force_all_UIDS) && $GLOBALS['BE_USER']->isAdmin()) {
2129 $this->import_data[$table][$ID]['uid'] = $uid;
2130 $this->suggestedInsertUids[$table . ':' . $uid] = 'DELETE';
2131 }
2132 }
2133 // Setting db/file blank:
2134 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
2135 switch ((string)$config['type']) {
2136 case 'db':
2137
2138 case 'file':
2139 // Fixed later in ->setRelations() [because we need to know ALL newly created IDs before we can map relations!]
2140 // In the meantime we set NO values for relations.
2141 //
2142 // BUT for field uid_local of table sys_file_reference the relation MUST not be cleared here,
2143 // because the value is already the uid of the right imported sys_file record.
2144 // @see fixUidLocalInSysFileReferenceRecords()
2145 // If it's empty or a uid to another record the FileExtensionFilter will throw an exception or
2146 // delete the reference record if the file extension of the related record doesn't match.
2147 if ($table !== 'sys_file_reference' && $field !== 'uid_local') {
2148 $this->import_data[$table][$ID][$field] = '';
2149 }
2150 break;
2151 case 'flex':
2152 // Fixed later in setFlexFormRelations()
2153 // In the meantime we set NO value for flexforms - this is mainly because file references
2154 // inside will not be processed properly; In fact references will point to no file
2155 // or existing files (in which case there will be double-references which is a big problem of course!)
2156 $this->import_data[$table][$ID][$field] = '';
2157 break;
2158 }
2159 }
2160 } elseif ($table . ':' . $uid != 'pages:0') {
2161 // On root level we don't want this error message.
2162 $this->error('Error: no record was found in data array!', 1);
2163 }
2164 }
2165
2166 /**
2167 * Registers the substNEWids in memory.
2168 *
2169 * @param array $substNEWwithIDs From tcemain to be merged into internal mapping variable in this object
2170 * @return void
2171 * @see writeRecords()
2172 */
2173 public function addToMapId($substNEWwithIDs) {
2174 foreach ($this->import_data as $table => $recs) {
2175 foreach ($recs as $id => $value) {
2176 $old_uid = $this->import_newId[$table . ':' . $id]['uid'];
2177 if (isset($substNEWwithIDs[$id])) {
2178 $this->import_mapId[$table][$old_uid] = $substNEWwithIDs[$id];
2179 } elseif ($this->update) {
2180 // Map same ID to same ID....
2181 $this->import_mapId[$table][$old_uid] = $id;
2182 } else {
2183 // if $this->import_mapId contains already the right mapping, skip the error msg.
2184 // See special handling of sys_file_metadata in addSingle() => nothing to do
2185 if (!($table === 'sys_file_metadata' && isset($this->import_mapId[$table][$old_uid]) && $this->import_mapId[$table][$old_uid] == $id)) {
2186 $this->error('Possible error: ' . $table . ':' . $old_uid . ' had no new id assigned to it. This indicates that the record was not added to database during import. Please check changelog!', 1);
2187 }
2188
2189 }
2190 }
2191 }
2192 }
2193
2194 /**
2195 * Returns a new $TCE object
2196 *
2197 * @return DataHandler $TCE object
2198 */
2199 public function getNewTCE() {
2200 $tce = GeneralUtility::makeInstance(\TYPO3\CMS\Core\DataHandling\DataHandler::class);
2201 $tce->stripslashes_values = 0;
2202 $tce->dontProcessTransformations = 1;
2203 $tce->enableLogging = $this->enableLogging;
2204 $tce->alternativeFileName = $this->alternativeFileName;
2205 $tce->alternativeFilePath = $this->alternativeFilePath;
2206 return $tce;
2207 }
2208
2209 /**
2210 * Cleaning up all the temporary files stored in typo3temp/ folder
2211 *
2212 * @return void
2213 */
2214 public function unlinkTempFiles() {
2215 foreach ($this->unlinkFiles as $fileName) {
2216 if (GeneralUtility::isFirstPartOfStr($fileName, PATH_site . 'typo3temp/')) {
2217 GeneralUtility::unlink_tempfile($fileName);
2218 clearstatcache();
2219 if (is_file($fileName)) {
2220 $this->error('Error: ' . $fileName . ' was NOT unlinked as it should have been!', 1);
2221 }
2222 } else {
2223 $this->error('Error: ' . $fileName . ' was not in temp-path. Not removed!', 1);
2224 }
2225 }
2226 $this->unlinkFiles = array();
2227 }
2228
2229 /***************************
2230 * Import / Relations setting
2231 ***************************/
2232
2233 /**
2234 * At the end of the import process all file and DB relations should be set properly (that is relations
2235 * to imported records are all re-created so imported records are correctly related again)
2236 * Relations in flexform fields are processed in setFlexFormRelations() after this function
2237 *
2238 * @return void
2239 * @see setFlexFormRelations()
2240 */
2241 public function setRelations() {
2242 $updateData = array();
2243 // import_newId contains a register of all records that was in the import memorys "records" key
2244 foreach ($this->import_newId as $nId => $dat) {
2245 $table = $dat['table'];
2246 $uid = $dat['uid'];
2247 // original UID - NOT the new one!
2248 // If the record has been written and received a new id, then proceed:
2249 if (is_array($this->import_mapId[$table]) && isset($this->import_mapId[$table][$uid])) {
2250 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
2251 if (is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {
2252 $thisNewPageUid = 0;
2253 if ($this->legacyImport) {
2254 if ($table != 'pages') {
2255 $oldPid = $this->dat['records'][$table . ':' . $uid]['data']['pid'];
2256 $thisNewPageUid = BackendUtility::wsMapId($table, $this->import_mapId['pages'][$oldPid]);
2257 } else {
2258 $thisNewPageUid = $thisNewUid;
2259 }
2260 }
2261 // Traverse relation fields of each record
2262 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
2263 // uid_local of sys_file_reference needs no update because the correct reference uid was already written
2264 // @see ImportExport::fixUidLocalInSysFileReferenceRecords()
2265 if ($table === 'sys_file_reference' && $field === 'uid_local') {
2266 continue;
2267 }
2268 switch ((string)$config['type']) {
2269 case 'db':
2270 if (is_array($config['itemArray']) && count($config['itemArray'])) {
2271 $itemConfig = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
2272 $valArray = $this->setRelations_db($config['itemArray'], $itemConfig);
2273 $updateData[$table][$thisNewUid][$field] = implode(',', $valArray);
2274 }
2275 break;
2276 case 'file':
2277 if (is_array($config['newValueFiles']) && count($config['newValueFiles'])) {
2278 $valArr = array();
2279 foreach ($config['newValueFiles'] as $fI) {
2280 $valArr[] = $this->import_addFileNameToBeCopied($fI);
2281 }
2282 if ($this->legacyImport && $this->legacyImportFolder === NULL && isset($this->legacyImportMigrationTables[$table][$field])) {
2283 // Do nothing - the legacy import folder is missing
2284 } elseif ($this->legacyImport && $this->legacyImportFolder !== NULL && isset($this->legacyImportMigrationTables[$table][$field])) {
2285 $refIds = array();
2286 foreach ($valArr as $tempFile) {
2287 $fileName = $this->alternativeFileName[$tempFile];
2288 $fileObject = NULL;
2289
2290 try {
2291 // check, if there is alreay the same file in the folder
2292 if ($this->legacyImportFolder->hasFile($fileName)) {
2293 $fileStorage = $this->legacyImportFolder->getStorage();
2294 $file = $fileStorage->getFile($this->legacyImportFolder->getIdentifier() . $fileName);
2295 if ($file->getSha1() === sha1_file($tempFile)) {
2296 $fileObject = $file;
2297 }
2298 }
2299 } catch (Exception $e) {}
2300
2301 if ($fileObject === NULL) {
2302 try {
2303 $fileObject = $this->legacyImportFolder->addFile($tempFile, $fileName, 'changeName');
2304 } catch (\TYPO3\CMS\Core\Exception $e) {
2305 $this->error('Error: no file could be added to the storage for file name' . $this->alternativeFileName[$tempFile]);
2306 }
2307 }
2308 if ($fileObject !== NULL) {
2309 $refId = uniqid('NEW', TRUE);
2310 $refIds[] = $refId;
2311 $updateData['sys_file_reference'][$refId] = array(
2312 'uid_local' => $fileObject->getUid(),
2313 'uid_foreign' => $thisNewUid, // uid of your content record
2314 'tablenames' => $table,
2315 'fieldname' => $field,
2316 'pid' => $thisNewPageUid, // parent id of the parent page
2317 'table_local' => 'sys_file',
2318 );
2319 }
2320 }
2321 $updateData[$table][$thisNewUid][$field] = implode(',', $refIds);
2322 if (!empty($this->legacyImportMigrationTables[$table][$field])) {
2323 $this->legacyImportMigrationRecords[$table][$thisNewUid][$field] = $refIds;
2324 }
2325 } else {
2326 $updateData[$table][$thisNewUid][$field] = implode(',', $valArr);
2327 }
2328 }
2329 break;
2330 }
2331 }
2332 } else {
2333 $this->error('Error: no record was found in data array!', 1);
2334 }
2335 } else {
2336 $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')', 1);
2337 }
2338 }
2339 if (count($updateData)) {
2340 $tce = $this->getNewTCE();
2341 $tce->isImporting = TRUE;
2342 $this->callHook('before_setRelation', array(
2343 'tce' => &$tce,
2344 'data' => &$updateData
2345 ));
2346 $tce->start($updateData, array());
2347 $tce->process_datamap();
2348 // Replace the temporary "NEW" ids with the final ones.
2349 foreach ($this->legacyImportMigrationRecords as $table => $records) {
2350 foreach ($records as $uid => $fields) {
2351 foreach ($fields as $field => $referenceIds) {
2352 foreach ($referenceIds as $key => $referenceId) {
2353 $this->legacyImportMigrationRecords[$table][$uid][$field][$key] = $tce->substNEWwithIDs[$referenceId];
2354 }
2355 }
2356 }
2357 }
2358 $this->callHook('after_setRelations', array(
2359 'tce' => &$tce
2360 ));
2361 }
2362 }
2363
2364 /**
2365 * Maps relations for database
2366 *
2367 * @param array $itemArray Array of item sets (table/uid) from a dbAnalysis object
2368 * @param array $itemConfig Array of TCA config of the field the relation to be set on
2369 * @return array Array with values [table]_[uid] or [uid] for field of type group / internal_type file_reference. These values have the regular tcemain-input group/select type which means they will automatically be processed into a uid-list or MM relations.
2370 */
2371 public function setRelations_db($itemArray, $itemConfig) {
2372 $valArray = array();
2373 foreach ($itemArray as $relDat) {
2374 if (is_array($this->import_mapId[$relDat['table']]) && isset($this->import_mapId[$relDat['table']][$relDat['id']])) {
2375 // Since non FAL file relation type group internal_type file_reference are handled as reference to
2376 // sys_file records Datahandler requires the value as uid of the the related sys_file record only
2377 if ($itemConfig['type'] === 'group' && $itemConfig['internal_type'] === 'file_reference') {
2378 $value = $this->import_mapId[$relDat['table']][$relDat['id']];
2379 } elseif ($itemConfig['type'] === 'input' && isset($itemConfig['wizards']['link'])) {
2380 // If an input field has a relation to a sys_file record this need to be converted back to
2381 // the public path. But use getPublicUrl here, because could normally only be a local file path.
2382 $fileUid = $this->import_mapId[$relDat['table']][$relDat['id']];
2383 // Fallback value
2384 $value = 'file:' . $fileUid;
2385 try {
2386 $file = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->retrieveFileOrFolderObject($fileUid);
2387 } catch (\Exception $e) {}
2388 if ($file instanceof \TYPO3\CMS\Core\Resource\FileInterface) {
2389 $value = $file->getPublicUrl();
2390 }
2391 } else {
2392 $value = $relDat['table'] . '_' . $this->import_mapId[$relDat['table']][$relDat['id']];
2393 }
2394 $valArray[] = $value;
2395 } elseif ($this->isTableStatic($relDat['table']) || $this->isExcluded($relDat['table'], $relDat['id']) || $relDat['id'] < 0) {
2396 // Checking for less than zero because some select types could contain negative values,
2397 // eg. fe_groups (-1, -2) and sys_language (-1 = ALL languages). This must be handled on both export and import.
2398 $valArray[] = $relDat['table'] . '_' . $relDat['id'];
2399 } else {
2400 $this->error('Lost relation: ' . $relDat['table'] . ':' . $relDat['id'], 1);
2401 }
2402 }
2403 return $valArray;
2404 }
2405
2406 /**
2407 * Writes the file from import array to temp dir and returns the filename of it.
2408 *
2409 * @param array $fI File information with three keys: "filename" = filename without path, "ID_absFile" = absolute filepath to the file (including the filename), "ID" = md5 hash of "ID_absFile
2410 * @return string|NULL Absolute filename of the temporary filename of the file. In ->alternativeFileName the original name is set.
2411 */
2412 public function import_addFileNameToBeCopied($fI) {
2413 if (is_array($this->dat['files'][$fI['ID']])) {
2414 $tmpFile = NULL;
2415 // check if there is the right file already in the local folder
2416 if ($this->filesPathForImport !== NULL) {
2417 if (is_file($this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5']) &&
2418 md5_file($this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5']) === $this->dat['files'][$fI['ID']]['content_md5']) {
2419 $tmpFile = $this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5'];
2420 }
2421 }
2422 if ($tmpFile === NULL) {
2423 $tmpFile = GeneralUtility::tempnam('import_temp_');
2424 GeneralUtility::writeFile($tmpFile, $this->dat['files'][$fI['ID']]['content']);
2425 }
2426 clearstatcache();
2427 if (@is_file($tmpFile)) {
2428 $this->unlinkFiles[] = $tmpFile;
2429 if (filesize($tmpFile) == $this->dat['files'][$fI['ID']]['filesize']) {
2430 $this->alternativeFileName[$tmpFile] = $fI['filename'];
2431 $this->alternativeFilePath[$tmpFile] = $this->dat['files'][$fI['ID']]['relFileRef'];
2432 return $tmpFile;
2433 } else {
2434 $this->error('Error: temporary file ' . $tmpFile . ' had a size (' . filesize($tmpFile) . ') different from the original (' . $this->dat['files'][$fI['ID']]['filesize'] . ')', 1);
2435 }
2436 } else {
2437 $this->error('Error: temporary file ' . $tmpFile . ' was not written as it should have been!', 1);
2438 }
2439 } else {
2440 $this->error('Error: No file found for ID ' . $fI['ID'], 1);
2441 }
2442 return NULL;
2443 }
2444
2445 /**
2446 * After all DB relations has been set in the end of the import (see setRelations()) then it is time to correct all relations inside of FlexForm fields.
2447 * The reason for doing this after is that the setting of relations may affect (quite often!) which data structure is used for the flexforms field!
2448 *
2449 * @return void
2450 * @see setRelations()
2451 */
2452 public function setFlexFormRelations() {
2453 $updateData = array();
2454 // import_newId contains a register of all records that was in the import memorys "records" key
2455 foreach ($this->import_newId as $nId => $dat) {
2456 $table = $dat['table'];
2457 $uid = $dat['uid'];
2458 // original UID - NOT the new one!
2459 // If the record has been written and received a new id, then proceed:
2460 if (!isset($this->import_mapId[$table][$uid])) {
2461 $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')', 1);
2462 continue;
2463 }
2464
2465 if (!is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {
2466 $this->error('Error: no record was found in data array!', 1);
2467 continue;
2468 }
2469 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
2470 // Traverse relation fields of each record
2471 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
2472 switch ((string)$config['type']) {
2473 case 'flex':
2474 // Get XML content and set as default value (string, non-processed):
2475 $updateData[$table][$thisNewUid][$field] = $this->dat['records'][$table . ':' . $uid]['data'][$field];
2476 // If there has been registered relations inside the flex form field, run processing on the content:
2477 if (count($config['flexFormRels']['db']) || count($config['flexFormRels']['file'])) {
2478 $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');
2479 // This will fetch the new row for the element (which should be updated with any references to data structures etc.)
2480 $conf = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
2481 if (is_array($origRecordRow) && is_array($conf) && $conf['type'] === 'flex') {
2482 // Get current data structure and value array:
2483 $dataStructArray = BackendUtility::getFlexFormDS($conf, $origRecordRow, $table, $field);
2484 $currentValueArray = GeneralUtility::xml2array($updateData[$table][$thisNewUid][$field]);
2485 // Do recursive processing of the XML data:
2486 $iteratorObj = GeneralUtility::makeInstance(\TYPO3\CMS\Core\DataHandling\DataHandler::class);
2487 $iteratorObj->callBackObj = $this;
2488 $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData(
2489 $currentValueArray['data'],
2490 array(),
2491 array(),
2492 $dataStructArray,
2493 array($table, $thisNewUid, $field, $config),
2494 'remapListedDBRecords_flexFormCallBack'
2495 );
2496 // The return value is set as an array which means it will be processed by tcemain for file and DB references!
2497 if (is_array($currentValueArray['data'])) {
2498 $updateData[$table][$thisNewUid][$field] = $currentValueArray;
2499 }
2500 }
2501 }
2502 break;
2503 }
2504 }
2505 }
2506 if (count($updateData)) {
2507 $tce = $this->getNewTCE();
2508 $tce->isImporting = TRUE;
2509 $this->callHook('before_setFlexFormRelations', array(
2510 'tce' => &$tce,
2511 'data' => &$updateData
2512 ));
2513 $tce->start($updateData, array());
2514 $tce->process_datamap();
2515 $this->callHook('after_setFlexFormRelations', array(
2516 'tce' => &$tce
2517 ));
2518 }
2519 }
2520
2521 /**
2522 * Callback function for traversing the FlexForm structure in relation to remapping database relations
2523 *
2524 * @param array $pParams Set of parameters in numeric array: table, uid, field
2525 * @param array $dsConf TCA config for field (from Data Structure of course)
2526 * @param string $dataValue Field value (from FlexForm XML)
2527 * @param string $dataValue_ext1 Not used
2528 * @param string $dataValue_ext2 Not used
2529 * @param string $path Path of where the data structure of the element is found
2530 * @return array Array where the "value" key carries the value.
2531 * @see setFlexFormRelations()
2532 */
2533 public function remapListedDBRecords_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path) {
2534 // Extract parameters:
2535 list($table, $uid, $field, $config) = $pParams;
2536 // In case the $path is used as index without a trailing slash we will remove that
2537 if (!is_array($config['flexFormRels']['db'][$path]) && is_array($config['flexFormRels']['db'][rtrim($path, '/')])) {
2538 $path = rtrim($path, '/');
2539 }
2540 if (is_array($config['flexFormRels']['db'][$path])) {
2541 $valArray = $this->setRelations_db($config['flexFormRels']['db'][$path], $dsConf);
2542 $dataValue = implode(',', $valArray);
2543 }
2544 if (is_array($config['flexFormRels']['file'][$path])) {
2545 $valArr = array();
2546 foreach ($config['flexFormRels']['file'][$path] as $fI) {
2547 $valArr[] = $this->import_addFileNameToBeCopied($fI);
2548 }
2549 $dataValue = implode(',', $valArr);
2550 }
2551 return array('value' => $dataValue);
2552 }
2553
2554 /**************************
2555 * Import / Soft References
2556 *************************/
2557
2558 /**
2559 * Processing of soft references
2560 *
2561 * @return void
2562 */
2563 public function processSoftReferences() {
2564 // Initialize:
2565 $inData = array();
2566 // Traverse records:
2567 if (is_array($this->dat['header']['records'])) {
2568 foreach ($this->dat['header']['records'] as $table => $recs) {
2569 foreach ($recs as $uid => $thisRec) {
2570 // If there are soft references defined, traverse those:
2571 if (isset($GLOBALS['TCA'][$table]) && is_array($thisRec['softrefs'])) {
2572 // First traversal is to collect softref configuration and split them up based on fields.
2573 // This could probably also have been done with the "records" key instead of the header.
2574 $fieldsIndex = array();
2575 foreach ($thisRec['softrefs'] as $softrefDef) {
2576 // If a substitution token is set:
2577 if ($softrefDef['field'] && is_array($softrefDef['subst']) && $softrefDef['subst']['tokenID']) {
2578 $fieldsIndex[$softrefDef['field']][$softrefDef['subst']['tokenID']] = $softrefDef;
2579 }
2580 }
2581 // The new id:
2582 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
2583 // Now, if there are any fields that require substitution to be done, lets go for that:
2584 foreach ($fieldsIndex as $field => $softRefCfgs) {
2585 if (is_array($GLOBALS['TCA'][$table]['columns'][$field])) {
2586 $conf = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
2587 if ($conf['type'] === 'flex') {
2588 // This will fetch the new row for the element (which should be updated with any references to data structures etc.)
2589 $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');
2590 if (is_array($origRecordRow)) {
2591 // Get current data structure and value array:
2592 $dataStructArray = BackendUtility::getFlexFormDS($conf, $origRecordRow, $table, $field);
2593 $currentValueArray = GeneralUtility::xml2array($origRecordRow[$field]);
2594 // Do recursive processing of the XML data:
2595 /** @var $iteratorObj \TYPO3\CMS\Core\DataHandling\DataHandler */
2596 $iteratorObj = GeneralUtility::makeInstance(\TYPO3\CMS\Core\DataHandling\DataHandler::class);
2597 $iteratorObj->callBackObj = $this;
2598 $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData($currentValueArray['data'], array(), array(), $dataStructArray, array($table, $uid, $field, $softRefCfgs), 'processSoftReferences_flexFormCallBack');
2599 // The return value is set as an array which means it will be processed by tcemain for file and DB references!
2600 if (is_array($currentValueArray['data'])) {
2601 $inData[$table][$thisNewUid][$field] = $currentValueArray;
2602 }
2603 }
2604 } else {
2605 // Get tokenizedContent string and proceed only if that is not blank:
2606 $tokenizedContent = $this->dat['records'][$table . ':' . $uid]['rels'][$field]['softrefs']['tokenizedContent'];
2607 if (strlen($tokenizedContent) && is_array($softRefCfgs)) {
2608 $inData[$table][$thisNewUid][$field] = $this->processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid);
2609 }
2610 }
2611 }
2612 }
2613 }
2614 }
2615 }
2616 }
2617 // Now write to database:
2618 $tce = $this->getNewTCE();
2619 $tce->isImporting = TRUE;
2620 $this->callHook('before_processSoftReferences', array(
2621 'tce' => $tce,
2622 'data' => &$inData
2623 ));
2624 $tce->enableLogging = TRUE;
2625 $tce->start($inData, array());
2626 $tce->process_datamap();
2627 $this->callHook('after_processSoftReferences', array(
2628 'tce' => $tce
2629 ));
2630 }
2631
2632 /**
2633 * Callback function for traversing the FlexForm structure in relation to remapping softreference relations
2634 *
2635 * @param array $pParams Set of parameters in numeric array: table, uid, field
2636 * @param array $dsConf TCA config for field (from Data Structure of course)
2637 * @param string $dataValue Field value (from FlexForm XML)
2638 * @param string $dataValue_ext1 Not used
2639 * @param string $dataValue_ext2 Not used
2640 * @param string $path Path of where the data structure where the element is found
2641 * @return array Array where the "value" key carries the value.
2642 * @see setFlexFormRelations()
2643 */
2644 public function processSoftReferences_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path) {
2645 // Extract parameters:
2646 list($table, $origUid, $field, $softRefCfgs) = $pParams;
2647 if (is_array($softRefCfgs)) {
2648 // First, find all soft reference configurations for this structure path (they are listed flat in the header):
2649 $thisSoftRefCfgList = array();
2650 foreach ($softRefCfgs as $sK => $sV) {
2651 if ($sV['structurePath'] === $path) {
2652 $thisSoftRefCfgList[$sK] = $sV;
2653 }
2654 }
2655 // If any was found, do processing:
2656 if (count($thisSoftRefCfgList)) {
2657 // Get tokenizedContent string and proceed only if that is not blank:
2658 $tokenizedContent = $this->dat['records'][$table . ':' . $origUid]['rels'][$field]['flexFormRels']['softrefs'][$path]['tokenizedContent'];
2659 if (strlen($tokenizedContent)) {
2660 $dataValue = $this->processSoftReferences_substTokens($tokenizedContent, $thisSoftRefCfgList, $table, $origUid);
2661 }
2662 }
2663 }
2664 // Return
2665 return array('value' => $dataValue);
2666 }
2667
2668 /**
2669 * Substition of softreference tokens
2670 *
2671 * @param string $tokenizedContent Content of field with soft reference tokens in.
2672 * @param array $softRefCfgs Soft reference configurations
2673 * @param string $table Table for which the processing occurs
2674 * @param string $uid UID of record from table
2675 * @return string The input content with tokens substituted according to entries in softRefCfgs
2676 */
2677 public function processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid) {
2678 // traverse each softref type for this field:
2679 foreach ($softRefCfgs as $cfg) {
2680 // Get token ID:
2681 $tokenID = $cfg['subst']['tokenID'];
2682 // Default is current token value:
2683 $insertValue = $cfg['subst']['tokenValue'];
2684 // Based on mode:
2685 switch ((string)$this->softrefCfg[$tokenID]['mode']) {
2686 case 'exclude':
2687 // Exclude is a simple passthrough of the value
2688 break;
2689 case 'editable':
2690 // Editable always picks up the value from this input array:
2691 $insertValue = $this->softrefInputValues[$tokenID];
2692 break;
2693 default:
2694 // Mapping IDs/creating files: Based on type, look up new value:
2695 switch ((string)$cfg['subst']['type']) {
2696 case 'file':
2697 // Create / Overwrite file:
2698 $insertValue = $this->processSoftReferences_saveFile($cfg['subst']['relFileName'], $cfg, $table, $uid);
2699 break;
2700 case 'db':
2701 default:
2702 // Trying to map database element if found in the mapID array:
2703 list($tempTable, $tempUid) = explode(':', $cfg['subst']['recordRef']);
2704 if (isset($this->import_mapId[$tempTable][$tempUid])) {
2705 $insertValue = BackendUtility::wsMapId($tempTable, $this->import_mapId[$tempTable][$tempUid]);
2706 // Look if reference is to a page and the original token value was NOT an integer - then we assume is was an alias and try to look up the new one!
2707 if ($tempTable === 'pages' && !\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($cfg['subst']['tokenValue'])) {
2708 $recWithUniqueValue = BackendUtility::getRecord($tempTable, $insertValue, 'alias');
2709 if ($recWithUniqueValue['alias']) {
2710 $insertValue = $recWithUniqueValue['alias'];
2711 }
2712 } elseif (strpos($cfg['subst']['tokenValue'], ':') !== FALSE) {
2713 list($tokenKey, $tokenId) = explode(':', $cfg['subst']['tokenValue']);
2714 $insertValue = $tokenKey . ':' . $insertValue;
2715 }
2716 }
2717 }
2718 }
2719 // Finally, swap the soft reference token in tokenized content with the insert value:
2720 $tokenizedContent = str_replace('{softref:' . $tokenID . '}', $insertValue, $tokenizedContent);
2721 }
2722 return $tokenizedContent;
2723 }
2724
2725 /**
2726 * Process a soft reference file
2727 *
2728 * @param string $relFileName Old Relative filename
2729 * @param array $cfg soft reference configuration array
2730 * @param string $table Table for which the processing occurs
2731 * @param string $uid UID of record from table
2732 * @return string New relative filename (value to insert instead of the softref token)
2733 */
2734 public function processSoftReferences_saveFile($relFileName, $cfg, $table, $uid) {
2735 if ($fileHeaderInfo = $this->dat['header']['files'][$cfg['file_ID']]) {
2736 // Initialize; Get directory prefix for file and find possible RTE filename
2737 $dirPrefix = PathUtility::dirname($relFileName) . '/';
2738 $rteOrigName = $this->getRTEoriginalFilename(PathUtility::basename($relFileName));
2739 // If filename looks like an RTE file, and the directory is in "uploads/", then process as a RTE file!
2740 if ($rteOrigName && GeneralUtility::isFirstPartOfStr($dirPrefix, 'uploads/')) {
2741 // RTE:
2742 // First, find unique RTE file name:
2743 if (@is_dir((PATH_site . $dirPrefix))) {
2744 // From the "original" RTE filename, produce a new "original" destination filename which is unused.
2745 // Even if updated, the image should be unique. Currently the problem with this is that it leaves a lot of unused RTE images...
2746 $fileProcObj = $this->getFileProcObj();
2747 $origDestName = $fileProcObj->getUniqueName($rteOrigName, PATH_site . $dirPrefix);
2748 // Create copy file name:
2749 $pI = pathinfo($relFileName);
2750 $copyDestName = PathUtility::dirname($origDestName) . '/RTEmagicC_' . substr(PathUtility::basename($origDestName), 10) . '.' . $pI['extension'];
2751 if (
2752 !@is_file($copyDestName) && !@is_file($origDestName)
2753 && $origDestName === GeneralUtility::getFileAbsFileName($origDestName)
2754 && $copyDestName === GeneralUtility::getFileAbsFileName($copyDestName)
2755 ) {
2756 if ($this->dat['header']['files'][$fileHeaderInfo['RTE_ORIG_ID']]) {
2757 if ($this->legacyImport) {
2758 $fileName = PathUtility::basename($copyDestName);
2759 $this->writeSysFileResourceForLegacyImport($fileName, $cfg['file_ID']);
2760 $relFileName = $this->filePathMap[$cfg['file_ID']] . '" data-htmlarea-file-uid="' . $fileName . '" data-htmlarea-file-table="sys_file';
2761 // Also save the original file
2762 $originalFileName = PathUtility::basename($origDestName);
2763 $this->writeSysFileResourceForLegacyImport($originalFileName, $fileHeaderInfo['RTE_ORIG_ID']);
2764 } else {
2765 // Write the copy and original RTE file to the respective filenames:
2766 $this->writeFileVerify($copyDestName, $cfg['file_ID'], TRUE);
2767 $this->writeFileVerify($origDestName, $fileHeaderInfo['RTE_ORIG_ID'], TRUE);
2768 // Return the relative path of the copy file name:
2769 return PathUtility::stripPathSitePrefix($copyDestName);
2770 }
2771 } else {
2772 $this->error('ERROR: Could not find original file ID');
2773 }
2774 } else {
2775 $this->error('ERROR: The destination filenames "' . $copyDestName . '" and "' . $origDestName . '" either existed or have non-valid names');
2776 }
2777 } else {
2778 $this->error('ERROR: "' . PATH_site . $dirPrefix . '" was not a directory, so could not process file "' . $relFileName . '"');
2779 }
2780 } elseif (GeneralUtility::isFirstPartOfStr($dirPrefix, $this->fileadminFolderName . '/')) {
2781 // File in fileadmin/ folder:
2782 // Create file (and possible resources)
2783 $newFileName = $this->processSoftReferences_saveFile_createRelFile($dirPrefix, PathUtility::basename($relFileName), $cfg['file_ID'], $table, $uid);
2784 if (strlen($newFileName)) {
2785 $relFileName = $newFileName;
2786 } else {
2787 $this->error('ERROR: No new file created for "' . $relFileName . '"');
2788 }
2789 } else {
2790 $this->error('ERROR: Sorry, cannot operate on non-RTE files which are outside the fileadmin folder.');
2791 }
2792 } else {
2793 $this->error('ERROR: Could not find file ID in header.');
2794 }
2795 // Return (new) filename relative to PATH_site:
2796 return $relFileName;
2797 }
2798
2799 /**
2800 * Create file in directory and return the new (unique) filename
2801 *
2802 * @param string $origDirPrefix Directory prefix, relative, with trailing slash
2803 * @param string $fileName Filename (without path)
2804 * @param string $fileID File ID from import memory
2805 * @param string $table Table for which the processing occurs
2806 * @param string $uid UID of record from table
2807 * @return string|NULL New relative filename, if any
2808 */
2809 public function processSoftReferences_saveFile_createRelFile($origDirPrefix, $fileName, $fileID, $table, $uid) {
2810 // If the fileID map contains an entry for this fileID then just return the relative filename of that entry;
2811 // we don't want to write another unique filename for this one!
2812