fabf0c6665b37cae23073ab9a4bfd2333f65d598
[Packages/TYPO3.CMS.git] / typo3 / sysext / impexp / Classes / ImportExport.php
1 <?php
2 namespace TYPO3\CMS\Impexp;
3
4 /*
5 * This file is part of the TYPO3 CMS project.
6 *
7 * It is free software; you can redistribute it and/or modify it under
8 * the terms of the GNU General Public License, either version 2
9 * of the License, or any later version.
10 *
11 * For the full copyright and license information, please read the
12 * LICENSE.txt file that was distributed with this source code.
13 *
14 * The TYPO3 project - inspiring people to share!
15 */
16
17 use TYPO3\CMS\Backend\Utility\BackendUtility;
18 use TYPO3\CMS\Core\Exception;
19 use TYPO3\CMS\Core\Resource\ResourceFactory;
20 use TYPO3\CMS\Core\Utility\GeneralUtility;
21 use TYPO3\CMS\Core\Utility\PathUtility;
22
23 /**
24 * EXAMPLE for using the impexp-class for exporting stuff:
25 *
26 * Create and initialize:
27 * $this->export = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Impexp\ImportExport::class);
28 * $this->export->init();
29 * Set which tables relations we will allow:
30 * $this->export->relOnlyTables[]="tt_news"; // exclusively includes. See comment in the class
31 *
32 * Adding records:
33 * $this->export->export_addRecord("pages", $this->pageinfo);
34 * $this->export->export_addRecord("pages", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("pages", 38));
35 * $this->export->export_addRecord("pages", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("pages", 39));
36 * $this->export->export_addRecord("tt_content", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("tt_content", 12));
37 * $this->export->export_addRecord("tt_content", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("tt_content", 74));
38 * $this->export->export_addRecord("sys_template", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("sys_template", 20));
39 *
40 * Adding all the relations (recursively in 5 levels so relations has THEIR relations registered as well)
41 * for($a=0;$a<5;$a++) {
42 * $addR = $this->export->export_addDBRelations($a);
43 * if (!count($addR)) break;
44 * }
45 *
46 * Finally load all the files.
47 * $this->export->export_addFilesFromRelations(); // MUST be after the DBrelations are set so that file from ALL added records are included!
48 *
49 * Write export
50 * $out = $this->export->compileMemoryToFileContent();
51 */
52
53 /**
54 * T3D file Import/Export library (TYPO3 Record Document)
55 *
56 * @author Kasper Skårhøj <kasperYYYY@typo3.com>
57 */
58 class ImportExport {
59
60 /**
61 * If set, static relations (not exported) will be shown in overview as well
62 *
63 * @var bool
64 */
65 public $showStaticRelations = FALSE;
66
67 /**
68 * Name of the "fileadmin" folder where files for export/import should be located
69 *
70 * @var string
71 */
72 public $fileadminFolderName = '';
73
74 /**
75 * Whether "import" or "export" mode of object. Set through init() function
76 *
77 * @var string
78 */
79 public $mode = '';
80
81 /**
82 * Updates all records that has same UID instead of creating new!
83 *
84 * @var bool
85 */
86 public $update = FALSE;
87
88 /**
89 * Is set by importData() when an import has been done.
90 *
91 * @var bool
92 */
93 public $doesImport = FALSE;
94
95 /**
96 * If set to a page-record, then the preview display of the content will expect this page-record to be the target
97 * for the import and accordingly display validation information. This triggers the visual view of the
98 * import/export memory to validate if import is possible
99 *
100 * @var array
101 */
102 public $display_import_pid_record = array();
103
104 /**
105 * Used to register the forged UID values for imported records that we want
106 * to create with the same UIDs as in the import file. Admin-only feature.
107 *
108 * @var array
109 */
110 public $suggestedInsertUids = array();
111
112 /**
113 * Setting import modes during update state: as_new, exclude, force_uid
114 *
115 * @var array
116 */
117 public $import_mode = array();
118
119 /**
120 * If set, PID correct is ignored globally
121 *
122 * @var bool
123 */
124 public $global_ignore_pid = FALSE;
125
126 /**
127 * If set, all UID values are forced! (update or import)
128 *
129 * @var bool
130 */
131 public $force_all_UIDS = FALSE;
132
133 /**
134 * If set, a diff-view column is added to the overview.
135 *
136 * @var bool
137 */
138 public $showDiff = FALSE;
139
140 /**
141 * If set, and if the user is admin, allow the writing of PHP scripts to fileadmin/ area.
142 *
143 * @var bool
144 */
145 public $allowPHPScripts = FALSE;
146
147 /**
148 * Disable logging when importing
149 *
150 * @var bool
151 */
152 public $enableLogging = FALSE;
153
154 /**
155 * Array of values to substitute in editable softreferences.
156 *
157 * @var array
158 */
159 public $softrefInputValues = array();
160
161 /**
162 * Mapping between the fileID from import memory and the final filenames they are written to.
163 *
164 * @var array
165 */
166 public $fileIDMap = array();
167
168 /**
169 * 1MB max file size
170 *
171 * @var int
172 */
173 public $maxFileSize = 1000000;
174
175 /**
176 * 1MB max record size
177 *
178 * @var int
179 */
180 public $maxRecordSize = 1000000;
181
182 /**
183 * 10MB max export size
184 *
185 * @var int
186 */
187 public $maxExportSize = 10000000;
188
189 /**
190 * Add table names here which are THE ONLY ones which will be included
191 * into export if found as relations. '_ALL' will allow all tables.
192 *
193 * @var array
194 */
195 public $relOnlyTables = array();
196
197 /**
198 * Add tables names here which should not be exported with the file.
199 * (Where relations should be mapped to same UIDs in target system).
200 *
201 * @var array
202 */
203 public $relStaticTables = array();
204
205 /**
206 * Exclude map. Keys are table:uid pairs and if set, records are not added to the export.
207 *
208 * @var array
209 */
210 public $excludeMap = array();
211
212 /**
213 * Soft Reference Token ID modes.
214 *
215 * @var array
216 */
217 public $softrefCfg = array();
218
219 /**
220 * Listing extension dependencies.
221 *
222 * @var array
223 */
224 public $extensionDependencies = array();
225
226 /**
227 * Set by user: If set, compression in t3d files is disabled
228 *
229 * @var bool
230 */
231 public $dontCompress = FALSE;
232
233 /**
234 * If set, HTML file resources are included.
235 *
236 * @var bool
237 */
238 public $includeExtFileResources = FALSE;
239
240 /**
241 * Files with external media (HTML/css style references inside)
242 *
243 * @var string
244 */
245 public $extFileResourceExtensions = 'html,htm,css';
246
247 /**
248 * After records are written this array is filled with [table][original_uid] = [new_uid]
249 *
250 * @var array
251 */
252 public $import_mapId = array();
253
254 /**
255 * Keys are [tablename]:[new NEWxxx ids (or when updating it is uids)]
256 * while values are arrays with table/uid of the original record it is based on.
257 * With the array keys the new ids can be looked up inside tcemain
258 *
259 * @var array
260 */
261 public $import_newId = array();
262
263 /**
264 * Page id map for page tree (import)
265 *
266 * @var array
267 */
268 public $import_newId_pids = array();
269
270 /**
271 * Internal data accumulation for writing records during import
272 *
273 * @var array
274 */
275 public $import_data = array();
276
277 /**
278 * Error log.
279 *
280 * @var array
281 */
282 public $errorLog = array();
283
284 /**
285 * Cache for record paths
286 *
287 * @var array
288 */
289 public $cache_getRecordPath = array();
290
291 /**
292 * Cache of checkPID values.
293 *
294 * @var array
295 */
296 public $checkPID_cache = array();
297
298 /**
299 * Set internally if the gzcompress function exists
300 * Used by ImportExportController
301 *
302 * @var bool
303 */
304 public $compress = FALSE;
305
306 /**
307 * Internal import/export memory
308 *
309 * @var array
310 */
311 public $dat = array();
312
313 /**
314 * File processing object
315 *
316 * @var \TYPO3\CMS\Core\Utility\File\ExtendedFileUtility
317 */
318 protected $fileProcObj = NULL;
319
320 /**
321 * Keys are [recordname], values are an array of fields to be included
322 * in the export
323 *
324 * @var array
325 */
326 protected $recordTypesIncludeFields = array();
327
328 /**
329 * Default array of fields to be included in the export
330 *
331 * @var array
332 */
333 protected $defaultRecordIncludeFields = array('uid', 'pid');
334
335 /**
336 * Array of current registered storage objects
337 *
338 * @var array
339 */
340 protected $storageObjects = array();
341
342 /**
343 * Is set, if the import file has a TYPO3 version below 6.0
344 *
345 * @var bool
346 */
347 protected $legacyImport = FALSE;
348
349 /**
350 * @var \TYPO3\CMS\Core\Resource\Folder
351 */
352 protected $legacyImportFolder = NULL;
353
354 /**
355 * Related to the default storage root
356 *
357 * @var string
358 */
359 protected $legacyImportTargetPath = '_imported/';
360
361 /**
362 * Table fields to migrate
363 *
364 * @var array
365 */
366 protected $legacyImportMigrationTables = array(
367 'tt_content' => array(
368 'image' => array(
369 'titleTexts' => 'titleText',
370 'description' => 'imagecaption',
371 'links' => 'image_link',
372 'alternativeTexts' => 'altText'
373 ),
374 'media' => array(
375 'description' => 'imagecaption',
376 )
377 ),
378 'pages' => array(
379 'media' => array()
380 ),
381 'pages_language_overlay' => array(
382 'media' => array()
383 )
384 );
385
386
387 /**
388 * Records to be migrated after all
389 * Multidimensional array [table][uid][field] = array([related sys_file_reference uids])
390 *
391 * @var array
392 */
393 protected $legacyImportMigrationRecords = array();
394
395 /**
396 * @var bool
397 */
398 protected $saveFilesOutsideExportFile = FALSE;
399
400 /**
401 * @var NULL|string
402 */
403 protected $temporaryFilesPathForExport = NULL;
404
405 /**
406 * @var NULL|string
407 */
408 protected $filesPathForImport = NULL;
409
410 /**
411 * @var array
412 */
413 protected $unlinkFiles = array();
414
415 /**
416 * @var array
417 */
418 protected $alternativeFileName = array();
419
420 /**
421 * @var array
422 */
423 protected $alternativeFilePath = array();
424
425 /**
426 * @var array
427 */
428 protected $filePathMap = array();
429
430 /**
431 * @var array
432 */
433 protected $remainHeader = array();
434
435 /**************************
436 * Initialize
437 *************************/
438
439 /**
440 * Init the object, both import and export
441 *
442 * @param bool $dontCompress If set, compression of t3d files is disabled
443 * @param string $mode Mode of usage, either "import" or "export
444 * @return void
445 */
446 public function init($dontCompress = FALSE, $mode = '') {
447 $this->compress = function_exists('gzcompress');
448 $this->dontCompress = $dontCompress;
449 $this->mode = $mode;
450 $this->fileadminFolderName = !empty($GLOBALS['TYPO3_CONF_VARS']['BE']['fileadminDir']) ? rtrim($GLOBALS['TYPO3_CONF_VARS']['BE']['fileadminDir'], '/') : 'fileadmin';
451 }
452
453 /**************************
454 * Export / Init + Meta Data
455 *************************/
456
457 /**
458 * Set header basics
459 *
460 * @return void
461 */
462 public function setHeaderBasics() {
463 // Initializing:
464 if (is_array($this->softrefCfg)) {
465 foreach ($this->softrefCfg as $key => $value) {
466 if (!strlen($value['mode'])) {
467 unset($this->softrefCfg[$key]);
468 }
469 }
470 }
471 // Setting in header memory:
472 // Version of file format
473 $this->dat['header']['XMLversion'] = '1.0';
474 // Initialize meta data array (to put it in top of file)
475 $this->dat['header']['meta'] = array();
476 // Add list of tables to consider static
477 $this->dat['header']['relStaticTables'] = $this->relStaticTables;
478 // The list of excluded records
479 $this->dat['header']['excludeMap'] = $this->excludeMap;
480 // Soft Reference mode for elements
481 $this->dat['header']['softrefCfg'] = $this->softrefCfg;
482 // List of extensions the import depends on.
483 $this->dat['header']['extensionDependencies'] = $this->extensionDependencies;
484 }
485
486 /**
487 * Set charset
488 *
489 * @param string $charset Charset for the content in the export. During import the character set will be converted if the target system uses another charset.
490 * @return void
491 */
492 public function setCharset($charset) {
493 $this->dat['header']['charset'] = $charset;
494 }
495
496 /**
497 * Sets meta data
498 *
499 * @param string $title Title of the export
500 * @param string $description Description of the export
501 * @param string $notes Notes about the contents
502 * @param string $packager_username Backend Username of the packager (the guy making the export)
503 * @param string $packager_name Real name of the packager
504 * @param string $packager_email Email of the packager
505 * @return void
506 */
507 public function setMetaData($title, $description, $notes, $packager_username, $packager_name, $packager_email) {
508 $this->dat['header']['meta'] = array(
509 'title' => $title,
510 'description' => $description,
511 'notes' => $notes,
512 'packager_username' => $packager_username,
513 'packager_name' => $packager_name,
514 'packager_email' => $packager_email,
515 'TYPO3_version' => TYPO3_version,
516 'created' => strftime('%A %e. %B %Y', $GLOBALS['EXEC_TIME'])
517 );
518 }
519
520 /**
521 * Option to enable having the files not included in the export file.
522 * The files are saved to a temporary folder instead.
523 *
524 * @param bool $saveFilesOutsideExportFile
525 * @see getTemporaryFilesPathForExport()
526 */
527 public function setSaveFilesOutsideExportFile($saveFilesOutsideExportFile) {
528 $this->saveFilesOutsideExportFile = $saveFilesOutsideExportFile;
529 }
530
531 /**
532 * Sets a thumbnail image to the exported file
533 *
534 * @param string $imgFilepath Filename reference, gif, jpg, png. Absolute path.
535 * @return void
536 */
537 public function addThumbnail($imgFilepath) {
538 if (@is_file($imgFilepath)) {
539 $imgInfo = @getimagesize($imgFilepath);
540 if (is_array($imgInfo)) {
541 $fileContent = GeneralUtility::getUrl($imgFilepath);
542 $this->dat['header']['thumbnail'] = array(
543 'imgInfo' => $imgInfo,
544 'content' => $fileContent,
545 'filesize' => strlen($fileContent),
546 'filemtime' => filemtime($imgFilepath),
547 'filename' => PathUtility::basename($imgFilepath)
548 );
549 }
550 }
551 }
552
553 /**************************
554 * Export / Init Page tree
555 *************************/
556
557 /**
558 * Sets the page-tree array in the export header and returns the array in a flattened version
559 *
560 * @param array $idH Hierarchy of ids, the page tree: array([uid] => array("uid" => [uid], "subrow" => array(.....)), [uid] => ....)
561 * @return array The hierarchical page tree converted to a one-dimensional list of pages
562 */
563 public function setPageTree($idH) {
564 $this->dat['header']['pagetree'] = $this->unsetExcludedSections($idH);
565 return $this->flatInversePageTree($this->dat['header']['pagetree']);
566 }
567
568 /**
569 * Removes entries in the page tree which are found in ->excludeMap[]
570 *
571 * @param array $idH Page uid hierarchy
572 * @return array Modified input array
573 * @access private
574 * @see setPageTree()
575 */
576 public function unsetExcludedSections($idH) {
577 if (is_array($idH)) {
578 foreach ($idH as $k => $v) {
579 if ($this->excludeMap['pages:' . $idH[$k]['uid']]) {
580 unset($idH[$k]);
581 } elseif (is_array($idH[$k]['subrow'])) {
582 $idH[$k]['subrow'] = $this->unsetExcludedSections($idH[$k]['subrow']);
583 }
584 }
585 }
586 return $idH;
587 }
588
589 /**
590 * Recursively flattening the idH array (for setPageTree() function)
591 *
592 * @param array $idH Page uid hierarchy
593 * @param array $a Accumulation array of pages (internal, don't set from outside)
594 * @return array Array with uid-uid pairs for all pages in the page tree.
595 * @see flatInversePageTree_pid()
596 */
597 public function flatInversePageTree($idH, $a = array()) {
598 if (is_array($idH)) {
599 $idH = array_reverse($idH);
600 foreach ($idH as $k => $v) {
601 $a[$v['uid']] = $v['uid'];
602 if (is_array($v['subrow'])) {
603 $a = $this->flatInversePageTree($v['subrow'], $a);
604 }
605 }
606 }
607 return $a;
608 }
609
610 /**
611 * Recursively flattening the idH array (for setPageTree() function), setting PIDs as values
612 *
613 * @param array $idH Page uid hierarchy
614 * @param array $a Accumulation array of pages (internal, don't set from outside)
615 * @param int $pid PID value (internal)
616 * @return array Array with uid-pid pairs for all pages in the page tree.
617 * @see flatInversePageTree()
618 */
619 public function flatInversePageTree_pid($idH, $a = array(), $pid = -1) {
620 if (is_array($idH)) {
621 $idH = array_reverse($idH);
622 foreach ($idH as $v) {
623 $a[$v['uid']] = $pid;
624 if (is_array($v['subrow'])) {
625 $a = $this->flatInversePageTree_pid($v['subrow'], $a, $v['uid']);
626 }
627 }
628 }
629 return $a;
630 }
631
632 /**************************
633 * Export
634 *************************/
635
636 /**
637 * Sets the fields of record types to be included in the export
638 *
639 * @param array $recordTypesIncludeFields Keys are [recordname], values are an array of fields to be included in the export
640 * @throws \TYPO3\CMS\Core\Exception if an array value is not type of array
641 * @return void
642 */
643 public function setRecordTypesIncludeFields(array $recordTypesIncludeFields) {
644 foreach ($recordTypesIncludeFields as $table => $fields) {
645 if (!is_array($fields)) {
646 throw new \TYPO3\CMS\Core\Exception('The include fields for record type ' . htmlspecialchars($table) . ' are not defined by an array.', 1391440658);
647 }
648 $this->setRecordTypeIncludeFields($table, $fields);
649 }
650 }
651
652 /**
653 * Sets the fields of a record type to be included in the export
654 *
655 * @param string $table The record type
656 * @param array $fields The fields to be included
657 * @return void
658 */
659 public function setRecordTypeIncludeFields($table, array $fields) {
660 $this->recordTypesIncludeFields[$table] = $fields;
661 }
662
663 /**
664 * Adds the record $row from $table.
665 * No checking for relations done here. Pure data.
666 *
667 * @param string $table Table name
668 * @param array $row Record row.
669 * @param int $relationLevel (Internal) if the record is added as a relation, this is set to the "level" it was on.
670 * @return void
671 */
672 public function export_addRecord($table, $row, $relationLevel = 0) {
673 BackendUtility::workspaceOL($table, $row);
674 if ((string)$table !== '' && is_array($row) && $row['uid'] > 0 && !$this->excludeMap[($table . ':' . $row['uid'])]) {
675 if ($this->checkPID($table === 'pages' ? $row['uid'] : $row['pid'])) {
676 if (!isset($this->dat['records'][($table . ':' . $row['uid'])])) {
677 // Prepare header info:
678 $row = $this->filterRecordFields($table, $row);
679 $headerInfo = array();
680 $headerInfo['uid'] = $row['uid'];
681 $headerInfo['pid'] = $row['pid'];
682 $headerInfo['title'] = GeneralUtility::fixed_lgd_cs(BackendUtility::getRecordTitle($table, $row), 40);
683 $headerInfo['size'] = strlen(serialize($row));
684 if ($relationLevel) {
685 $headerInfo['relationLevel'] = $relationLevel;
686 }
687 // If record content is not too large in size, set the header content and add the rest:
688 if ($headerInfo['size'] < $this->maxRecordSize) {
689 // Set the header summary:
690 $this->dat['header']['records'][$table][$row['uid']] = $headerInfo;
691 // Create entry in the PID lookup:
692 $this->dat['header']['pid_lookup'][$row['pid']][$table][$row['uid']] = 1;
693 // Initialize reference index object:
694 $refIndexObj = GeneralUtility::makeInstance(\TYPO3\CMS\Core\Database\ReferenceIndex::class);
695 // Yes to workspace overlays for exporting....
696 $refIndexObj->WSOL = TRUE;
697 $relations = $refIndexObj->getRelations($table, $row);
698 $relations = $this->fixFileIDsInRelations($relations);
699 // Data:
700 $this->dat['records'][$table . ':' . $row['uid']] = array();
701 $this->dat['records'][$table . ':' . $row['uid']]['data'] = $row;
702 $this->dat['records'][$table . ':' . $row['uid']]['rels'] = $relations;
703 $this->errorLog = array_merge($this->errorLog, $refIndexObj->errorLog);
704 // Merge error logs.
705 // Add information about the relations in the record in the header:
706 $this->dat['header']['records'][$table][$row['uid']]['rels'] = $this->flatDBrels($this->dat['records'][$table . ':' . $row['uid']]['rels']);
707 // Add information about the softrefs to header:
708 $this->dat['header']['records'][$table][$row['uid']]['softrefs'] = $this->flatSoftRefs($this->dat['records'][$table . ':' . $row['uid']]['rels']);
709 } else {
710 $this->error('Record ' . $table . ':' . $row['uid'] . ' was larger than maxRecordSize (' . GeneralUtility::formatSize($this->maxRecordSize) . ')');
711 }
712 } else {
713 $this->error('Record ' . $table . ':' . $row['uid'] . ' already added.');
714 }
715 } else {
716 $this->error('Record ' . $table . ':' . $row['uid'] . ' was outside your DB mounts!');
717 }
718 }
719 }
720
721 /**
722 * This changes the file reference ID from a hash based on the absolute file path
723 * (coming from ReferenceIndex) to a hash based on the relative file path.
724 *
725 * @param array $relations
726 * @return array
727 */
728 protected function fixFileIDsInRelations(array $relations) {
729 foreach ($relations as $field => $relation) {
730 if (isset($relation['type']) && $relation['type'] === 'file') {
731 foreach ($relation['newValueFiles'] as $key => $fileRelationData) {
732 $absoluteFilePath = $fileRelationData['ID_absFile'];
733 if (GeneralUtility::isFirstPartOfStr($absoluteFilePath, PATH_site)) {
734 $relatedFilePath = PathUtility::stripPathSitePrefix($absoluteFilePath);
735 $relations[$field]['newValueFiles'][$key]['ID'] = md5($relatedFilePath);
736 }
737 }
738 }
739 if ($relation['type'] === 'flex') {
740 if (is_array($relation['flexFormRels']['file'])) {
741 foreach ($relation['flexFormRels']['file'] as $key => $subList) {
742 foreach ($subList as $subKey => $fileRelationData) {
743 $absoluteFilePath = $fileRelationData['ID_absFile'];
744 if (GeneralUtility::isFirstPartOfStr($absoluteFilePath, PATH_site)) {
745 $relatedFilePath = PathUtility::stripPathSitePrefix($absoluteFilePath);
746 $relations[$field]['flexFormRels']['file'][$key][$subKey]['ID'] = md5($relatedFilePath);
747 }
748 }
749 }
750 }
751 }
752 }
753 return $relations;
754 }
755
756
757
758 /**
759 * This analyses the existing added records, finds all database relations to records and adds these records to the export file.
760 * This function can be called repeatedly until it returns an empty array.
761 * In principle it should not allow to infinite recursivity, but you better set a limit...
762 * Call this BEFORE the ext_addFilesFromRelations (so files from added relations are also included of course)
763 *
764 * @param int $relationLevel Recursion level
765 * @return array overview of relations found and added: Keys [table]:[uid], values array with table and id
766 * @see export_addFilesFromRelations()
767 */
768 public function export_addDBRelations($relationLevel = 0) {
769 // Traverse all "rels" registered for "records"
770 if (!is_array($this->dat['records'])) {
771 $this->error('There were no records available.');
772 return array();
773 }
774 $addR = array();
775 foreach ($this->dat['records'] as $k => $value) {
776 if (!is_array($this->dat['records'][$k])) {
777 continue;
778 }
779 foreach ($this->dat['records'][$k]['rels'] as $fieldname => $vR) {
780 // For all DB types of relations:
781 if ($vR['type'] == 'db') {
782 foreach ($vR['itemArray'] as $fI) {
783 $this->export_addDBRelations_registerRelation($fI, $addR);
784 }
785 }
786 // For all flex/db types of relations:
787 if ($vR['type'] == 'flex') {
788 // DB relations in flex form fields:
789 if (is_array($vR['flexFormRels']['db'])) {
790 foreach ($vR['flexFormRels']['db'] as $subList) {
791 foreach ($subList as $fI) {
792 $this->export_addDBRelations_registerRelation($fI, $addR);
793 }
794 }
795 }
796 // DB oriented soft references in flex form fields:
797 if (is_array($vR['flexFormRels']['softrefs'])) {
798 foreach ($vR['flexFormRels']['softrefs'] as $subList) {
799 foreach ($subList['keys'] as $spKey => $elements) {
800 foreach ($elements as $el) {
801 if ($el['subst']['type'] === 'db' && $this->includeSoftref($el['subst']['tokenID'])) {
802 list($tempTable, $tempUid) = explode(':', $el['subst']['recordRef']);
803 $fI = array(
804 'table' => $tempTable,
805 'id' => $tempUid
806 );
807 $this->export_addDBRelations_registerRelation($fI, $addR, $el['subst']['tokenID']);
808 }
809 }
810 }
811 }
812 }
813 }
814 // In any case, if there are soft refs:
815 if (is_array($vR['softrefs']['keys'])) {
816 foreach ($vR['softrefs']['keys'] as $spKey => $elements) {
817 foreach ($elements as $el) {
818 if ($el['subst']['type'] === 'db' && $this->includeSoftref($el['subst']['tokenID'])) {
819 list($tempTable, $tempUid) = explode(':', $el['subst']['recordRef']);
820 $fI = array(
821 'table' => $tempTable,
822 'id' => $tempUid
823 );
824 $this->export_addDBRelations_registerRelation($fI, $addR, $el['subst']['tokenID']);
825 }
826 }
827 }
828 }
829 }
830 }
831
832 // Now, if there were new records to add, do so:
833 if (!empty($addR)) {
834 foreach ($addR as $fI) {
835 // Get and set record:
836 $row = BackendUtility::getRecord($fI['table'], $fI['id']);
837 if (is_array($row)) {
838 $this->export_addRecord($fI['table'], $row, $relationLevel + 1);
839 }
840 // Set status message
841 // Relation pointers always larger than zero except certain "select" types with
842 // negative values pointing to uids - but that is not supported here.
843 if ($fI['id'] > 0) {
844 $rId = $fI['table'] . ':' . $fI['id'];
845 if (!isset($this->dat['records'][$rId])) {
846 $this->dat['records'][$rId] = 'NOT_FOUND';
847 $this->error('Relation record ' . $rId . ' was not found!');
848 }
849 }
850 }
851 }
852 // Return overview of relations found and added
853 return $addR;
854 }
855
856 /**
857 * Helper function for export_addDBRelations()
858 *
859 * @param array $fI Array with table/id keys to add
860 * @param array $addR Add array, passed by reference to be modified
861 * @param string $tokenID Softref Token ID, if applicable.
862 * @return void
863 * @see export_addDBRelations()
864 */
865 public function export_addDBRelations_registerRelation($fI, &$addR, $tokenID = '') {
866 $rId = $fI['table'] . ':' . $fI['id'];
867 if (
868 isset($GLOBALS['TCA'][$fI['table']]) && !$this->isTableStatic($fI['table']) && !$this->isExcluded($fI['table'], $fI['id'])
869 && (!$tokenID || $this->includeSoftref($tokenID)) && $this->inclRelation($fI['table'])
870 ) {
871 if (!isset($this->dat['records'][$rId])) {
872 // Set this record to be included since it is not already.
873 $addR[$rId] = $fI;
874 }
875 }
876 }
877
878 /**
879 * This adds all files in relations.
880 * Call this method AFTER adding all records including relations.
881 *
882 * @return void
883 * @see export_addDBRelations()
884 */
885 public function export_addFilesFromRelations() {
886 // Traverse all "rels" registered for "records"
887 if (!is_array($this->dat['records'])) {
888 $this->error('There were no records available.');
889 return;
890 }
891 foreach ($this->dat['records'] as $k => $value) {
892 if (!isset($this->dat['records'][$k]['rels']) || !is_array($this->dat['records'][$k]['rels'])) {
893 continue;
894 }
895 foreach ($this->dat['records'][$k]['rels'] as $fieldname => $vR) {
896 // For all file type relations:
897 if ($vR['type'] == 'file') {
898 foreach ($vR['newValueFiles'] as $key => $fI) {
899 $this->export_addFile($fI, $k, $fieldname);
900 // Remove the absolute reference to the file so it doesn't expose absolute paths from source server:
901 unset($this->dat['records'][$k]['rels'][$fieldname]['newValueFiles'][$key]['ID_absFile']);
902 }
903 }
904 // For all flex type relations:
905 if ($vR['type'] == 'flex') {
906 if (is_array($vR['flexFormRels']['file'])) {
907 foreach ($vR['flexFormRels']['file'] as $key => $subList) {
908 foreach ($subList as $subKey => $fI) {
909 $this->export_addFile($fI, $k, $fieldname);
910 // Remove the absolute reference to the file so it doesn't expose absolute paths from source server:
911 unset($this->dat['records'][$k]['rels'][$fieldname]['flexFormRels']['file'][$key][$subKey]['ID_absFile']);
912 }
913 }
914 }
915 // DB oriented soft references in flex form fields:
916 if (is_array($vR['flexFormRels']['softrefs'])) {
917 foreach ($vR['flexFormRels']['softrefs'] as $key => $subList) {
918 foreach ($subList['keys'] as $spKey => $elements) {
919 foreach ($elements as $subKey => $el) {
920 if ($el['subst']['type'] === 'file' && $this->includeSoftref($el['subst']['tokenID'])) {
921 // Create abs path and ID for file:
922 $ID_absFile = GeneralUtility::getFileAbsFileName(PATH_site . $el['subst']['relFileName']);
923 $ID = md5($el['subst']['relFileName']);
924 if ($ID_absFile) {
925 if (!$this->dat['files'][$ID]) {
926 $fI = array(
927 'filename' => PathUtility::basename($ID_absFile),
928 'ID_absFile' => $ID_absFile,
929 'ID' => $ID,
930 'relFileName' => $el['subst']['relFileName']
931 );
932 $this->export_addFile($fI, '_SOFTREF_');
933 }
934 $this->dat['records'][$k]['rels'][$fieldname]['flexFormRels']['softrefs'][$key]['keys'][$spKey][$subKey]['file_ID'] = $ID;
935 }
936 }
937 }
938 }
939 }
940 }
941 }
942 // In any case, if there are soft refs:
943 if (is_array($vR['softrefs']['keys'])) {
944 foreach ($vR['softrefs']['keys'] as $spKey => $elements) {
945 foreach ($elements as $subKey => $el) {
946 if ($el['subst']['type'] === 'file' && $this->includeSoftref($el['subst']['tokenID'])) {
947 // Create abs path and ID for file:
948 $ID_absFile = GeneralUtility::getFileAbsFileName(PATH_site . $el['subst']['relFileName']);
949 $ID = md5($el['subst']['relFileName']);
950 if ($ID_absFile) {
951 if (!$this->dat['files'][$ID]) {
952 $fI = array(
953 'filename' => PathUtility::basename($ID_absFile),
954 'ID_absFile' => $ID_absFile,
955 'ID' => $ID,
956 'relFileName' => $el['subst']['relFileName']
957 );
958 $this->export_addFile($fI, '_SOFTREF_');
959 }
960 $this->dat['records'][$k]['rels'][$fieldname]['softrefs']['keys'][$spKey][$subKey]['file_ID'] = $ID;
961 }
962 }
963 }
964 }
965 }
966 }
967 }
968 }
969
970 /**
971 * This adds all files from sys_file records
972 *
973 * @return void
974 */
975 public function export_addFilesFromSysFilesRecords() {
976 if (!isset($this->dat['header']['records']['sys_file']) || !is_array($this->dat['header']['records']['sys_file'])) {
977 return;
978 }
979 foreach ($this->dat['header']['records']['sys_file'] as $sysFileUid => $_) {
980 $recordData = $this->dat['records']['sys_file:' . $sysFileUid]['data'];
981 $file = ResourceFactory::getInstance()->createFileObject($recordData);
982 $this->export_addSysFile($file);
983 }
984 }
985
986 /**
987 * Adds a files content from a sys file record to the export memory
988 *
989 * @param \TYPO3\CMS\Core\Resource\File $file
990 * @return void
991 */
992 public function export_addSysFile(\TYPO3\CMS\Core\Resource\File $file) {
993 if ($file->getProperty('size') >= $this->maxFileSize) {
994 $this->error('File ' . $file->getPublicUrl() . ' was larger (' . GeneralUtility::formatSize($file->getProperty('size')) . ') than the maxFileSize (' . GeneralUtility::formatSize($this->maxFileSize) . ')! Skipping.');
995 return;
996 }
997 $fileContent = '';
998 try {
999 if (!$this->saveFilesOutsideExportFile) {
1000 $fileContent = $file->getContents();
1001 } else {
1002 $file->checkActionPermission('read');
1003 }
1004
1005 } catch (\TYPO3\CMS\Core\Resource\Exception\InsufficientFileAccessPermissionsException $e) {
1006 $this->error('File ' . $file->getPublicUrl() . ': ' . $e->getMessage());
1007 return;
1008 } catch (\TYPO3\CMS\Core\Resource\Exception\IllegalFileExtensionException $e) {
1009 $this->error('File ' . $file->getPublicUrl() . ': ' . $e->getMessage());
1010 return;
1011 }
1012 $fileUid = $file->getUid();
1013 $fileInfo = $file->getStorage()->getFileInfo($file);
1014 // we sadly have to cast it to string here, because the size property is also returning a string
1015 $fileSize = (string)$fileInfo['size'];
1016 if ($fileSize !== $file->getProperty('size')) {
1017 $this->error('File size of ' . $file->getCombinedIdentifier() . ' is not up-to-date in index! File added with current size.');
1018 $this->dat['records']['sys_file:' . $fileUid]['data']['size'] = $fileSize;
1019 }
1020 $fileSha1 = $file->getStorage()->hashFile($file, 'sha1');
1021 if ($fileSha1 !== $file->getProperty('sha1')) {
1022 $this->error('File sha1 hash of ' . $file->getCombinedIdentifier() . ' is not up-to-date in index! File added on current sha1.');
1023 $this->dat['records']['sys_file:' . $fileUid]['data']['sha1'] = $fileSha1;
1024 }
1025
1026 $fileRec = array();
1027 $fileRec['filesize'] = $fileSize;
1028 $fileRec['filename'] = $file->getProperty('name');
1029 $fileRec['filemtime'] = $file->getProperty('modification_date');
1030
1031 // build unique id based on the storage and the file identifier
1032 $fileId = md5($file->getStorage()->getUid() . ':' . $file->getProperty('identifier_hash'));
1033
1034 // Setting this data in the header
1035 $this->dat['header']['files_fal'][$fileId] = $fileRec;
1036
1037 if (!$this->saveFilesOutsideExportFile) {
1038 // ... and finally add the heavy stuff:
1039 $fileRec['content'] = $fileContent;
1040 } else {
1041 GeneralUtility::upload_copy_move($file->getForLocalProcessing(FALSE), $this->getTemporaryFilesPathForExport() . $file->getProperty('sha1'));
1042 }
1043 $fileRec['content_sha1'] = $fileSha1;
1044
1045 $this->dat['files_fal'][$fileId] = $fileRec;
1046 }
1047
1048
1049 /**
1050 * Adds a files content to the export memory
1051 *
1052 * @param array $fI File information with three keys: "filename" = filename without path, "ID_absFile" = absolute filepath to the file (including the filename), "ID" = md5 hash of "ID_absFile". "relFileName" is optional for files attached to records, but mandatory for soft referenced files (since the relFileName determines where such a file should be stored!)
1053 * @param string $recordRef If the file is related to a record, this is the id on the form [table]:[id]. Information purposes only.
1054 * @param string $fieldname If the file is related to a record, this is the field name it was related to. Information purposes only.
1055 * @return void
1056 */
1057 public function export_addFile($fI, $recordRef = '', $fieldname = '') {
1058 if (!@is_file($fI['ID_absFile'])) {
1059 $this->error($fI['ID_absFile'] . ' was not a file! Skipping.');
1060 return;
1061 }
1062 if (filesize($fI['ID_absFile']) >= $this->maxFileSize) {
1063 $this->error($fI['ID_absFile'] . ' was larger (' . GeneralUtility::formatSize(filesize($fI['ID_absFile'])) . ') than the maxFileSize (' . GeneralUtility::formatSize($this->maxFileSize) . ')! Skipping.');
1064 return;
1065 }
1066 $fileInfo = stat($fI['ID_absFile']);
1067 $fileRec = array();
1068 $fileRec['filesize'] = $fileInfo['size'];
1069 $fileRec['filename'] = PathUtility::basename($fI['ID_absFile']);
1070 $fileRec['filemtime'] = $fileInfo['mtime'];
1071 //for internal type file_reference
1072 $fileRec['relFileRef'] = PathUtility::stripPathSitePrefix($fI['ID_absFile']);
1073 if ($recordRef) {
1074 $fileRec['record_ref'] = $recordRef . '/' . $fieldname;
1075 }
1076 if ($fI['relFileName']) {
1077 $fileRec['relFileName'] = $fI['relFileName'];
1078 }
1079 // Setting this data in the header
1080 $this->dat['header']['files'][$fI['ID']] = $fileRec;
1081 // ... and for the recordlisting, why not let us know WHICH relations there was...
1082 if ($recordRef && $recordRef !== '_SOFTREF_') {
1083 $refParts = explode(':', $recordRef, 2);
1084 if (!is_array($this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'])) {
1085 $this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'] = array();
1086 }
1087 $this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'][] = $fI['ID'];
1088 }
1089 $fileMd5 = md5_file($fI['ID_absFile']);
1090 if (!$this->saveFilesOutsideExportFile) {
1091 // ... and finally add the heavy stuff:
1092 $fileRec['content'] = GeneralUtility::getUrl($fI['ID_absFile']);
1093 } else {
1094 GeneralUtility::upload_copy_move($fI['ID_absFile'], $this->getTemporaryFilesPathForExport() . $fileMd5);
1095 }
1096 $fileRec['content_md5'] = $fileMd5;
1097 $this->dat['files'][$fI['ID']] = $fileRec;
1098 // For soft references, do further processing:
1099 if ($recordRef === '_SOFTREF_') {
1100 // RTE files?
1101 if ($RTEoriginal = $this->getRTEoriginalFilename(PathUtility::basename($fI['ID_absFile']))) {
1102 $RTEoriginal_absPath = PathUtility::dirname($fI['ID_absFile']) . '/' . $RTEoriginal;
1103 if (@is_file($RTEoriginal_absPath)) {
1104 $RTEoriginal_ID = md5($RTEoriginal_absPath);
1105 $fileInfo = stat($RTEoriginal_absPath);
1106 $fileRec = array();
1107 $fileRec['filesize'] = $fileInfo['size'];
1108 $fileRec['filename'] = PathUtility::basename($RTEoriginal_absPath);
1109 $fileRec['filemtime'] = $fileInfo['mtime'];
1110 $fileRec['record_ref'] = '_RTE_COPY_ID:' . $fI['ID'];
1111 $this->dat['header']['files'][$fI['ID']]['RTE_ORIG_ID'] = $RTEoriginal_ID;
1112 // Setting this data in the header
1113 $this->dat['header']['files'][$RTEoriginal_ID] = $fileRec;
1114 $fileMd5 = md5_file($RTEoriginal_absPath);
1115 if (!$this->saveFilesOutsideExportFile) {
1116 // ... and finally add the heavy stuff:
1117 $fileRec['content'] = GeneralUtility::getUrl($RTEoriginal_absPath);
1118 } else {
1119 GeneralUtility::upload_copy_move($RTEoriginal_absPath, $this->getTemporaryFilesPathForExport() . $fileMd5);
1120 }
1121 $fileRec['content_md5'] = $fileMd5;
1122 $this->dat['files'][$RTEoriginal_ID] = $fileRec;
1123 } else {
1124 $this->error('RTE original file "' . PathUtility::stripPathSitePrefix($RTEoriginal_absPath) . '" was not found!');
1125 }
1126 }
1127 // Files with external media?
1128 // This is only done with files grabbed by a softreference parser since it is deemed improbable that hard-referenced files should undergo this treatment.
1129 $html_fI = pathinfo(PathUtility::basename($fI['ID_absFile']));
1130 if ($this->includeExtFileResources && GeneralUtility::inList($this->extFileResourceExtensions, strtolower($html_fI['extension']))) {
1131 $uniquePrefix = '###' . md5($GLOBALS['EXEC_TIME']) . '###';
1132 if (strtolower($html_fI['extension']) === 'css') {
1133 $prefixedMedias = explode($uniquePrefix, preg_replace('/(url[[:space:]]*\\([[:space:]]*["\']?)([^"\')]*)(["\']?[[:space:]]*\\))/i', '\\1' . $uniquePrefix . '\\2' . $uniquePrefix . '\\3', $fileRec['content']));
1134 } else {
1135 // html, htm:
1136 $htmlParser = GeneralUtility::makeInstance(\TYPO3\CMS\Core\Html\HtmlParser::class);
1137 $prefixedMedias = explode($uniquePrefix, $htmlParser->prefixResourcePath($uniquePrefix, $fileRec['content'], array(), $uniquePrefix));
1138 }
1139 $htmlResourceCaptured = FALSE;
1140 foreach ($prefixedMedias as $k => $v) {
1141 if ($k % 2) {
1142 $EXTres_absPath = GeneralUtility::resolveBackPath(PathUtility::dirname($fI['ID_absFile']) . '/' . $v);
1143 $EXTres_absPath = GeneralUtility::getFileAbsFileName($EXTres_absPath);
1144 if ($EXTres_absPath && GeneralUtility::isFirstPartOfStr($EXTres_absPath, PATH_site . $this->fileadminFolderName . '/') && @is_file($EXTres_absPath)) {
1145 $htmlResourceCaptured = TRUE;
1146 $EXTres_ID = md5($EXTres_absPath);
1147 $this->dat['header']['files'][$fI['ID']]['EXT_RES_ID'][] = $EXTres_ID;
1148 $prefixedMedias[$k] = '{EXT_RES_ID:' . $EXTres_ID . '}';
1149 // Add file to memory if it is not set already:
1150 if (!isset($this->dat['header']['files'][$EXTres_ID])) {
1151 $fileInfo = stat($EXTres_absPath);
1152 $fileRec = array();
1153 $fileRec['filesize'] = $fileInfo['size'];
1154 $fileRec['filename'] = PathUtility::basename($EXTres_absPath);
1155 $fileRec['filemtime'] = $fileInfo['mtime'];
1156 $fileRec['record_ref'] = '_EXT_PARENT_:' . $fI['ID'];
1157 // Media relative to the HTML file.
1158 $fileRec['parentRelFileName'] = $v;
1159 // Setting this data in the header
1160 $this->dat['header']['files'][$EXTres_ID] = $fileRec;
1161 // ... and finally add the heavy stuff:
1162 $fileRec['content'] = GeneralUtility::getUrl($EXTres_absPath);
1163 $fileRec['content_md5'] = md5($fileRec['content']);
1164 $this->dat['files'][$EXTres_ID] = $fileRec;
1165 }
1166 }
1167 }
1168 }
1169 if ($htmlResourceCaptured) {
1170 $this->dat['files'][$fI['ID']]['tokenizedContent'] = implode('', $prefixedMedias);
1171 }
1172 }
1173 }
1174 }
1175
1176 /**
1177 * If saveFilesOutsideExportFile is enabled, this function returns the path
1178 * where the files referenced in the export are copied to.
1179 *
1180 * @return string
1181 * @throws \RuntimeException
1182 * @see setSaveFilesOutsideExportFile()
1183 */
1184 public function getTemporaryFilesPathForExport() {
1185 if (!$this->saveFilesOutsideExportFile) {
1186 throw new \RuntimeException('You need to set saveFilesOutsideExportFile to TRUE before you want to get the temporary files path for export.', 1401205213);
1187 }
1188 if ($this->temporaryFilesPathForExport === NULL) {
1189 $temporaryFolderName = $this->getTemporaryFolderName();
1190 $this->temporaryFilesPathForExport = $temporaryFolderName . '/';
1191 }
1192 return $this->temporaryFilesPathForExport;
1193 }
1194
1195 /**
1196 *
1197 * @return string
1198 */
1199 protected function getTemporaryFolderName() {
1200 $temporaryPath = PATH_site . 'typo3temp/';
1201 do {
1202 $temporaryFolderName = $temporaryPath . 'export_temp_files_' . mt_rand(1, PHP_INT_MAX);
1203 } while (is_dir($temporaryFolderName));
1204 GeneralUtility::mkdir($temporaryFolderName);
1205 return $temporaryFolderName;
1206 }
1207
1208 /**
1209 * DB relations flattend to 1-dim array.
1210 * The list will be unique, no table/uid combination will appear twice.
1211 *
1212 * @param array $dbrels 2-dim Array of database relations organized by table key
1213 * @return array 1-dim array where entries are table:uid and keys are array with table/id
1214 */
1215 public function flatDBrels($dbrels) {
1216 $list = array();
1217 foreach ($dbrels as $dat) {
1218 if ($dat['type'] == 'db') {
1219 foreach ($dat['itemArray'] as $i) {
1220 $list[$i['table'] . ':' . $i['id']] = $i;
1221 }
1222 }
1223 if ($dat['type'] == 'flex' && is_array($dat['flexFormRels']['db'])) {
1224 foreach ($dat['flexFormRels']['db'] as $subList) {
1225 foreach ($subList as $i) {
1226 $list[$i['table'] . ':' . $i['id']] = $i;
1227 }
1228 }
1229 }
1230 }
1231 return $list;
1232 }
1233
1234 /**
1235 * Soft References flattend to 1-dim array.
1236 *
1237 * @param array $dbrels 2-dim Array of database relations organized by table key
1238 * @return array 1-dim array where entries are arrays with properties of the soft link found and keys are a unique combination of field, spKey, structure path if applicable and token ID
1239 */
1240 public function flatSoftRefs($dbrels) {
1241 $list = array();
1242 foreach ($dbrels as $field => $dat) {
1243 if (is_array($dat['softrefs']['keys'])) {
1244 foreach ($dat['softrefs']['keys'] as $spKey => $elements) {
1245 if (is_array($elements)) {
1246 foreach ($elements as $subKey => $el) {
1247 $lKey = $field . ':' . $spKey . ':' . $subKey;
1248 $list[$lKey] = array_merge(array('field' => $field, 'spKey' => $spKey), $el);
1249 // Add file_ID key to header - slightly "risky" way of doing this because if the calculation
1250 // changes for the same value in $this->records[...] this will not work anymore!
1251 if ($el['subst'] && $el['subst']['relFileName']) {
1252 $list[$lKey]['file_ID'] = md5(PATH_site . $el['subst']['relFileName']);
1253 }
1254 }
1255 }
1256 }
1257 }
1258 if ($dat['type'] == 'flex' && is_array($dat['flexFormRels']['softrefs'])) {
1259 foreach ($dat['flexFormRels']['softrefs'] as $structurePath => $subSoftrefs) {
1260 if (is_array($subSoftrefs['keys'])) {
1261 foreach ($subSoftrefs['keys'] as $spKey => $elements) {
1262 foreach ($elements as $subKey => $el) {
1263 $lKey = $field . ':' . $structurePath . ':' . $spKey . ':' . $subKey;
1264 $list[$lKey] = array_merge(array('field' => $field, 'spKey' => $spKey, 'structurePath' => $structurePath), $el);
1265 // Add file_ID key to header - slightly "risky" way of doing this because if the calculation
1266 // changes for the same value in $this->records[...] this will not work anymore!
1267 if ($el['subst'] && $el['subst']['relFileName']) {
1268 $list[$lKey]['file_ID'] = md5(PATH_site . $el['subst']['relFileName']);
1269 }
1270 }
1271 }
1272 }
1273 }
1274 }
1275 }
1276 return $list;
1277 }
1278
1279 /**
1280 * If include fields for a specific record type are set, the data
1281 * are filtered out with fields are not included in the fields.
1282 *
1283 * @param string $table The record type to be filtered
1284 * @param array $row The data to be filtered
1285 * @return array The filtered record row
1286 */
1287 protected function filterRecordFields($table, array $row) {
1288 if (isset($this->recordTypesIncludeFields[$table])) {
1289 $includeFields = array_unique(array_merge(
1290 $this->recordTypesIncludeFields[$table],
1291 $this->defaultRecordIncludeFields
1292 ));
1293 $newRow = array();
1294 foreach ($row as $key => $value) {
1295 if (in_array($key, $includeFields)) {
1296 $newRow[$key] = $value;
1297 }
1298 }
1299 } else {
1300 $newRow = $row;
1301 }
1302 return $newRow;
1303 }
1304
1305
1306 /**************************
1307 * File Output
1308 *************************/
1309
1310 /**
1311 * This compiles and returns the data content for an exported file
1312 *
1313 * @param string $type Type of output; "xml" gives xml, otherwise serialized array, possibly compressed.
1314 * @return string The output file stream
1315 */
1316 public function compileMemoryToFileContent($type = '') {
1317 if ($type == 'xml') {
1318 $out = $this->createXML();
1319 } else {
1320 $compress = $this->doOutputCompress();
1321 $out = '';
1322 // adding header:
1323 $out .= $this->addFilePart(serialize($this->dat['header']), $compress);
1324 // adding records:
1325 $out .= $this->addFilePart(serialize($this->dat['records']), $compress);
1326 // adding files:
1327 $out .= $this->addFilePart(serialize($this->dat['files']), $compress);
1328 // adding files_fal:
1329 $out .= $this->addFilePart(serialize($this->dat['files_fal']), $compress);
1330 }
1331 return $out;
1332 }
1333
1334 /**
1335 * Creates XML string from input array
1336 *
1337 * @return string XML content
1338 */
1339 public function createXML() {
1340 // Options:
1341 $options = array(
1342 'alt_options' => array(
1343 '/header' => array(
1344 'disableTypeAttrib' => TRUE,
1345 'clearStackPath' => TRUE,
1346 'parentTagMap' => array(
1347 'files' => 'file',
1348 'files_fal' => 'file',
1349 'records' => 'table',
1350 'table' => 'rec',
1351 'rec:rels' => 'relations',
1352 'relations' => 'element',
1353 'filerefs' => 'file',
1354 'pid_lookup' => 'page_contents',
1355 'header:relStaticTables' => 'static_tables',
1356 'static_tables' => 'tablename',
1357 'excludeMap' => 'item',
1358 'softrefCfg' => 'softrefExportMode',
1359 'extensionDependencies' => 'extkey',
1360 'softrefs' => 'softref_element'
1361 ),
1362 'alt_options' => array(
1363 '/pagetree' => array(
1364 'disableTypeAttrib' => TRUE,
1365 'useIndexTagForNum' => 'node',
1366 'parentTagMap' => array(
1367 'node:subrow' => 'node'
1368 )
1369 ),
1370 '/pid_lookup/page_contents' => array(
1371 'disableTypeAttrib' => TRUE,
1372 'parentTagMap' => array(
1373 'page_contents' => 'table'
1374 ),
1375 'grandParentTagMap' => array(
1376 'page_contents/table' => 'item'
1377 )
1378 )
1379 )
1380 ),
1381 '/records' => array(
1382 'disableTypeAttrib' => TRUE,
1383 'parentTagMap' => array(
1384 'records' => 'tablerow',
1385 'tablerow:data' => 'fieldlist',
1386 'tablerow:rels' => 'related',
1387 'related' => 'field',
1388 'field:itemArray' => 'relations',
1389 'field:newValueFiles' => 'filerefs',
1390 'field:flexFormRels' => 'flexform',
1391 'relations' => 'element',
1392 'filerefs' => 'file',
1393 'flexform:db' => 'db_relations',
1394 'flexform:file' => 'file_relations',
1395 'flexform:softrefs' => 'softref_relations',
1396 'softref_relations' => 'structurePath',
1397 'db_relations' => 'path',
1398 'file_relations' => 'path',
1399 'path' => 'element',
1400 'keys' => 'softref_key',
1401 'softref_key' => 'softref_element'
1402 ),
1403 'alt_options' => array(
1404 '/records/tablerow/fieldlist' => array(
1405 'useIndexTagForAssoc' => 'field'
1406 )
1407 )
1408 ),
1409 '/files' => array(
1410 'disableTypeAttrib' => TRUE,
1411 'parentTagMap' => array(
1412 'files' => 'file'
1413 )
1414 ),
1415 '/files_fal' => array(
1416 'disableTypeAttrib' => TRUE,
1417 'parentTagMap' => array(
1418 'files_fal' => 'file'
1419 )
1420 )
1421 )
1422 );
1423 // Creating XML file from $outputArray:
1424 $charset = $this->dat['header']['charset'] ?: 'utf-8';
1425 $XML = '<?xml version="1.0" encoding="' . $charset . '" standalone="yes" ?>' . LF;
1426 $XML .= GeneralUtility::array2xml($this->dat, '', 0, 'T3RecordDocument', 0, $options);
1427 return $XML;
1428 }
1429
1430 /**
1431 * Returns TRUE if the output should be compressed.
1432 *
1433 * @return bool TRUE if compression is possible AND requested.
1434 */
1435 public function doOutputCompress() {
1436 return $this->compress && !$this->dontCompress;
1437 }
1438
1439 /**
1440 * Returns a content part for a filename being build.
1441 *
1442 * @param array $data Data to store in part
1443 * @param bool $compress Compress file?
1444 * @return string Content stream.
1445 */
1446 public function addFilePart($data, $compress = FALSE) {
1447 if ($compress) {
1448 $data = gzcompress($data);
1449 }
1450 return md5($data) . ':' . ($compress ? '1' : '0') . ':' . str_pad(strlen($data), 10, '0', STR_PAD_LEFT) . ':' . $data . ':';
1451 }
1452
1453 /***********************
1454 * Import
1455 ***********************/
1456
1457 /**
1458 * Initialize all settings for the import
1459 *
1460 * @return void
1461 */
1462 protected function initializeImport() {
1463 // Set this flag to indicate that an import is being/has been done.
1464 $this->doesImport = 1;
1465 // Initialize:
1466 // These vars MUST last for the whole section not being cleared. They are used by the method setRelations() which are called at the end of the import session.
1467 $this->import_mapId = array();
1468 $this->import_newId = array();
1469 $this->import_newId_pids = array();
1470 // Temporary files stack initialized:
1471 $this->unlinkFiles = array();
1472 $this->alternativeFileName = array();
1473 $this->alternativeFilePath = array();
1474
1475 $this->initializeStorageObjects();
1476 }
1477
1478 /**
1479 * Initialize the all present storage objects
1480 *
1481 * @return void
1482 */
1483 protected function initializeStorageObjects() {
1484 /** @var $storageRepository \TYPO3\CMS\Core\Resource\StorageRepository */
1485 $storageRepository = GeneralUtility::makeInstance(\TYPO3\CMS\Core\Resource\StorageRepository::class);
1486 $this->storageObjects = $storageRepository->findAll();
1487 }
1488
1489 /**
1490 * Imports the internal data array to $pid.
1491 *
1492 * @param int $pid Page ID in which to import the content
1493 * @return void
1494 */
1495 public function importData($pid) {
1496
1497 $this->initializeImport();
1498
1499 // Write sys_file_storages first
1500 $this->writeSysFileStorageRecords();
1501 // Write sys_file records and write the binary file data
1502 $this->writeSysFileRecords();
1503 // Write records, first pages, then the rest
1504 // Fields with "hard" relations to database, files and flexform fields are kept empty during this run
1505 $this->writeRecords_pages($pid);
1506 $this->writeRecords_records($pid);
1507 // Finally all the file and DB record references must be fixed. This is done after all records have supposedly been written to database:
1508 // $this->import_mapId will indicate two things: 1) that a record WAS written to db and 2) that it has got a new id-number.
1509 $this->setRelations();
1510 // And when all DB relations are in place, we can fix file and DB relations in flexform fields (since data structures often depends on relations to a DS record):
1511 $this->setFlexFormRelations();
1512 // Unlink temporary files:
1513 $this->unlinkTempFiles();
1514 // Finally, traverse all records and process softreferences with substitution attributes.
1515 $this->processSoftReferences();
1516 // After all migrate records using sys_file_reference now
1517 if ($this->legacyImport) {
1518 $this->migrateLegacyImportRecords();
1519 }
1520 }
1521
1522 /**
1523 * Imports the sys_file_storage records from internal data array.
1524 *
1525 * @return void
1526 */
1527 protected function writeSysFileStorageRecords() {
1528 if (!isset($this->dat['header']['records']['sys_file_storage'])) {
1529 return;
1530 }
1531 $sysFileStorageUidsToBeResetToDefaultStorage = array();
1532 foreach ($this->dat['header']['records']['sys_file_storage'] as $sysFileStorageUid => $_) {
1533 $storageRecord = $this->dat['records']['sys_file_storage:' . $sysFileStorageUid]['data'];
1534 // continue with Local, writable and online storage only
1535 if ($storageRecord['driver'] === 'Local' && $storageRecord['is_writable'] && $storageRecord['is_online']) {
1536 $useThisStorageUidInsteadOfTheOneInImport = 0;
1537 /** @var $localStorage \TYPO3\CMS\Core\Resource\ResourceStorage */
1538 foreach ($this->storageObjects as $localStorage) {
1539 // check the available storage for Local, writable and online ones
1540 if ($localStorage->getDriverType() === 'Local' && $localStorage->isWritable() && $localStorage->isOnline()) {
1541 // check if there is already a identical storage present (same pathType and basePath)
1542 $storageRecordConfiguration = ResourceFactory::getInstance()->convertFlexFormDataToConfigurationArray($storageRecord['configuration']);
1543 $localStorageRecordConfiguration = $localStorage->getConfiguration();
1544 if (
1545 $storageRecordConfiguration['pathType'] === $localStorageRecordConfiguration['pathType']
1546 && $storageRecordConfiguration['basePath'] === $localStorageRecordConfiguration['basePath']
1547 ) {
1548 // same storage is already present
1549 $useThisStorageUidInsteadOfTheOneInImport = $localStorage->getUid();
1550 break;
1551 }
1552 }
1553 }
1554 if ($useThisStorageUidInsteadOfTheOneInImport > 0) {
1555 // same storage is already present; map the to be imported one to the present one
1556 $this->import_mapId['sys_file_storage'][$sysFileStorageUid] = $useThisStorageUidInsteadOfTheOneInImport;
1557 } else {
1558 // Local, writable and online storage. Is allowed to be used to later write files in.
1559 $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);
1560 }
1561 } else {
1562 // Storage with non Local drivers could be imported but must not be used to saves files in, because you
1563 // could not be sure, that this is supported. The default storage will be used in this case.
1564 // It could happen that non writable and non online storage will be created as dupes because you could not
1565 // check the detailed configuration options at this point
1566 $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);
1567 $sysFileStorageUidsToBeResetToDefaultStorage[] = $sysFileStorageUid;
1568 }
1569
1570 }
1571
1572 // Importing the added ones
1573 $tce = $this->getNewTCE();
1574 // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.
1575 $tce->reverseOrder = 1;
1576 $tce->isImporting = TRUE;
1577 $tce->start($this->import_data, array());
1578 $tce->process_datamap();
1579 $this->addToMapId($tce->substNEWwithIDs);
1580
1581 $defaultStorageUid = NULL;
1582 // get default storage
1583 $defaultStorage = ResourceFactory::getInstance()->getDefaultStorage();
1584 if ($defaultStorage !== NULL) {
1585 $defaultStorageUid = $defaultStorage->getUid();
1586 }
1587 foreach ($sysFileStorageUidsToBeResetToDefaultStorage as $sysFileStorageUidToBeResetToDefaultStorage) {
1588 $this->import_mapId['sys_file_storage'][$sysFileStorageUidToBeResetToDefaultStorage] = $defaultStorageUid;
1589 }
1590
1591 // unset the sys_file_storage records to prevent a import in writeRecords_records
1592 unset($this->dat['header']['records']['sys_file_storage']);
1593 }
1594
1595 /**
1596 * Imports the sys_file records and the binary files data from internal data array.
1597 *
1598 * @return void
1599 */
1600 protected function writeSysFileRecords() {
1601 if (!isset($this->dat['header']['records']['sys_file'])) {
1602 return;
1603 }
1604 $this->addGeneralErrorsByTable('sys_file');
1605
1606 // fetch fresh storage records from database
1607 $storageRecords = $this->fetchStorageRecords();
1608
1609 $defaultStorage = ResourceFactory::getInstance()->getDefaultStorage();
1610
1611 $sanitizedFolderMappings = array();
1612
1613 foreach ($this->dat['header']['records']['sys_file'] as $sysFileUid => $_) {
1614 $fileRecord = $this->dat['records']['sys_file:' . $sysFileUid]['data'];
1615
1616 $temporaryFile = NULL;
1617 // check if there is the right file already in the local folder
1618 if ($this->filesPathForImport !== NULL) {
1619 if (is_file($this->filesPathForImport . '/' . $fileRecord['sha1']) && sha1_file($this->filesPathForImport . '/' . $fileRecord['sha1']) === $fileRecord['sha1']) {
1620 $temporaryFile = $this->filesPathForImport . '/' . $fileRecord['sha1'];
1621 }
1622 }
1623
1624 // save file to disk
1625 if ($temporaryFile === NULL) {
1626 $fileId = md5($fileRecord['storage'] . ':' . $fileRecord['identifier_hash']);
1627 $temporaryFile = $this->writeTemporaryFileFromData($fileId);
1628 if ($temporaryFile === NULL) {
1629 // error on writing the file. Error message was already added
1630 continue;
1631 }
1632 }
1633
1634 $originalStorageUid = $fileRecord['storage'];
1635 $useStorageFromStorageRecords = FALSE;
1636
1637 // replace storage id, if a alternative one was registered
1638 if (isset($this->import_mapId['sys_file_storage'][$fileRecord['storage']])) {
1639 $fileRecord['storage'] = $this->import_mapId['sys_file_storage'][$fileRecord['storage']];
1640 $useStorageFromStorageRecords = TRUE;
1641 }
1642
1643 if (empty($fileRecord['storage']) && !$this->isFallbackStorage($fileRecord['storage'])) {
1644 // no storage for the file is defined, mostly because of a missing default storage.
1645 $this->error('Error: No storage for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $originalStorageUid . '"');
1646 continue;
1647 }
1648
1649 // using a storage from the local storage is only allowed, if the uid is present in the
1650 // mapping. Only in this case we could be sure, that it's a local, online and writable storage.
1651 if ($useStorageFromStorageRecords && isset($storageRecords[$fileRecord['storage']])) {
1652 /** @var $storage \TYPO3\CMS\Core\Resource\ResourceStorage */
1653 $storage = ResourceFactory::getInstance()->getStorageObject($fileRecord['storage'], $storageRecords[$fileRecord['storage']]);
1654 } elseif ($this->isFallbackStorage($fileRecord['storage'])) {
1655 $storage = ResourceFactory::getInstance()->getStorageObject(0);
1656 } elseif ($defaultStorage !== NULL) {
1657 $storage = $defaultStorage;
1658 } else {
1659 $this->error('Error: No storage available for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1660 continue;
1661 }
1662
1663 $newFile = NULL;
1664
1665 // check, if there is a identical file
1666 try {
1667 if ($storage->hasFile($fileRecord['identifier'])) {
1668 $file = $storage->getFile($fileRecord['identifier']);
1669 if ($file->getSha1() === $fileRecord['sha1']) {
1670 $newFile = $file;
1671 }
1672 }
1673 } catch (Exception $e) {}
1674
1675 if ($newFile === NULL) {
1676
1677 $folderName = PathUtility::dirname(ltrim($fileRecord['identifier'], '/'));
1678 if (in_array($folderName, $sanitizedFolderMappings)) {
1679 $folderName = $sanitizedFolderMappings[$folderName];
1680 }
1681 if (!$storage->hasFolder($folderName)) {
1682 try {
1683 $importFolder = $storage->createFolder($folderName);
1684 if ($importFolder->getIdentifier() !== $folderName && !in_array($folderName, $sanitizedFolderMappings)) {
1685 $sanitizedFolderMappings[$folderName] = $importFolder->getIdentifier();
1686 }
1687 } catch (Exception $e) {
1688 $this->error('Error: Folder could not be created for file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1689 continue;
1690 }
1691 } else {
1692 $importFolder = $storage->getFolder($folderName);
1693 }
1694
1695 try {
1696 /** @var $newFile \TYPO3\CMS\Core\Resource\File */
1697 $newFile = $storage->addFile($temporaryFile, $importFolder, $fileRecord['name']);
1698 } catch (Exception $e) {
1699 $this->error('Error: File could not be added to the storage: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1700 continue;
1701 }
1702
1703 if ($newFile->getSha1() !== $fileRecord['sha1']) {
1704 $this->error('Error: The hash of the written file is not identical to the import data! File could be corrupted! File: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1705 }
1706 }
1707
1708 // save the new uid in the import id map
1709 $this->import_mapId['sys_file'][$fileRecord['uid']] = $newFile->getUid();
1710 $this->fixUidLocalInSysFileReferenceRecords($fileRecord['uid'], $newFile->getUid());
1711
1712 }
1713
1714 // unset the sys_file records to prevent a import in writeRecords_records
1715 unset($this->dat['header']['records']['sys_file']);
1716 }
1717
1718 /**
1719 * Checks if the $storageId is the id of the fallback storage
1720 *
1721 * @param int|string $storageId
1722 * @return bool
1723 */
1724 protected function isFallbackStorage($storageId) {
1725 return $storageId === 0 || $storageId === '0';
1726 }
1727
1728 /**
1729 * Normally the importer works like the following:
1730 * Step 1: import the records with cleared field values of relation fields (see addSingle())
1731 * Step 2: update the records with the right relation ids (see setRelations())
1732 *
1733 * In step 2 the saving fields of type "relation to sys_file_reference" checks the related sys_file_reference
1734 * record (created in step 1) with the FileExtensionFilter for matching file extensions of the related file.
1735 * To make this work correct, the uid_local of sys_file_reference records has to be not empty AND has to
1736 * relate to the correct (imported) sys_file record uid!!!
1737 *
1738 * This is fixed here.
1739 *
1740 * @param int $oldFileUid
1741 * @param int $newFileUid
1742 * @return void
1743 */
1744 protected function fixUidLocalInSysFileReferenceRecords($oldFileUid, $newFileUid) {
1745 if (!isset($this->dat['header']['records']['sys_file_reference'])) {
1746 return;
1747 }
1748
1749 foreach ($this->dat['header']['records']['sys_file_reference'] as $sysFileReferenceUid => $_) {
1750 $fileReferenceRecord = $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'];
1751 if ($fileReferenceRecord['uid_local'] == $oldFileUid) {
1752 $fileReferenceRecord['uid_local'] = $newFileUid;
1753 $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'] = $fileReferenceRecord;
1754 }
1755 }
1756 }
1757
1758 /**
1759 * Initializes the folder for legacy imports as subfolder of backend users default upload folder
1760 *
1761 * @return void
1762 */
1763 protected function initializeLegacyImportFolder() {
1764 /** @var \TYPO3\CMS\Core\Resource\Folder $folder */
1765 $folder = $GLOBALS['BE_USER']->getDefaultUploadFolder();
1766 if ($folder === FALSE) {
1767 $this->error('Error: the backend users default upload folder is missing! No files will be imported!');
1768 }
1769 if (!$folder->hasFolder($this->legacyImportTargetPath)) {
1770 try {
1771 $this->legacyImportFolder = $folder->createFolder($this->legacyImportTargetPath);
1772 } catch (\TYPO3\CMS\Core\Exception $e) {
1773 $this->error('Error: the import folder in the default upload folder could not be created! No files will be imported!');
1774 }
1775 } else {
1776 $this->legacyImportFolder = $folder->getSubFolder($this->legacyImportTargetPath);
1777 }
1778
1779 }
1780
1781 /**
1782 * Fetched fresh storage records from database because the new imported
1783 * ones are not in cached data of the StorageRepository
1784 *
1785 * @return bool|array
1786 */
1787 protected function fetchStorageRecords() {
1788 $whereClause = BackendUtility::BEenableFields('sys_file_storage');
1789 $whereClause .= BackendUtility::deleteClause('sys_file_storage');
1790
1791 $rows = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows(
1792 '*',
1793 'sys_file_storage',
1794 '1=1' . $whereClause,
1795 '',
1796 '',
1797 '',
1798 'uid'
1799 );
1800
1801 return $rows;
1802 }
1803
1804 /**
1805 * Writes the file from import array to temp dir and returns the filename of it.
1806 *
1807 * @param string $fileId
1808 * @param string $dataKey
1809 * @return string Absolute filename of the temporary filename of the file
1810 */
1811 protected function writeTemporaryFileFromData($fileId, $dataKey = 'files_fal') {
1812 $temporaryFilePath = NULL;
1813 if (is_array($this->dat[$dataKey][$fileId])) {
1814 $temporaryFilePathInternal = GeneralUtility::tempnam('import_temp_');
1815 GeneralUtility::writeFile($temporaryFilePathInternal, $this->dat[$dataKey][$fileId]['content']);
1816 clearstatcache();
1817 if (@is_file($temporaryFilePathInternal)) {
1818 $this->unlinkFiles[] = $temporaryFilePathInternal;
1819 if (filesize($temporaryFilePathInternal) == $this->dat[$dataKey][$fileId]['filesize']) {
1820 $temporaryFilePath = $temporaryFilePathInternal;
1821 } else {
1822 $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' had a size (' . filesize($temporaryFilePathInternal) . ') different from the original (' . $this->dat[$dataKey][$fileId]['filesize'] . ')', 1);
1823 }
1824 } else {
1825 $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' was not written as it should have been!', 1);
1826 }
1827 } else {
1828 $this->error('Error: No file found for ID ' . $fileId, 1);
1829 }
1830 return $temporaryFilePath;
1831 }
1832
1833 /**
1834 * Writing pagetree/pages to database:
1835 *
1836 * @param int $pid PID in which to import. If the operation is an update operation, the root of the page tree inside will be moved to this PID unless it is the same as the root page from the import
1837 * @return void
1838 * @see writeRecords_records()
1839 */
1840 public function writeRecords_pages($pid) {
1841 // First, write page structure if any:
1842 if (is_array($this->dat['header']['records']['pages'])) {
1843 $this->addGeneralErrorsByTable('pages');
1844 // $pageRecords is a copy of the pages array in the imported file. Records here are unset one by one when the addSingle function is called.
1845 $pageRecords = $this->dat['header']['records']['pages'];
1846 $this->import_data = array();
1847 // First add page tree if any
1848 if (is_array($this->dat['header']['pagetree'])) {
1849 $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);
1850 foreach ($pagesFromTree as $uid) {
1851 $thisRec = $this->dat['header']['records']['pages'][$uid];
1852 // PID: Set the main $pid, unless a NEW-id is found
1853 $setPid = isset($this->import_newId_pids[$thisRec['pid']]) ? $this->import_newId_pids[$thisRec['pid']] : $pid;
1854 $this->addSingle('pages', $uid, $setPid);
1855 unset($pageRecords[$uid]);
1856 }
1857 }
1858 // Then add all remaining pages not in tree on root level:
1859 if (count($pageRecords)) {
1860 $remainingPageUids = array_keys($pageRecords);
1861 foreach ($remainingPageUids as $pUid) {
1862 $this->addSingle('pages', $pUid, $pid);
1863 }
1864 }
1865 // Now write to database:
1866 $tce = $this->getNewTCE();
1867 $tce->isImporting = TRUE;
1868 $this->callHook('before_writeRecordsPages', array(
1869 'tce' => &$tce,
1870 'data' => &$this->import_data
1871 ));
1872 $tce->suggestedInsertUids = $this->suggestedInsertUids;
1873 $tce->start($this->import_data, array());
1874 $tce->process_datamap();
1875 $this->callHook('after_writeRecordsPages', array(
1876 'tce' => &$tce
1877 ));
1878 // post-processing: Registering new ids (end all tcemain sessions with this)
1879 $this->addToMapId($tce->substNEWwithIDs);
1880 // In case of an update, order pages from the page tree correctly:
1881 if ($this->update && is_array($this->dat['header']['pagetree'])) {
1882 $this->writeRecords_pages_order($pid);
1883 }
1884 }
1885 }
1886
1887 /**
1888 * Organize all updated pages in page tree so they are related like in the import file
1889 * Only used for updates and when $this->dat['header']['pagetree'] is an array.
1890 *
1891 * @param int $pid Page id in which to import
1892 * @return void
1893 * @access private
1894 * @see writeRecords_pages(), writeRecords_records_order()
1895 */
1896 public function writeRecords_pages_order($pid) {
1897 $cmd_data = array();
1898 // Get uid-pid relations and traverse them in order to map to possible new IDs
1899 $pidsFromTree = $this->flatInversePageTree_pid($this->dat['header']['pagetree']);
1900 foreach ($pidsFromTree as $origPid => $newPid) {
1901 if ($newPid >= 0 && $this->dontIgnorePid('pages', $origPid)) {
1902 // If the page had a new id (because it was created) use that instead!
1903 if (substr($this->import_newId_pids[$origPid], 0, 3) === 'NEW') {
1904 if ($this->import_mapId['pages'][$origPid]) {
1905 $mappedPid = $this->import_mapId['pages'][$origPid];
1906 $cmd_data['pages'][$mappedPid]['move'] = $newPid;
1907 }
1908 } else {
1909 $cmd_data['pages'][$origPid]['move'] = $newPid;
1910 }
1911 }
1912 }
1913 // Execute the move commands if any:
1914 if (count($cmd_data)) {
1915 $tce = $this->getNewTCE();
1916 $this->callHook('before_writeRecordsPagesOrder', array(
1917 'tce' => &$tce,
1918 'data' => &$cmd_data
1919 ));
1920 $tce->start(array(), $cmd_data);
1921 $tce->process_cmdmap();
1922 $this->callHook('after_writeRecordsPagesOrder', array(
1923 'tce' => &$tce
1924 ));
1925 }
1926 }
1927
1928 /**
1929 * Write all database records except pages (writtein in writeRecords_pages())
1930 *
1931 * @param int $pid Page id in which to import
1932 * @return void
1933 * @see writeRecords_pages()
1934 */
1935 public function writeRecords_records($pid) {
1936 // Write the rest of the records
1937 $this->import_data = array();
1938 if (is_array($this->dat['header']['records'])) {
1939 foreach ($this->dat['header']['records'] as $table => $recs) {
1940 $this->addGeneralErrorsByTable($table);
1941 if ($table != 'pages') {
1942 foreach ($recs as $uid => $thisRec) {
1943 // PID: Set the main $pid, unless a NEW-id is found
1944 $setPid = isset($this->import_mapId['pages'][$thisRec['pid']]) ? $this->import_mapId['pages'][$thisRec['pid']] : $pid;
1945 if (is_array($GLOBALS['TCA'][$table]) && $GLOBALS['TCA'][$table]['ctrl']['rootLevel']) {
1946 $setPid = 0;
1947 }
1948 // Add record:
1949 $this->addSingle($table, $uid, $setPid);
1950 }
1951 }
1952 }
1953 } else {
1954 $this->error('Error: No records defined in internal data array.');
1955 }
1956 // Now write to database:
1957 $tce = $this->getNewTCE();
1958 $this->callHook('before_writeRecordsRecords', array(
1959 'tce' => &$tce,
1960 'data' => &$this->import_data
1961 ));
1962 $tce->suggestedInsertUids = $this->suggestedInsertUids;
1963 // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.
1964 $tce->reverseOrder = 1;
1965 $tce->isImporting = TRUE;
1966 $tce->start($this->import_data, array());
1967 $tce->process_datamap();
1968 $this->callHook('after_writeRecordsRecords', array(
1969 'tce' => &$tce
1970 ));
1971 // post-processing: Removing files and registering new ids (end all tcemain sessions with this)
1972 $this->addToMapId($tce->substNEWwithIDs);
1973 // In case of an update, order pages from the page tree correctly:
1974 if ($this->update) {
1975 $this->writeRecords_records_order($pid);
1976 }
1977 }
1978
1979 /**
1980 * Organize all updated record to their new positions.
1981 * Only used for updates
1982 *
1983 * @param int $mainPid Main PID into which we import.
1984 * @return void
1985 * @access private
1986 * @see writeRecords_records(), writeRecords_pages_order()
1987 */
1988 public function writeRecords_records_order($mainPid) {
1989 $cmd_data = array();
1990 if (is_array($this->dat['header']['pagetree'])) {
1991 $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);
1992 } else {
1993 $pagesFromTree = array();
1994 }
1995 if (is_array($this->dat['header']['pid_lookup'])) {
1996 foreach ($this->dat['header']['pid_lookup'] as $pid => $recList) {
1997 $newPid = isset($this->import_mapId['pages'][$pid]) ? $this->import_mapId['pages'][$pid] : $mainPid;
1998 if (\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($newPid)) {
1999 foreach ($recList as $tableName => $uidList) {
2000 // If $mainPid===$newPid then we are on root level and we can consider to move pages as well!
2001 // (they will not be in the page tree!)
2002 if (($tableName != 'pages' || !$pagesFromTree[$pid]) && is_array($uidList)) {
2003 $uidList = array_reverse(array_keys($uidList));
2004 foreach ($uidList as $uid) {
2005 if ($this->dontIgnorePid($tableName, $uid)) {
2006 $cmd_data[$tableName][$uid]['move'] = $newPid;
2007 } else {
2008
2009 }
2010 }
2011 }
2012 }
2013 }
2014 }
2015 }
2016 // Execute the move commands if any:
2017 if (count($cmd_data)) {
2018 $tce = $this->getNewTCE();
2019 $this->callHook('before_writeRecordsRecordsOrder', array(
2020 'tce' => &$tce,
2021 'data' => &$cmd_data
2022 ));
2023 $tce->start(array(), $cmd_data);
2024 $tce->process_cmdmap();
2025 $this->callHook('after_writeRecordsRecordsOrder', array(
2026 'tce' => &$tce
2027 ));
2028 }
2029 }
2030
2031 /**
2032 * Adds a single record to the $importData array. Also copies files to tempfolder.
2033 * However all File/DB-references and flexform field contents are set to blank for now!
2034 * That is done with setRelations() later
2035 *
2036 * @param string $table Table name (from import memory)
2037 * @param int $uid Record UID (from import memory)
2038 * @param int $pid Page id
2039 * @return void
2040 * @see writeRecords()
2041 */
2042 public function addSingle($table, $uid, $pid) {
2043 if ($this->import_mode[$table . ':' . $uid] === 'exclude') {
2044 return;
2045 }
2046 $record = $this->dat['records'][$table . ':' . $uid]['data'];
2047 if (is_array($record)) {
2048 if ($this->update && $this->doesRecordExist($table, $uid) && $this->import_mode[$table . ':' . $uid] !== 'as_new') {
2049 $ID = $uid;
2050 } elseif ($table === 'sys_file_metadata' && $record['sys_language_uid'] == '0' && $this->import_mapId['sys_file'][$record['file']]) {
2051 // on adding sys_file records the belonging sys_file_metadata record was also created
2052 // if there is one the record need to be overwritten instead of creating a new one.
2053 $recordInDatabase = $GLOBALS['TYPO3_DB']->exec_SELECTgetSingleRow(
2054 'uid',
2055 'sys_file_metadata',
2056 'file = ' . $this->import_mapId['sys_file'][$record['file']] . ' AND sys_language_uid = 0 AND pid = 0'
2057 );
2058 // if no record could be found, $this->import_mapId['sys_file'][$record['file']] is pointing
2059 // to a file, that was already there, thus a new metadata record should be created
2060 if (is_array($recordInDatabase)) {
2061 $this->import_mapId['sys_file_metadata'][$record['uid']] = $recordInDatabase['uid'];
2062 $ID = $recordInDatabase['uid'];
2063 } else {
2064 $ID = uniqid('NEW', TRUE);
2065 }
2066
2067 } else {
2068 $ID = uniqid('NEW', TRUE);
2069 }
2070 $this->import_newId[$table . ':' . $ID] = array('table' => $table, 'uid' => $uid);
2071 if ($table == 'pages') {
2072 $this->import_newId_pids[$uid] = $ID;
2073 }
2074 // Set main record data:
2075 $this->import_data[$table][$ID] = $record;
2076 $this->import_data[$table][$ID]['tx_impexp_origuid'] = $this->import_data[$table][$ID]['uid'];
2077 // Reset permission data:
2078 if ($table === 'pages') {
2079 // Have to reset the user/group IDs so pages are owned by importing user. Otherwise strange things may happen for non-admins!
2080 unset($this->import_data[$table][$ID]['perms_userid']);
2081 unset($this->import_data[$table][$ID]['perms_groupid']);
2082 }
2083 // PID and UID:
2084 unset($this->import_data[$table][$ID]['uid']);
2085 // Updates:
2086 if (\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($ID)) {
2087 unset($this->import_data[$table][$ID]['pid']);
2088 } else {
2089 // Inserts:
2090 $this->import_data[$table][$ID]['pid'] = $pid;
2091 if (($this->import_mode[$table . ':' . $uid] === 'force_uid' && $this->update || $this->force_all_UIDS) && $GLOBALS['BE_USER']->isAdmin()) {
2092 $this->import_data[$table][$ID]['uid'] = $uid;
2093 $this->suggestedInsertUids[$table . ':' . $uid] = 'DELETE';
2094 }
2095 }
2096 // Setting db/file blank:
2097 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
2098 switch ((string)$config['type']) {
2099 case 'db':
2100
2101 case 'file':
2102 // Fixed later in ->setRelations() [because we need to know ALL newly created IDs before we can map relations!]
2103 // In the meantime we set NO values for relations.
2104 //
2105 // BUT for field uid_local of table sys_file_reference the relation MUST not be cleared here,
2106 // because the value is already the uid of the right imported sys_file record.
2107 // @see fixUidLocalInSysFileReferenceRecords()
2108 // If it's empty or a uid to another record the FileExtensionFilter will throw an exception or
2109 // delete the reference record if the file extension of the related record doesn't match.
2110 if ($table !== 'sys_file_reference' && $field !== 'uid_local') {
2111 $this->import_data[$table][$ID][$field] = '';
2112 }
2113 break;
2114 case 'flex':
2115 // Fixed later in setFlexFormRelations()
2116 // In the meantime we set NO value for flexforms - this is mainly because file references
2117 // inside will not be processed properly; In fact references will point to no file
2118 // or existing files (in which case there will be double-references which is a big problem of course!)
2119 $this->import_data[$table][$ID][$field] = '';
2120 break;
2121 }
2122 }
2123 } elseif ($table . ':' . $uid != 'pages:0') {
2124 // On root level we don't want this error message.
2125 $this->error('Error: no record was found in data array!', 1);
2126 }
2127 }
2128
2129 /**
2130 * Registers the substNEWids in memory.
2131 *
2132 * @param array $substNEWwithIDs From tcemain to be merged into internal mapping variable in this object
2133 * @return void
2134 * @see writeRecords()
2135 */
2136 public function addToMapId($substNEWwithIDs) {
2137 foreach ($this->import_data as $table => $recs) {
2138 foreach ($recs as $id => $value) {
2139 $old_uid = $this->import_newId[$table . ':' . $id]['uid'];
2140 if (isset($substNEWwithIDs[$id])) {
2141 $this->import_mapId[$table][$old_uid] = $substNEWwithIDs[$id];
2142 } elseif ($this->update) {
2143 // Map same ID to same ID....
2144 $this->import_mapId[$table][$old_uid] = $id;
2145 } else {
2146 // if $this->import_mapId contains already the right mapping, skip the error msg.
2147 // See special handling of sys_file_metadata in addSingle() => nothing to do
2148 if (!($table === 'sys_file_metadata' && isset($this->import_mapId[$table][$old_uid]) && $this->import_mapId[$table][$old_uid] == $id)) {
2149 $this->error('Possible error: ' . $table . ':' . $old_uid . ' had no new id assigned to it. This indicates that the record was not added to database during import. Please check changelog!', 1);
2150 }
2151
2152 }
2153 }
2154 }
2155 }
2156
2157 /**
2158 * Returns a new $TCE object
2159 *
2160 * @return object $TCE object
2161 */
2162 public function getNewTCE() {
2163 $tce = GeneralUtility::makeInstance(\TYPO3\CMS\Core\DataHandling\DataHandler::class);
2164 $tce->stripslashes_values = 0;
2165 $tce->dontProcessTransformations = 1;
2166 $tce->enableLogging = $this->enableLogging;
2167 $tce->alternativeFileName = $this->alternativeFileName;
2168 $tce->alternativeFilePath = $this->alternativeFilePath;
2169 return $tce;
2170 }
2171
2172 /**
2173 * Cleaning up all the temporary files stored in typo3temp/ folder
2174 *
2175 * @return void
2176 */
2177 public function unlinkTempFiles() {
2178 foreach ($this->unlinkFiles as $fileName) {
2179 if (GeneralUtility::isFirstPartOfStr($fileName, PATH_site . 'typo3temp/')) {
2180 GeneralUtility::unlink_tempfile($fileName);
2181 clearstatcache();
2182 if (is_file($fileName)) {
2183 $this->error('Error: ' . $fileName . ' was NOT unlinked as it should have been!', 1);
2184 }
2185 } else {
2186 $this->error('Error: ' . $fileName . ' was not in temp-path. Not removed!', 1);
2187 }
2188 }
2189 $this->unlinkFiles = array();
2190 }
2191
2192 /***************************
2193 * Import / Relations setting
2194 ***************************/
2195
2196 /**
2197 * At the end of the import process all file and DB relations should be set properly (that is relations
2198 * to imported records are all re-created so imported records are correctly related again)
2199 * Relations in flexform fields are processed in setFlexFormRelations() after this function
2200 *
2201 * @return void
2202 * @see setFlexFormRelations()
2203 */
2204 public function setRelations() {
2205 $updateData = array();
2206 // import_newId contains a register of all records that was in the import memorys "records" key
2207 foreach ($this->import_newId as $nId => $dat) {
2208 $table = $dat['table'];
2209 $uid = $dat['uid'];
2210 // original UID - NOT the new one!
2211 // If the record has been written and received a new id, then proceed:
2212 if (is_array($this->import_mapId[$table]) && isset($this->import_mapId[$table][$uid])) {
2213 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
2214 if (is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {
2215 $thisNewPageUid = 0;
2216 if ($this->legacyImport) {
2217 if ($table != 'pages') {
2218 $oldPid = $this->dat['records'][$table . ':' . $uid]['data']['pid'];
2219 $thisNewPageUid = BackendUtility::wsMapId($table, $this->import_mapId['pages'][$oldPid]);
2220 } else {
2221 $thisNewPageUid = $thisNewUid;
2222 }
2223 }
2224 // Traverse relation fields of each record
2225 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
2226 // uid_local of sys_file_reference needs no update because the correct reference uid was already written
2227 // @see ImportExport::fixUidLocalInSysFileReferenceRecords()
2228 if ($table === 'sys_file_reference' && $field === 'uid_local') {
2229 continue;
2230 }
2231 switch ((string)$config['type']) {
2232 case 'db':
2233 if (is_array($config['itemArray']) && count($config['itemArray'])) {
2234 $itemConfig = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
2235 $valArray = $this->setRelations_db($config['itemArray'], $itemConfig);
2236 $updateData[$table][$thisNewUid][$field] = implode(',', $valArray);
2237 }
2238 break;
2239 case 'file':
2240 if (is_array($config['newValueFiles']) && count($config['newValueFiles'])) {
2241 $valArr = array();
2242 foreach ($config['newValueFiles'] as $fI) {
2243 $valArr[] = $this->import_addFileNameToBeCopied($fI);
2244 }
2245 if ($this->legacyImport && $this->legacyImportFolder === NULL && isset($this->legacyImportMigrationTables[$table][$field])) {
2246 // Do nothing - the legacy import folder is missing
2247 } elseif ($this->legacyImport && $this->legacyImportFolder !== NULL && isset($this->legacyImportMigrationTables[$table][$field])) {
2248 $refIds = array();
2249 foreach ($valArr as $tempFile) {
2250 $fileName = $this->alternativeFileName[$tempFile];
2251 $fileObject = NULL;
2252
2253 try {
2254 // check, if there is alreay the same file in the folder
2255 if ($this->legacyImportFolder->hasFile($fileName)) {
2256 $fileStorage = $this->legacyImportFolder->getStorage();
2257 $file = $fileStorage->getFile($this->legacyImportFolder->getIdentifier() . $fileName);
2258 if ($file->getSha1() === sha1_file($tempFile)) {
2259 $fileObject = $file;
2260 }
2261 }
2262 } catch (Exception $e) {}
2263
2264 if ($fileObject === NULL) {
2265 try {
2266 $fileObject = $this->legacyImportFolder->addFile($tempFile, $fileName, 'changeName');
2267 } catch (\TYPO3\CMS\Core\Exception $e) {
2268 $this->error('Error: no file could be added to the storage for file name' . $this->alternativeFileName[$tempFile]);
2269 }
2270 }
2271 if ($fileObject !== NULL) {
2272 $refId = uniqid('NEW', TRUE);
2273 $refIds[] = $refId;
2274 $updateData['sys_file_reference'][$refId] = array(
2275 'uid_local' => $fileObject->getUid(),
2276 'uid_foreign' => $thisNewUid, // uid of your content record
2277 'tablenames' => $table,
2278 'fieldname' => $field,
2279 'pid' => $thisNewPageUid, // parent id of the parent page
2280 'table_local' => 'sys_file',
2281 );
2282 }
2283 }
2284 $updateData[$table][$thisNewUid][$field] = implode(',', $refIds);
2285 if (!empty($this->legacyImportMigrationTables[$table][$field])) {
2286 $this->legacyImportMigrationRecords[$table][$thisNewUid][$field] = $refIds;
2287 }
2288 } else {
2289 $updateData[$table][$thisNewUid][$field] = implode(',', $valArr);
2290 }
2291 }
2292 break;
2293 }
2294 }
2295 } else {
2296 $this->error('Error: no record was found in data array!', 1);
2297 }
2298 } else {
2299 $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')', 1);
2300 }
2301 }
2302 if (count($updateData)) {
2303 $tce = $this->getNewTCE();
2304 $tce->isImporting = TRUE;
2305 $this->callHook('before_setRelation', array(
2306 'tce' => &$tce,
2307 'data' => &$updateData
2308 ));
2309 $tce->start($updateData, array());
2310 $tce->process_datamap();
2311 // Replace the temporary "NEW" ids with the final ones.
2312 foreach ($this->legacyImportMigrationRecords as $table => $records) {
2313 foreach ($records as $uid => $fields) {
2314 foreach ($fields as $field => $referenceIds) {
2315 foreach ($referenceIds as $key => $referenceId) {
2316 $this->legacyImportMigrationRecords[$table][$uid][$field][$key] = $tce->substNEWwithIDs[$referenceId];
2317 }
2318 }
2319 }
2320 }
2321 $this->callHook('after_setRelations', array(
2322 'tce' => &$tce
2323 ));
2324 }
2325 }
2326
2327 /**
2328 * Maps relations for database
2329 *
2330 * @param array $itemArray Array of item sets (table/uid) from a dbAnalysis object
2331 * @param array $itemConfig Array of TCA config of the field the relation to be set on
2332 * @return array Array with values [table]_[uid] or [uid] for field of type group / internal_type file_reference. These values have the regular tcemain-input group/select type which means they will automatically be processed into a uid-list or MM relations.
2333 */
2334 public function setRelations_db($itemArray, $itemConfig) {
2335 $valArray = array();
2336 foreach ($itemArray as $relDat) {
2337 if (is_array($this->import_mapId[$relDat['table']]) && isset($this->import_mapId[$relDat['table']][$relDat['id']])) {
2338 // Since non FAL file relation type group internal_type file_reference are handled as reference to
2339 // sys_file records Datahandler requires the value as uid of the the related sys_file record only
2340 if ($itemConfig['type'] === 'group' && $itemConfig['internal_type'] === 'file_reference') {
2341 $value = $this->import_mapId[$relDat['table']][$relDat['id']];
2342 } else {
2343 $value = $relDat['table'] . '_' . $this->import_mapId[$relDat['table']][$relDat['id']];
2344 }
2345 $valArray[] = $value;
2346 } elseif ($this->isTableStatic($relDat['table']) || $this->isExcluded($relDat['table'], $relDat['id']) || $relDat['id'] < 0) {
2347 // Checking for less than zero because some select types could contain negative values,
2348 // eg. fe_groups (-1, -2) and sys_language (-1 = ALL languages). This must be handled on both export and import.
2349 $valArray[] = $relDat['table'] . '_' . $relDat['id'];
2350 } else {
2351 $this->error('Lost relation: ' . $relDat['table'] . ':' . $relDat['id'], 1);
2352 }
2353 }
2354 return $valArray;
2355 }
2356
2357 /**
2358 * Writes the file from import array to temp dir and returns the filename of it.
2359 *
2360 * @param array $fI File information with three keys: "filename" = filename without path, "ID_absFile" = absolute filepath to the file (including the filename), "ID" = md5 hash of "ID_absFile
2361 * @return string|NULL Absolute filename of the temporary filename of the file. In ->alternativeFileName the original name is set.
2362 */
2363 public function import_addFileNameToBeCopied($fI) {
2364 if (is_array($this->dat['files'][$fI['ID']])) {
2365 $tmpFile = NULL;
2366 // check if there is the right file already in the local folder
2367 if ($this->filesPathForImport !== NULL) {
2368 if (is_file($this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5']) &&
2369 md5_file($this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5']) === $this->dat['files'][$fI['ID']]['content_md5']) {
2370 $tmpFile = $this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5'];
2371 }
2372 }
2373 if ($tmpFile === NULL) {
2374 $tmpFile = GeneralUtility::tempnam('import_temp_');
2375 GeneralUtility::writeFile($tmpFile, $this->dat['files'][$fI['ID']]['content']);
2376 }
2377 clearstatcache();
2378 if (@is_file($tmpFile)) {
2379 $this->unlinkFiles[] = $tmpFile;
2380 if (filesize($tmpFile) == $this->dat['files'][$fI['ID']]['filesize']) {
2381 $this->alternativeFileName[$tmpFile] = $fI['filename'];
2382 $this->alternativeFilePath[$tmpFile] = $this->dat['files'][$fI['ID']]['relFileRef'];
2383 return $tmpFile;
2384 } else {
2385 $this->error('Error: temporary file ' . $tmpFile . ' had a size (' . filesize($tmpFile) . ') different from the original (' . $this->dat['files'][$fI['ID']]['filesize'] . ')', 1);
2386 }
2387 } else {
2388 $this->error('Error: temporary file ' . $tmpFile . ' was not written as it should have been!', 1);
2389 }
2390 } else {
2391 $this->error('Error: No file found for ID ' . $fI['ID'], 1);
2392 }
2393 return NULL;
2394 }
2395
2396 /**
2397 * After all DB relations has been set in the end of the import (see setRelations()) then it is time to correct all relations inside of FlexForm fields.
2398 * The reason for doing this after is that the setting of relations may affect (quite often!) which data structure is used for the flexforms field!
2399 *
2400 * @return void
2401 * @see setRelations()
2402 */
2403 public function setFlexFormRelations() {
2404 $updateData = array();
2405 // import_newId contains a register of all records that was in the import memorys "records" key
2406 foreach ($this->import_newId as $nId => $dat) {
2407 $table = $dat['table'];
2408 $uid = $dat['uid'];
2409 // original UID - NOT the new one!
2410 // If the record has been written and received a new id, then proceed:
2411 if (!isset($this->import_mapId[$table][$uid])) {
2412 $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')', 1);
2413 continue;
2414 }
2415
2416 if (!is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {
2417 $this->error('Error: no record was found in data array!', 1);
2418 continue;
2419 }
2420 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
2421 // Traverse relation fields of each record
2422 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
2423 switch ((string)$config['type']) {
2424 case 'flex':
2425 // Get XML content and set as default value (string, non-processed):
2426 $updateData[$table][$thisNewUid][$field] = $this->dat['records'][$table . ':' . $uid]['data'][$field];
2427 // If there has been registered relations inside the flex form field, run processing on the content:
2428 if (count($config['flexFormRels']['db']) || count($config['flexFormRels']['file'])) {
2429 $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');
2430 // This will fetch the new row for the element (which should be updated with any references to data structures etc.)
2431 $conf = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
2432 if (is_array($origRecordRow) && is_array($conf) && $conf['type'] === 'flex') {
2433 // Get current data structure and value array:
2434 $dataStructArray = BackendUtility::getFlexFormDS($conf, $origRecordRow, $table, $field);
2435 $currentValueArray = GeneralUtility::xml2array($updateData[$table][$thisNewUid][$field]);
2436 // Do recursive processing of the XML data:
2437 $iteratorObj = GeneralUtility::makeInstance(\TYPO3\CMS\Core\DataHandling\DataHandler::class);
2438 $iteratorObj->callBackObj = $this;
2439 $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData(
2440 $currentValueArray['data'],
2441 array(),
2442 array(),
2443 $dataStructArray,
2444 array($table, $thisNewUid, $field, $config),
2445 'remapListedDBRecords_flexFormCallBack'
2446 );
2447 // The return value is set as an array which means it will be processed by tcemain for file and DB references!
2448 if (is_array($currentValueArray['data'])) {
2449 $updateData[$table][$thisNewUid][$field] = $currentValueArray;
2450 }
2451 }
2452 }
2453 break;
2454 }
2455 }
2456 }
2457 if (count($updateData)) {
2458 $tce = $this->getNewTCE();
2459 $tce->isImporting = TRUE;
2460 $this->callHook('before_setFlexFormRelations', array(
2461 'tce' => &$tce,
2462 'data' => &$updateData
2463 ));
2464 $tce->start($updateData, array());
2465 $tce->process_datamap();
2466 $this->callHook('after_setFlexFormRelations', array(
2467 'tce' => &$tce
2468 ));
2469 }
2470 }
2471
2472 /**
2473 * Callback function for traversing the FlexForm structure in relation to remapping database relations
2474 *
2475 * @param array $pParams Set of parameters in numeric array: table, uid, field
2476 * @param array $dsConf TCA config for field (from Data Structure of course)
2477 * @param string $dataValue Field value (from FlexForm XML)
2478 * @param string $dataValue_ext1 Not used
2479 * @param string $dataValue_ext2 Not used
2480 * @param string $path Path of where the data structure of the element is found
2481 * @return array Array where the "value" key carries the value.
2482 * @see setFlexFormRelations()
2483 */
2484 public function remapListedDBRecords_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path) {
2485 // Extract parameters:
2486 list($table, $uid, $field, $config) = $pParams;
2487 // In case the $path is used as index without a trailing slash we will remove that
2488 if (!is_array($config['flexFormRels']['db'][$path]) && is_array($config['flexFormRels']['db'][rtrim($path, '/')])) {
2489 $path = rtrim($path, '/');
2490 }
2491 if (is_array($config['flexFormRels']['db'][$path])) {
2492 $valArray = $this->setRelations_db($config['flexFormRels']['db'][$path], $dsConf);
2493 $dataValue = implode(',', $valArray);
2494 }
2495 if (is_array($config['flexFormRels']['file'][$path])) {
2496 $valArr = array();
2497 foreach ($config['flexFormRels']['file'][$path] as $fI) {
2498 $valArr[] = $this->import_addFileNameToBeCopied($fI);
2499 }
2500 $dataValue = implode(',', $valArr);
2501 }
2502 return array('value' => $dataValue);
2503 }
2504
2505 /**************************
2506 * Import / Soft References
2507 *************************/
2508
2509 /**
2510 * Processing of soft references
2511 *
2512 * @return void
2513 */
2514 public function processSoftReferences() {
2515 // Initialize:
2516 $inData = array();
2517 // Traverse records:
2518 if (is_array($this->dat['header']['records'])) {
2519 foreach ($this->dat['header']['records'] as $table => $recs) {
2520 foreach ($recs as $uid => $thisRec) {
2521 // If there are soft references defined, traverse those:
2522 if (isset($GLOBALS['TCA'][$table]) && is_array($thisRec['softrefs'])) {
2523 // First traversal is to collect softref configuration and split them up based on fields.
2524 // This could probably also have been done with the "records" key instead of the header.
2525 $fieldsIndex = array();
2526 foreach ($thisRec['softrefs'] as $softrefDef) {
2527 // If a substitution token is set:
2528 if ($softrefDef['field'] && is_array($softrefDef['subst']) && $softrefDef['subst']['tokenID']) {
2529 $fieldsIndex[$softrefDef['field']][$softrefDef['subst']['tokenID']] = $softrefDef;
2530 }
2531 }
2532 // The new id:
2533 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
2534 // Now, if there are any fields that require substitution to be done, lets go for that:
2535 foreach ($fieldsIndex as $field => $softRefCfgs) {
2536 if (is_array($GLOBALS['TCA'][$table]['columns'][$field])) {
2537 $conf = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
2538 if ($conf['type'] === 'flex') {
2539 // This will fetch the new row for the element (which should be updated with any references to data structures etc.)
2540 $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');
2541 if (is_array($origRecordRow)) {
2542 // Get current data structure and value array:
2543 $dataStructArray = BackendUtility::getFlexFormDS($conf, $origRecordRow, $table, $field);
2544 $currentValueArray = GeneralUtility::xml2array($origRecordRow[$field]);
2545 // Do recursive processing of the XML data:
2546 /** @var $iteratorObj \TYPO3\CMS\Core\DataHandling\DataHandler */
2547 $iteratorObj = GeneralUtility::makeInstance(\TYPO3\CMS\Core\DataHandling\DataHandler::class);
2548 $iteratorObj->callBackObj = $this;
2549 $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData($currentValueArray['data'], array(), array(), $dataStructArray, array($table, $uid, $field, $softRefCfgs), 'processSoftReferences_flexFormCallBack');
2550 // The return value is set as an array which means it will be processed by tcemain for file and DB references!
2551 if (is_array($currentValueArray['data'])) {
2552 $inData[$table][$thisNewUid][$field] = $currentValueArray;
2553 }
2554 }
2555 } else {
2556 // Get tokenizedContent string and proceed only if that is not blank:
2557 $tokenizedContent = $this->dat['records'][$table . ':' . $uid]['rels'][$field]['softrefs']['tokenizedContent'];
2558 if (strlen($tokenizedContent) && is_array($softRefCfgs)) {
2559 $inData[$table][$thisNewUid][$field] = $this->processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid);
2560 }
2561 }
2562 }
2563 }
2564 }
2565 }
2566 }
2567 }
2568 // Now write to database:
2569 $tce = $this->getNewTCE();
2570 $tce->isImporting = TRUE;
2571 $this->callHook('before_processSoftReferences', array(
2572 'tce' => $tce,
2573 'data' => &$inData
2574 ));
2575 $tce->enableLogging = TRUE;
2576 $tce->start($inData, array());
2577 $tce->process_datamap();
2578 $this->callHook('after_processSoftReferences', array(
2579 'tce' => $tce
2580 ));
2581 }
2582
2583 /**
2584 * Callback function for traversing the FlexForm structure in relation to remapping softreference relations
2585 *
2586 * @param array $pParams Set of parameters in numeric array: table, uid, field
2587 * @param array $dsConf TCA config for field (from Data Structure of course)
2588 * @param string $dataValue Field value (from FlexForm XML)
2589 * @param string $dataValue_ext1 Not used
2590 * @param string $dataValue_ext2 Not used
2591 * @param string $path Path of where the data structure where the element is found
2592 * @return array Array where the "value" key carries the value.
2593 * @see setFlexFormRelations()
2594 */
2595 public function processSoftReferences_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path) {
2596 // Extract parameters:
2597 list($table, $origUid, $field, $softRefCfgs) = $pParams;
2598 if (is_array($softRefCfgs)) {
2599 // First, find all soft reference configurations for this structure path (they are listed flat in the header):
2600 $thisSoftRefCfgList = array();
2601 foreach ($softRefCfgs as $sK => $sV) {
2602 if ($sV['structurePath'] === $path) {
2603 $thisSoftRefCfgList[$sK] = $sV;
2604 }
2605 }
2606 // If any was found, do processing:
2607 if (count($thisSoftRefCfgList)) {
2608 // Get tokenizedContent string and proceed only if that is not blank:
2609 $tokenizedContent = $this->dat['records'][$table . ':' . $origUid]['rels'][$field]['flexFormRels']['softrefs'][$path]['tokenizedContent'];
2610 if (strlen($tokenizedContent)) {
2611 $dataValue = $this->processSoftReferences_substTokens($tokenizedContent, $thisSoftRefCfgList, $table, $origUid);
2612 }
2613 }
2614 }
2615 // Return
2616 return array('value' => $dataValue);
2617 }
2618
2619 /**
2620 * Substition of softreference tokens
2621 *
2622 * @param string $tokenizedContent Content of field with soft reference tokens in.
2623 * @param array $softRefCfgs Soft reference configurations
2624 * @param string $table Table for which the processing occurs
2625 * @param string $uid UID of record from table
2626 * @return string The input content with tokens substituted according to entries in softRefCfgs
2627 */
2628 public function processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid) {
2629 // traverse each softref type for this field:
2630 foreach ($softRefCfgs as $cfg) {
2631 // Get token ID:
2632 $tokenID = $cfg['subst']['tokenID'];
2633 // Default is current token value:
2634 $insertValue = $cfg['subst']['tokenValue'];
2635 // Based on mode:
2636 switch ((string)$this->softrefCfg[$tokenID]['mode']) {
2637 case 'exclude':
2638 // Exclude is a simple passthrough of the value
2639 break;
2640 case 'editable':
2641 // Editable always picks up the value from this input array:
2642 $insertValue = $this->softrefInputValues[$tokenID];
2643 break;
2644 default:
2645 // Mapping IDs/creating files: Based on type, look up new value:
2646 switch ((string)$cfg['subst']['type']) {
2647 case 'file':
2648 // Create / Overwrite file:
2649 $insertValue = $this->processSoftReferences_saveFile($cfg['subst']['relFileName'], $cfg, $table, $uid);
2650 break;
2651 case 'db':
2652 default:
2653 // Trying to map database element if found in the mapID array:
2654 list($tempTable, $tempUid) = explode(':', $cfg['subst']['recordRef']);
2655 if (isset($this->import_mapId[$tempTable][$tempUid])) {
2656 $insertValue = BackendUtility::wsMapId($tempTable, $this->import_mapId[$tempTable][$tempUid]);
2657 // Look if reference is to a page and the original token value was NOT an integer - then we assume is was an alias and try to look up the new one!
2658 if ($tempTable === 'pages' && !\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($cfg['subst']['tokenValue'])) {
2659 $recWithUniqueValue = BackendUtility::getRecord($tempTable, $insertValue, 'alias');
2660 if ($recWithUniqueValue['alias']) {
2661 $insertValue = $recWithUniqueValue['alias'];
2662 }
2663 } elseif (strpos($cfg['subst']['tokenValue'], ':') !== FALSE) {
2664 list($tokenKey, $tokenId) = explode(':', $cfg['subst']['tokenValue']);
2665 $insertValue = $tokenKey . ':' . $insertValue;
2666 }
2667 }
2668 }
2669 }
2670 // Finally, swap the soft reference token in tokenized content with the insert value:
2671 $tokenizedContent = str_replace('{softref:' . $tokenID . '}', $insertValue, $tokenizedContent);
2672 }
2673 return $tokenizedContent;
2674 }
2675
2676 /**
2677 * Process a soft reference file
2678 *
2679 * @param string $relFileName Old Relative filename
2680 * @param array $cfg soft reference configuration array
2681 * @param string $table Table for which the processing occurs
2682 * @param string $uid UID of record from table
2683 * @return string New relative filename (value to insert instead of the softref token)
2684 */
2685 public function processSoftReferences_saveFile($relFileName, $cfg, $table, $uid) {
2686 if ($fileHeaderInfo = $this->dat['header']['files'][$cfg['file_ID']]) {
2687 // Initialize; Get directory prefix for file and find possible RTE filename
2688 $dirPrefix = PathUtility::dirname($relFileName) . '/';
2689 $rteOrigName = $this->getRTEoriginalFilename(PathUtility::basename($relFileName));
2690 // If filename looks like an RTE file, and the directory is in "uploads/", then process as a RTE file!
2691 if ($rteOrigName && GeneralUtility::isFirstPartOfStr($dirPrefix, 'uploads/')) {
2692 // RTE:
2693 // First, find unique RTE file name:
2694 if (@is_dir((PATH_site . $dirPrefix))) {
2695 // From the "original" RTE filename, produce a new "original" destination filename which is unused.
2696 // Even if updated, the image should be unique. Currently the problem with this is that it leaves a lot of unused RTE images...
2697 $fileProcObj = $this->getFileProcObj();
2698 $origDestName = $fileProcObj->getUniqueName($rteOrigName, PATH_site . $dirPrefix);
2699 // Create copy file name:
2700 $pI = pathinfo($relFileName);
2701 $copyDestName = PathUtility::dirname($origDestName) . '/RTEmagicC_' . substr(PathUtility::basename($origDestName), 10) . '.' . $pI['extension'];
2702 if (
2703 !@is_file($copyDestName) && !@is_file($origDestName)
2704 && $origDestName === GeneralUtility::getFileAbsFileName($origDestName)
2705 && $copyDestName === GeneralUtility::getFileAbsFileName($copyDestName)
2706 ) {
2707 if ($this->dat['header']['files'][$fileHeaderInfo['RTE_ORIG_ID']]) {
2708 if ($this->legacyImport) {
2709 $fileName = PathUtility::basename($copyDestName);
2710 $this->writeSysFileResourceForLegacyImport($fileName, $cfg['file_ID']);
2711 $relFileName = $this->filePathMap[$cfg['file_ID']] . '" data-htmlarea-file-uid="' . $fileName . '" data-htmlarea-file-table="sys_file';
2712 // Also save the original file
2713 $originalFileName = PathUtility::basename($origDestName);
2714 $this->writeSysFileResourceForLegacyImport($originalFileName, $fileHeaderInfo['RTE_ORIG_ID']);
2715 } else {
2716 // Write the copy and original RTE file to the respective filenames:
2717 $this->writeFileVerify($copyDestName, $cfg['file_ID'], TRUE);
2718 $this->writeFileVerify($origDestName, $fileHeaderInfo['RTE_ORIG_ID'], TRUE);
2719 // Return the relative path of the copy file name:
2720 return PathUtility::stripPathSitePrefix($copyDestName);
2721 }
2722 } else {
2723 $this->error('ERROR: Could not find original file ID');
2724 }
2725 } else {
2726 $this->error('ERROR: The destination filenames "' . $copyDestName . '" and "' . $origDestName . '" either existed or have non-valid names');
2727 }
2728 } else {
2729 $this->error('ERROR: "' . PATH_site . $dirPrefix . '" was not a directory, so could not process file "' . $relFileName . '"');
2730 }
2731 } elseif (GeneralUtility::isFirstPartOfStr($dirPrefix, $this->fileadminFolderName . '/')) {
2732 // File in fileadmin/ folder:
2733 // Create file (and possible resources)
2734 $newFileName = $this->processSoftReferences_saveFile_createRelFile($dirPrefix, PathUtility::basename($relFileName), $cfg['file_ID'], $table, $uid);
2735 if (strlen($newFileName)) {
2736 $relFileName = $newFileName;
2737 } else {
2738 $this->error('ERROR: No new file created for "' . $relFileName . '"');
2739 }
2740 } else {
2741 $this->error('ERROR: Sorry, cannot operate on non-RTE files which are outside the fileadmin folder.');
2742 }
2743 } else {
2744 $this->error('ERROR: Could not find file ID in header.');
2745 }
2746 // Return (new) filename relative to PATH_site:
2747 return $relFileName;
2748 }
2749
2750 /**
2751 * Create file in directory and return the new (unique) filename
2752 *
2753 * @param string $origDirPrefix Directory prefix, relative, with trailing slash
2754 * @param string $fileName Filename (without path)
2755 * @param string $fileID File ID from import memory
2756 * @param string $table Table for which the processing occurs
2757 * @param string $uid UID of record from table
2758 * @return string|NULL New relative filename, if any
2759 */
2760 public function processSoftReferences_saveFile_createRelFile($origDirPrefix, $fileName, $fileID, $table, $uid) {
2761 // If the fileID map contains an entry for this fileID then just return the relative filename of that entry;
2762 // we don't want to write another unique filename for this one!
2763 if (isset($this->fileIDMap[$fileID])) {
2764 return PathUtility::stripPathSitePrefix($this->fileIDMap[$fileID]);
2765 }
2766 if ($this->legacyImport) {
2767 // set dirPrefix to fileadmin because the right target folder is set and checked for permissions later
2768 $dirPrefix = $this->fileadminFolderName . '/';
2769 } else {
2770 // Verify FileMount access to dir-prefix. Returns the best alternative relative path if any
2771 $dirPrefix = $this->verifyFolderAccess($origDirPrefix);
2772 }
2773 if ($dirPrefix && (!$this->update || $origDirPrefix === $dirPrefix) && $this->checkOrCreateDir($dirPrefix)) {
2774 $fileHeaderInfo = $this->dat['header']['files'][$fileID];
2775 $updMode = $this->update && $this->import_mapId[$table][$uid] === $uid && $this->import_mode[$table . ':' . $uid] !== 'as_new';
2776 // Create new name for file:
2777 // Must have same ID in map array (just for security, is not really needed) and NOT be set "as_new".
2778
2779 // Write main file:
2780 if ($this->legacyImport) {
2781 $fileWritten = $this->writeSysFileResourceForLegacyImport($fileName, $fileID);
2782 if ($fileWritten) {
2783 $newName = 'file:' . $fileName;
2784 return $newName;
2785 // no support for HTML/CSS file resources attached ATM - see below
2786 }
2787 } else {
2788 if ($updMode) {
2789 $newName = PATH_site . $dirPrefix . $fileName;
2790 } else {
2791 // Create unique filename:
2792 $fileProcObj = $this->getFileProcObj();
2793 $newName = $fileProcObj->getUniqueName($fileName, PATH_site . $dirPrefix);
2794 }
2795 if ($this->writeFileVerify($newName, $fileID)) {
2796 // If the resource was an HTML/CSS file with resources attached, we will write those as well!
2797 if (is_array($fileHeaderInfo['EXT_RES_ID'])) {
2798 $tokenizedContent = $this->dat['files'][$fileID]['tokenizedContent'];
2799 $tokenSubstituted = FALSE;
2800 $fileProcObj = $this->getFileProcObj();
2801 if ($updMode) {
2802 foreach ($fileHeaderInfo['EXT_RES_ID'] as $res_fileID) {
2803 if ($this->dat['files'][$res_fileID]['filename']) {
2804 // Resolve original filename:
2805 $relResourceFileName = $this->dat['files'][$res_fileID]['parentRelFileName'];
2806 $absResourceFileName = GeneralUtility::resolveBackPath(PATH_site . $origDirPrefix . $relResourceFileName);
2807 $absResourceFileName = GeneralUtility::getFileAbsFileName($absResourceFileName);
2808 if ($absResourceFileName && GeneralUtility::isFirstPartOfStr($absResourceFileName, PATH_site . $this->fileadminFolderName . '/')) {
2809 $destDir = PathUtility::stripPathSitePrefix(PathUtility::dirname($absResourceFileName) . '/');
2810 if ($this->verifyFolderAccess($destDir, TRUE) && $this->checkOrCreateDir($destDir)) {
2811 $this->writeFileVerify($absResourceFileName, $res_fileID);
2812 } else {
2813 $this->error('ERROR: Could not create file in directory "' . $destDir . '"');
2814 }
2815 } else {
2816 $this->error('ERROR: Could not resolve path for "' . $relResourceFileName . '"');
2817 }
2818 $tokenizedContent = str_replace('{EXT_RES_ID:' . $res_fileID . '}', $relResourceFileName, $tokenizedContent);
2819 $tokenSubstituted = TRUE;
2820 }
2821 }
2822 } else {
2823 // Create the resouces directory name (filename without extension, suffixed "_FILES")
2824 $resourceDir = PathUtility::dirname($newName) . '/' . preg_replace('/\\.[^.]*$/', '', PathUtility::basename($newName)) . '_FILES';
2825 if (GeneralUtility::mkdir($resourceDir)) {
2826 foreach ($fileHeaderInfo['EXT_RES_ID'] as $res_fileID) {
2827 if ($this->dat['files'][$res_fileID]['filename']) {
2828 $absResourceFileName = $fileProcObj->getUniqueName($this->dat['files'][$res_fileID]['filename'], $resourceDir);
2829 $relResourceFileName = substr($absResourceFileName, strlen(PathUtility::dirname($resourceDir)) + 1);
2830 $this->writeFileVerify($absResourceFileName, $res_fileID);
2831 $tokenizedContent = str_replace('{EXT_RES_ID:' . $res_fileID . '}', $relResourceFileName, $tokenizedContent);
2832 $tokenSubstituted = TRUE;
2833 }
2834 }
2835 }