[CLEANUP] Improve the @param/@return/@var PHPDoc
[Packages/TYPO3.CMS.git] / typo3 / sysext / impexp / Classes / ImportExport.php
1 <?php
2 namespace TYPO3\CMS\Impexp;
3
4 /*
5 * This file is part of the TYPO3 CMS project.
6 *
7 * It is free software; you can redistribute it and/or modify it under
8 * the terms of the GNU General Public License, either version 2
9 * of the License, or any later version.
10 *
11 * For the full copyright and license information, please read the
12 * LICENSE.txt file that was distributed with this source code.
13 *
14 * The TYPO3 project - inspiring people to share!
15 */
16
17 use TYPO3\CMS\Backend\Utility\BackendUtility;
18 use TYPO3\CMS\Core\DataHandling\DataHandler;
19 use TYPO3\CMS\Core\Exception;
20 use TYPO3\CMS\Core\Resource\ResourceFactory;
21 use TYPO3\CMS\Core\Utility\File\ExtendedFileUtility;
22 use TYPO3\CMS\Core\Utility\GeneralUtility;
23 use TYPO3\CMS\Core\Utility\PathUtility;
24
25 /**
26 * EXAMPLE for using the impexp-class for exporting stuff:
27 *
28 * Create and initialize:
29 * $this->export = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Impexp\ImportExport::class);
30 * $this->export->init();
31 * Set which tables relations we will allow:
32 * $this->export->relOnlyTables[]="tt_news"; // exclusively includes. See comment in the class
33 *
34 * Adding records:
35 * $this->export->export_addRecord("pages", $this->pageinfo);
36 * $this->export->export_addRecord("pages", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("pages", 38));
37 * $this->export->export_addRecord("pages", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("pages", 39));
38 * $this->export->export_addRecord("tt_content", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("tt_content", 12));
39 * $this->export->export_addRecord("tt_content", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("tt_content", 74));
40 * $this->export->export_addRecord("sys_template", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("sys_template", 20));
41 *
42 * Adding all the relations (recursively in 5 levels so relations has THEIR relations registered as well)
43 * for($a=0;$a<5;$a++) {
44 * $addR = $this->export->export_addDBRelations($a);
45 * if (!count($addR)) break;
46 * }
47 *
48 * Finally load all the files.
49 * $this->export->export_addFilesFromRelations(); // MUST be after the DBrelations are set so that file from ALL added records are included!
50 *
51 * Write export
52 * $out = $this->export->compileMemoryToFileContent();
53 */
54
55 /**
56 * T3D file Import/Export library (TYPO3 Record Document)
57 *
58 * @author Kasper Skårhøj <kasperYYYY@typo3.com>
59 */
60 class ImportExport {
61
62 /**
63 * If set, static relations (not exported) will be shown in overview as well
64 *
65 * @var bool
66 */
67 public $showStaticRelations = FALSE;
68
69 /**
70 * Name of the "fileadmin" folder where files for export/import should be located
71 *
72 * @var string
73 */
74 public $fileadminFolderName = '';
75
76 /**
77 * Whether "import" or "export" mode of object. Set through init() function
78 *
79 * @var string
80 */
81 public $mode = '';
82
83 /**
84 * Updates all records that has same UID instead of creating new!
85 *
86 * @var bool
87 */
88 public $update = FALSE;
89
90 /**
91 * Is set by importData() when an import has been done.
92 *
93 * @var bool
94 */
95 public $doesImport = FALSE;
96
97 /**
98 * If set to a page-record, then the preview display of the content will expect this page-record to be the target
99 * for the import and accordingly display validation information. This triggers the visual view of the
100 * import/export memory to validate if import is possible
101 *
102 * @var array
103 */
104 public $display_import_pid_record = array();
105
106 /**
107 * Used to register the forged UID values for imported records that we want
108 * to create with the same UIDs as in the import file. Admin-only feature.
109 *
110 * @var array
111 */
112 public $suggestedInsertUids = array();
113
114 /**
115 * Setting import modes during update state: as_new, exclude, force_uid
116 *
117 * @var array
118 */
119 public $import_mode = array();
120
121 /**
122 * If set, PID correct is ignored globally
123 *
124 * @var bool
125 */
126 public $global_ignore_pid = FALSE;
127
128 /**
129 * If set, all UID values are forced! (update or import)
130 *
131 * @var bool
132 */
133 public $force_all_UIDS = FALSE;
134
135 /**
136 * If set, a diff-view column is added to the overview.
137 *
138 * @var bool
139 */
140 public $showDiff = FALSE;
141
142 /**
143 * If set, and if the user is admin, allow the writing of PHP scripts to fileadmin/ area.
144 *
145 * @var bool
146 */
147 public $allowPHPScripts = FALSE;
148
149 /**
150 * Disable logging when importing
151 *
152 * @var bool
153 */
154 public $enableLogging = FALSE;
155
156 /**
157 * Array of values to substitute in editable softreferences.
158 *
159 * @var array
160 */
161 public $softrefInputValues = array();
162
163 /**
164 * Mapping between the fileID from import memory and the final filenames they are written to.
165 *
166 * @var array
167 */
168 public $fileIDMap = array();
169
170 /**
171 * 1MB max file size
172 *
173 * @var int
174 */
175 public $maxFileSize = 1000000;
176
177 /**
178 * 1MB max record size
179 *
180 * @var int
181 */
182 public $maxRecordSize = 1000000;
183
184 /**
185 * 10MB max export size
186 *
187 * @var int
188 */
189 public $maxExportSize = 10000000;
190
191 /**
192 * Add table names here which are THE ONLY ones which will be included
193 * into export if found as relations. '_ALL' will allow all tables.
194 *
195 * @var array
196 */
197 public $relOnlyTables = array();
198
199 /**
200 * Add tables names here which should not be exported with the file.
201 * (Where relations should be mapped to same UIDs in target system).
202 *
203 * @var array
204 */
205 public $relStaticTables = array();
206
207 /**
208 * Exclude map. Keys are table:uid pairs and if set, records are not added to the export.
209 *
210 * @var array
211 */
212 public $excludeMap = array();
213
214 /**
215 * Soft Reference Token ID modes.
216 *
217 * @var array
218 */
219 public $softrefCfg = array();
220
221 /**
222 * Listing extension dependencies.
223 *
224 * @var array
225 */
226 public $extensionDependencies = array();
227
228 /**
229 * Set by user: If set, compression in t3d files is disabled
230 *
231 * @var bool
232 */
233 public $dontCompress = FALSE;
234
235 /**
236 * If set, HTML file resources are included.
237 *
238 * @var bool
239 */
240 public $includeExtFileResources = FALSE;
241
242 /**
243 * Files with external media (HTML/css style references inside)
244 *
245 * @var string
246 */
247 public $extFileResourceExtensions = 'html,htm,css';
248
249 /**
250 * After records are written this array is filled with [table][original_uid] = [new_uid]
251 *
252 * @var array
253 */
254 public $import_mapId = array();
255
256 /**
257 * Keys are [tablename]:[new NEWxxx ids (or when updating it is uids)]
258 * while values are arrays with table/uid of the original record it is based on.
259 * With the array keys the new ids can be looked up inside tcemain
260 *
261 * @var array
262 */
263 public $import_newId = array();
264
265 /**
266 * Page id map for page tree (import)
267 *
268 * @var array
269 */
270 public $import_newId_pids = array();
271
272 /**
273 * Internal data accumulation for writing records during import
274 *
275 * @var array
276 */
277 public $import_data = array();
278
279 /**
280 * Error log.
281 *
282 * @var array
283 */
284 public $errorLog = array();
285
286 /**
287 * Cache for record paths
288 *
289 * @var array
290 */
291 public $cache_getRecordPath = array();
292
293 /**
294 * Cache of checkPID values.
295 *
296 * @var array
297 */
298 public $checkPID_cache = array();
299
300 /**
301 * Set internally if the gzcompress function exists
302 * Used by ImportExportController
303 *
304 * @var bool
305 */
306 public $compress = FALSE;
307
308 /**
309 * Internal import/export memory
310 *
311 * @var array
312 */
313 public $dat = array();
314
315 /**
316 * File processing object
317 *
318 * @var \TYPO3\CMS\Core\Utility\File\ExtendedFileUtility
319 */
320 protected $fileProcObj = NULL;
321
322 /**
323 * Keys are [recordname], values are an array of fields to be included
324 * in the export
325 *
326 * @var array
327 */
328 protected $recordTypesIncludeFields = array();
329
330 /**
331 * Default array of fields to be included in the export
332 *
333 * @var array
334 */
335 protected $defaultRecordIncludeFields = array('uid', 'pid');
336
337 /**
338 * Array of current registered storage objects
339 *
340 * @var array
341 */
342 protected $storageObjects = array();
343
344 /**
345 * Is set, if the import file has a TYPO3 version below 6.0
346 *
347 * @var bool
348 */
349 protected $legacyImport = FALSE;
350
351 /**
352 * @var \TYPO3\CMS\Core\Resource\Folder
353 */
354 protected $legacyImportFolder = NULL;
355
356 /**
357 * Related to the default storage root
358 *
359 * @var string
360 */
361 protected $legacyImportTargetPath = '_imported/';
362
363 /**
364 * Table fields to migrate
365 *
366 * @var array
367 */
368 protected $legacyImportMigrationTables = array(
369 'tt_content' => array(
370 'image' => array(
371 'titleTexts' => 'titleText',
372 'description' => 'imagecaption',
373 'links' => 'image_link',
374 'alternativeTexts' => 'altText'
375 ),
376 'media' => array(
377 'description' => 'imagecaption',
378 )
379 ),
380 'pages' => array(
381 'media' => array()
382 ),
383 'pages_language_overlay' => array(
384 'media' => array()
385 )
386 );
387
388
389 /**
390 * Records to be migrated after all
391 * Multidimensional array [table][uid][field] = array([related sys_file_reference uids])
392 *
393 * @var array
394 */
395 protected $legacyImportMigrationRecords = array();
396
397 /**
398 * @var bool
399 */
400 protected $saveFilesOutsideExportFile = FALSE;
401
402 /**
403 * @var NULL|string
404 */
405 protected $temporaryFilesPathForExport = NULL;
406
407 /**
408 * @var NULL|string
409 */
410 protected $filesPathForImport = NULL;
411
412 /**
413 * @var array
414 */
415 protected $unlinkFiles = array();
416
417 /**
418 * @var array
419 */
420 protected $alternativeFileName = array();
421
422 /**
423 * @var array
424 */
425 protected $alternativeFilePath = array();
426
427 /**
428 * @var array
429 */
430 protected $filePathMap = array();
431
432 /**
433 * @var array
434 */
435 protected $remainHeader = array();
436
437 /**************************
438 * Initialize
439 *************************/
440
441 /**
442 * Init the object, both import and export
443 *
444 * @param bool $dontCompress If set, compression of t3d files is disabled
445 * @param string $mode Mode of usage, either "import" or "export
446 * @return void
447 */
448 public function init($dontCompress = FALSE, $mode = '') {
449 $this->compress = function_exists('gzcompress');
450 $this->dontCompress = $dontCompress;
451 $this->mode = $mode;
452 $this->fileadminFolderName = !empty($GLOBALS['TYPO3_CONF_VARS']['BE']['fileadminDir']) ? rtrim($GLOBALS['TYPO3_CONF_VARS']['BE']['fileadminDir'], '/') : 'fileadmin';
453 }
454
455 /**************************
456 * Export / Init + Meta Data
457 *************************/
458
459 /**
460 * Set header basics
461 *
462 * @return void
463 */
464 public function setHeaderBasics() {
465 // Initializing:
466 if (is_array($this->softrefCfg)) {
467 foreach ($this->softrefCfg as $key => $value) {
468 if (!strlen($value['mode'])) {
469 unset($this->softrefCfg[$key]);
470 }
471 }
472 }
473 // Setting in header memory:
474 // Version of file format
475 $this->dat['header']['XMLversion'] = '1.0';
476 // Initialize meta data array (to put it in top of file)
477 $this->dat['header']['meta'] = array();
478 // Add list of tables to consider static
479 $this->dat['header']['relStaticTables'] = $this->relStaticTables;
480 // The list of excluded records
481 $this->dat['header']['excludeMap'] = $this->excludeMap;
482 // Soft Reference mode for elements
483 $this->dat['header']['softrefCfg'] = $this->softrefCfg;
484 // List of extensions the import depends on.
485 $this->dat['header']['extensionDependencies'] = $this->extensionDependencies;
486 }
487
488 /**
489 * Set charset
490 *
491 * @param string $charset Charset for the content in the export. During import the character set will be converted if the target system uses another charset.
492 * @return void
493 */
494 public function setCharset($charset) {
495 $this->dat['header']['charset'] = $charset;
496 }
497
498 /**
499 * Sets meta data
500 *
501 * @param string $title Title of the export
502 * @param string $description Description of the export
503 * @param string $notes Notes about the contents
504 * @param string $packager_username Backend Username of the packager (the guy making the export)
505 * @param string $packager_name Real name of the packager
506 * @param string $packager_email Email of the packager
507 * @return void
508 */
509 public function setMetaData($title, $description, $notes, $packager_username, $packager_name, $packager_email) {
510 $this->dat['header']['meta'] = array(
511 'title' => $title,
512 'description' => $description,
513 'notes' => $notes,
514 'packager_username' => $packager_username,
515 'packager_name' => $packager_name,
516 'packager_email' => $packager_email,
517 'TYPO3_version' => TYPO3_version,
518 'created' => strftime('%A %e. %B %Y', $GLOBALS['EXEC_TIME'])
519 );
520 }
521
522 /**
523 * Option to enable having the files not included in the export file.
524 * The files are saved to a temporary folder instead.
525 *
526 * @param bool $saveFilesOutsideExportFile
527 * @see getTemporaryFilesPathForExport()
528 */
529 public function setSaveFilesOutsideExportFile($saveFilesOutsideExportFile) {
530 $this->saveFilesOutsideExportFile = $saveFilesOutsideExportFile;
531 }
532
533 /**
534 * Sets a thumbnail image to the exported file
535 *
536 * @param string $imgFilepath Filename reference, gif, jpg, png. Absolute path.
537 * @return void
538 */
539 public function addThumbnail($imgFilepath) {
540 if (@is_file($imgFilepath)) {
541 $imgInfo = @getimagesize($imgFilepath);
542 if (is_array($imgInfo)) {
543 $fileContent = GeneralUtility::getUrl($imgFilepath);
544 $this->dat['header']['thumbnail'] = array(
545 'imgInfo' => $imgInfo,
546 'content' => $fileContent,
547 'filesize' => strlen($fileContent),
548 'filemtime' => filemtime($imgFilepath),
549 'filename' => PathUtility::basename($imgFilepath)
550 );
551 }
552 }
553 }
554
555 /**************************
556 * Export / Init Page tree
557 *************************/
558
559 /**
560 * Sets the page-tree array in the export header and returns the array in a flattened version
561 *
562 * @param array $idH Hierarchy of ids, the page tree: array([uid] => array("uid" => [uid], "subrow" => array(.....)), [uid] => ....)
563 * @return array The hierarchical page tree converted to a one-dimensional list of pages
564 */
565 public function setPageTree($idH) {
566 $this->dat['header']['pagetree'] = $this->unsetExcludedSections($idH);
567 return $this->flatInversePageTree($this->dat['header']['pagetree']);
568 }
569
570 /**
571 * Removes entries in the page tree which are found in ->excludeMap[]
572 *
573 * @param array $idH Page uid hierarchy
574 * @return array Modified input array
575 * @access private
576 * @see setPageTree()
577 */
578 public function unsetExcludedSections($idH) {
579 if (is_array($idH)) {
580 foreach ($idH as $k => $v) {
581 if ($this->excludeMap['pages:' . $idH[$k]['uid']]) {
582 unset($idH[$k]);
583 } elseif (is_array($idH[$k]['subrow'])) {
584 $idH[$k]['subrow'] = $this->unsetExcludedSections($idH[$k]['subrow']);
585 }
586 }
587 }
588 return $idH;
589 }
590
591 /**
592 * Recursively flattening the idH array (for setPageTree() function)
593 *
594 * @param array $idH Page uid hierarchy
595 * @param array $a Accumulation array of pages (internal, don't set from outside)
596 * @return array Array with uid-uid pairs for all pages in the page tree.
597 * @see flatInversePageTree_pid()
598 */
599 public function flatInversePageTree($idH, $a = array()) {
600 if (is_array($idH)) {
601 $idH = array_reverse($idH);
602 foreach ($idH as $k => $v) {
603 $a[$v['uid']] = $v['uid'];
604 if (is_array($v['subrow'])) {
605 $a = $this->flatInversePageTree($v['subrow'], $a);
606 }
607 }
608 }
609 return $a;
610 }
611
612 /**
613 * Recursively flattening the idH array (for setPageTree() function), setting PIDs as values
614 *
615 * @param array $idH Page uid hierarchy
616 * @param array $a Accumulation array of pages (internal, don't set from outside)
617 * @param int $pid PID value (internal)
618 * @return array Array with uid-pid pairs for all pages in the page tree.
619 * @see flatInversePageTree()
620 */
621 public function flatInversePageTree_pid($idH, $a = array(), $pid = -1) {
622 if (is_array($idH)) {
623 $idH = array_reverse($idH);
624 foreach ($idH as $v) {
625 $a[$v['uid']] = $pid;
626 if (is_array($v['subrow'])) {
627 $a = $this->flatInversePageTree_pid($v['subrow'], $a, $v['uid']);
628 }
629 }
630 }
631 return $a;
632 }
633
634 /**************************
635 * Export
636 *************************/
637
638 /**
639 * Sets the fields of record types to be included in the export
640 *
641 * @param array $recordTypesIncludeFields Keys are [recordname], values are an array of fields to be included in the export
642 * @throws \TYPO3\CMS\Core\Exception if an array value is not type of array
643 * @return void
644 */
645 public function setRecordTypesIncludeFields(array $recordTypesIncludeFields) {
646 foreach ($recordTypesIncludeFields as $table => $fields) {
647 if (!is_array($fields)) {
648 throw new \TYPO3\CMS\Core\Exception('The include fields for record type ' . htmlspecialchars($table) . ' are not defined by an array.', 1391440658);
649 }
650 $this->setRecordTypeIncludeFields($table, $fields);
651 }
652 }
653
654 /**
655 * Sets the fields of a record type to be included in the export
656 *
657 * @param string $table The record type
658 * @param array $fields The fields to be included
659 * @return void
660 */
661 public function setRecordTypeIncludeFields($table, array $fields) {
662 $this->recordTypesIncludeFields[$table] = $fields;
663 }
664
665 /**
666 * Adds the record $row from $table.
667 * No checking for relations done here. Pure data.
668 *
669 * @param string $table Table name
670 * @param array $row Record row.
671 * @param int $relationLevel (Internal) if the record is added as a relation, this is set to the "level" it was on.
672 * @return void
673 */
674 public function export_addRecord($table, $row, $relationLevel = 0) {
675 BackendUtility::workspaceOL($table, $row);
676 if ((string)$table !== '' && is_array($row) && $row['uid'] > 0 && !$this->excludeMap[($table . ':' . $row['uid'])]) {
677 if ($this->checkPID($table === 'pages' ? $row['uid'] : $row['pid'])) {
678 if (!isset($this->dat['records'][($table . ':' . $row['uid'])])) {
679 // Prepare header info:
680 $row = $this->filterRecordFields($table, $row);
681 $headerInfo = array();
682 $headerInfo['uid'] = $row['uid'];
683 $headerInfo['pid'] = $row['pid'];
684 $headerInfo['title'] = GeneralUtility::fixed_lgd_cs(BackendUtility::getRecordTitle($table, $row), 40);
685 $headerInfo['size'] = strlen(serialize($row));
686 if ($relationLevel) {
687 $headerInfo['relationLevel'] = $relationLevel;
688 }
689 // If record content is not too large in size, set the header content and add the rest:
690 if ($headerInfo['size'] < $this->maxRecordSize) {
691 // Set the header summary:
692 $this->dat['header']['records'][$table][$row['uid']] = $headerInfo;
693 // Create entry in the PID lookup:
694 $this->dat['header']['pid_lookup'][$row['pid']][$table][$row['uid']] = 1;
695 // Initialize reference index object:
696 $refIndexObj = GeneralUtility::makeInstance(\TYPO3\CMS\Core\Database\ReferenceIndex::class);
697 // Yes to workspace overlays for exporting....
698 $refIndexObj->WSOL = TRUE;
699 $relations = $refIndexObj->getRelations($table, $row);
700 $relations = $this->fixFileIDsInRelations($relations);
701 // Data:
702 $this->dat['records'][$table . ':' . $row['uid']] = array();
703 $this->dat['records'][$table . ':' . $row['uid']]['data'] = $row;
704 $this->dat['records'][$table . ':' . $row['uid']]['rels'] = $relations;
705 $this->errorLog = array_merge($this->errorLog, $refIndexObj->errorLog);
706 // Merge error logs.
707 // Add information about the relations in the record in the header:
708 $this->dat['header']['records'][$table][$row['uid']]['rels'] = $this->flatDBrels($this->dat['records'][$table . ':' . $row['uid']]['rels']);
709 // Add information about the softrefs to header:
710 $this->dat['header']['records'][$table][$row['uid']]['softrefs'] = $this->flatSoftRefs($this->dat['records'][$table . ':' . $row['uid']]['rels']);
711 } else {
712 $this->error('Record ' . $table . ':' . $row['uid'] . ' was larger than maxRecordSize (' . GeneralUtility::formatSize($this->maxRecordSize) . ')');
713 }
714 } else {
715 $this->error('Record ' . $table . ':' . $row['uid'] . ' already added.');
716 }
717 } else {
718 $this->error('Record ' . $table . ':' . $row['uid'] . ' was outside your DB mounts!');
719 }
720 }
721 }
722
723 /**
724 * This changes the file reference ID from a hash based on the absolute file path
725 * (coming from ReferenceIndex) to a hash based on the relative file path.
726 *
727 * @param array $relations
728 * @return array
729 */
730 protected function fixFileIDsInRelations(array $relations) {
731 foreach ($relations as $field => $relation) {
732 if (isset($relation['type']) && $relation['type'] === 'file') {
733 foreach ($relation['newValueFiles'] as $key => $fileRelationData) {
734 $absoluteFilePath = $fileRelationData['ID_absFile'];
735 if (GeneralUtility::isFirstPartOfStr($absoluteFilePath, PATH_site)) {
736 $relatedFilePath = PathUtility::stripPathSitePrefix($absoluteFilePath);
737 $relations[$field]['newValueFiles'][$key]['ID'] = md5($relatedFilePath);
738 }
739 }
740 }
741 if ($relation['type'] === 'flex') {
742 if (is_array($relation['flexFormRels']['file'])) {
743 foreach ($relation['flexFormRels']['file'] as $key => $subList) {
744 foreach ($subList as $subKey => $fileRelationData) {
745 $absoluteFilePath = $fileRelationData['ID_absFile'];
746 if (GeneralUtility::isFirstPartOfStr($absoluteFilePath, PATH_site)) {
747 $relatedFilePath = PathUtility::stripPathSitePrefix($absoluteFilePath);
748 $relations[$field]['flexFormRels']['file'][$key][$subKey]['ID'] = md5($relatedFilePath);
749 }
750 }
751 }
752 }
753 }
754 }
755 return $relations;
756 }
757
758
759
760 /**
761 * This analyses the existing added records, finds all database relations to records and adds these records to the export file.
762 * This function can be called repeatedly until it returns an empty array.
763 * In principle it should not allow to infinite recursivity, but you better set a limit...
764 * Call this BEFORE the ext_addFilesFromRelations (so files from added relations are also included of course)
765 *
766 * @param int $relationLevel Recursion level
767 * @return array overview of relations found and added: Keys [table]:[uid], values array with table and id
768 * @see export_addFilesFromRelations()
769 */
770 public function export_addDBRelations($relationLevel = 0) {
771 // Traverse all "rels" registered for "records"
772 if (!is_array($this->dat['records'])) {
773 $this->error('There were no records available.');
774 return array();
775 }
776 $addR = array();
777 foreach ($this->dat['records'] as $k => $value) {
778 if (!is_array($this->dat['records'][$k])) {
779 continue;
780 }
781 foreach ($this->dat['records'][$k]['rels'] as $fieldname => $vR) {
782 // For all DB types of relations:
783 if ($vR['type'] == 'db') {
784 foreach ($vR['itemArray'] as $fI) {
785 $this->export_addDBRelations_registerRelation($fI, $addR);
786 }
787 }
788 // For all flex/db types of relations:
789 if ($vR['type'] == 'flex') {
790 // DB relations in flex form fields:
791 if (is_array($vR['flexFormRels']['db'])) {
792 foreach ($vR['flexFormRels']['db'] as $subList) {
793 foreach ($subList as $fI) {
794 $this->export_addDBRelations_registerRelation($fI, $addR);
795 }
796 }
797 }
798 // DB oriented soft references in flex form fields:
799 if (is_array($vR['flexFormRels']['softrefs'])) {
800 foreach ($vR['flexFormRels']['softrefs'] as $subList) {
801 foreach ($subList['keys'] as $spKey => $elements) {
802 foreach ($elements as $el) {
803 if ($el['subst']['type'] === 'db' && $this->includeSoftref($el['subst']['tokenID'])) {
804 list($tempTable, $tempUid) = explode(':', $el['subst']['recordRef']);
805 $fI = array(
806 'table' => $tempTable,
807 'id' => $tempUid
808 );
809 $this->export_addDBRelations_registerRelation($fI, $addR, $el['subst']['tokenID']);
810 }
811 }
812 }
813 }
814 }
815 }
816 // In any case, if there are soft refs:
817 if (is_array($vR['softrefs']['keys'])) {
818 foreach ($vR['softrefs']['keys'] as $spKey => $elements) {
819 foreach ($elements as $el) {
820 if ($el['subst']['type'] === 'db' && $this->includeSoftref($el['subst']['tokenID'])) {
821 list($tempTable, $tempUid) = explode(':', $el['subst']['recordRef']);
822 $fI = array(
823 'table' => $tempTable,
824 'id' => $tempUid
825 );
826 $this->export_addDBRelations_registerRelation($fI, $addR, $el['subst']['tokenID']);
827 }
828 }
829 }
830 }
831 }
832 }
833
834 // Now, if there were new records to add, do so:
835 if (!empty($addR)) {
836 foreach ($addR as $fI) {
837 // Get and set record:
838 $row = BackendUtility::getRecord($fI['table'], $fI['id']);
839 if (is_array($row)) {
840 $this->export_addRecord($fI['table'], $row, $relationLevel + 1);
841 }
842 // Set status message
843 // Relation pointers always larger than zero except certain "select" types with
844 // negative values pointing to uids - but that is not supported here.
845 if ($fI['id'] > 0) {
846 $rId = $fI['table'] . ':' . $fI['id'];
847 if (!isset($this->dat['records'][$rId])) {
848 $this->dat['records'][$rId] = 'NOT_FOUND';
849 $this->error('Relation record ' . $rId . ' was not found!');
850 }
851 }
852 }
853 }
854 // Return overview of relations found and added
855 return $addR;
856 }
857
858 /**
859 * Helper function for export_addDBRelations()
860 *
861 * @param array $fI Array with table/id keys to add
862 * @param array $addR Add array, passed by reference to be modified
863 * @param string $tokenID Softref Token ID, if applicable.
864 * @return void
865 * @see export_addDBRelations()
866 */
867 public function export_addDBRelations_registerRelation($fI, &$addR, $tokenID = '') {
868 $rId = $fI['table'] . ':' . $fI['id'];
869 if (
870 isset($GLOBALS['TCA'][$fI['table']]) && !$this->isTableStatic($fI['table']) && !$this->isExcluded($fI['table'], $fI['id'])
871 && (!$tokenID || $this->includeSoftref($tokenID)) && $this->inclRelation($fI['table'])
872 ) {
873 if (!isset($this->dat['records'][$rId])) {
874 // Set this record to be included since it is not already.
875 $addR[$rId] = $fI;
876 }
877 }
878 }
879
880 /**
881 * This adds all files in relations.
882 * Call this method AFTER adding all records including relations.
883 *
884 * @return void
885 * @see export_addDBRelations()
886 */
887 public function export_addFilesFromRelations() {
888 // Traverse all "rels" registered for "records"
889 if (!is_array($this->dat['records'])) {
890 $this->error('There were no records available.');
891 return;
892 }
893 foreach ($this->dat['records'] as $k => $value) {
894 if (!isset($this->dat['records'][$k]['rels']) || !is_array($this->dat['records'][$k]['rels'])) {
895 continue;
896 }
897 foreach ($this->dat['records'][$k]['rels'] as $fieldname => $vR) {
898 // For all file type relations:
899 if ($vR['type'] == 'file') {
900 foreach ($vR['newValueFiles'] as $key => $fI) {
901 $this->export_addFile($fI, $k, $fieldname);
902 // Remove the absolute reference to the file so it doesn't expose absolute paths from source server:
903 unset($this->dat['records'][$k]['rels'][$fieldname]['newValueFiles'][$key]['ID_absFile']);
904 }
905 }
906 // For all flex type relations:
907 if ($vR['type'] == 'flex') {
908 if (is_array($vR['flexFormRels']['file'])) {
909 foreach ($vR['flexFormRels']['file'] as $key => $subList) {
910 foreach ($subList as $subKey => $fI) {
911 $this->export_addFile($fI, $k, $fieldname);
912 // Remove the absolute reference to the file so it doesn't expose absolute paths from source server:
913 unset($this->dat['records'][$k]['rels'][$fieldname]['flexFormRels']['file'][$key][$subKey]['ID_absFile']);
914 }
915 }
916 }
917 // DB oriented soft references in flex form fields:
918 if (is_array($vR['flexFormRels']['softrefs'])) {
919 foreach ($vR['flexFormRels']['softrefs'] as $key => $subList) {
920 foreach ($subList['keys'] as $spKey => $elements) {
921 foreach ($elements as $subKey => $el) {
922 if ($el['subst']['type'] === 'file' && $this->includeSoftref($el['subst']['tokenID'])) {
923 // Create abs path and ID for file:
924 $ID_absFile = GeneralUtility::getFileAbsFileName(PATH_site . $el['subst']['relFileName']);
925 $ID = md5($el['subst']['relFileName']);
926 if ($ID_absFile) {
927 if (!$this->dat['files'][$ID]) {
928 $fI = array(
929 'filename' => PathUtility::basename($ID_absFile),
930 'ID_absFile' => $ID_absFile,
931 'ID' => $ID,
932 'relFileName' => $el['subst']['relFileName']
933 );
934 $this->export_addFile($fI, '_SOFTREF_');
935 }
936 $this->dat['records'][$k]['rels'][$fieldname]['flexFormRels']['softrefs'][$key]['keys'][$spKey][$subKey]['file_ID'] = $ID;
937 }
938 }
939 }
940 }
941 }
942 }
943 }
944 // In any case, if there are soft refs:
945 if (is_array($vR['softrefs']['keys'])) {
946 foreach ($vR['softrefs']['keys'] as $spKey => $elements) {
947 foreach ($elements as $subKey => $el) {
948 if ($el['subst']['type'] === 'file' && $this->includeSoftref($el['subst']['tokenID'])) {
949 // Create abs path and ID for file:
950 $ID_absFile = GeneralUtility::getFileAbsFileName(PATH_site . $el['subst']['relFileName']);
951 $ID = md5($el['subst']['relFileName']);
952 if ($ID_absFile) {
953 if (!$this->dat['files'][$ID]) {
954 $fI = array(
955 'filename' => PathUtility::basename($ID_absFile),
956 'ID_absFile' => $ID_absFile,
957 'ID' => $ID,
958 'relFileName' => $el['subst']['relFileName']
959 );
960 $this->export_addFile($fI, '_SOFTREF_');
961 }
962 $this->dat['records'][$k]['rels'][$fieldname]['softrefs']['keys'][$spKey][$subKey]['file_ID'] = $ID;
963 }
964 }
965 }
966 }
967 }
968 }
969 }
970 }
971
972 /**
973 * This adds all files from sys_file records
974 *
975 * @return void
976 */
977 public function export_addFilesFromSysFilesRecords() {
978 if (!isset($this->dat['header']['records']['sys_file']) || !is_array($this->dat['header']['records']['sys_file'])) {
979 return;
980 }
981 foreach ($this->dat['header']['records']['sys_file'] as $sysFileUid => $_) {
982 $recordData = $this->dat['records']['sys_file:' . $sysFileUid]['data'];
983 $file = ResourceFactory::getInstance()->createFileObject($recordData);
984 $this->export_addSysFile($file);
985 }
986 }
987
988 /**
989 * Adds a files content from a sys file record to the export memory
990 *
991 * @param \TYPO3\CMS\Core\Resource\File $file
992 * @return void
993 */
994 public function export_addSysFile(\TYPO3\CMS\Core\Resource\File $file) {
995 if ($file->getProperty('size') >= $this->maxFileSize) {
996 $this->error('File ' . $file->getPublicUrl() . ' was larger (' . GeneralUtility::formatSize($file->getProperty('size')) . ') than the maxFileSize (' . GeneralUtility::formatSize($this->maxFileSize) . ')! Skipping.');
997 return;
998 }
999 $fileContent = '';
1000 try {
1001 if (!$this->saveFilesOutsideExportFile) {
1002 $fileContent = $file->getContents();
1003 } else {
1004 $file->checkActionPermission('read');
1005 }
1006
1007 } catch (\TYPO3\CMS\Core\Resource\Exception\InsufficientFileAccessPermissionsException $e) {
1008 $this->error('File ' . $file->getPublicUrl() . ': ' . $e->getMessage());
1009 return;
1010 } catch (\TYPO3\CMS\Core\Resource\Exception\IllegalFileExtensionException $e) {
1011 $this->error('File ' . $file->getPublicUrl() . ': ' . $e->getMessage());
1012 return;
1013 }
1014 $fileUid = $file->getUid();
1015 $fileInfo = $file->getStorage()->getFileInfo($file);
1016 // we sadly have to cast it to string here, because the size property is also returning a string
1017 $fileSize = (string)$fileInfo['size'];
1018 if ($fileSize !== $file->getProperty('size')) {
1019 $this->error('File size of ' . $file->getCombinedIdentifier() . ' is not up-to-date in index! File added with current size.');
1020 $this->dat['records']['sys_file:' . $fileUid]['data']['size'] = $fileSize;
1021 }
1022 $fileSha1 = $file->getStorage()->hashFile($file, 'sha1');
1023 if ($fileSha1 !== $file->getProperty('sha1')) {
1024 $this->error('File sha1 hash of ' . $file->getCombinedIdentifier() . ' is not up-to-date in index! File added on current sha1.');
1025 $this->dat['records']['sys_file:' . $fileUid]['data']['sha1'] = $fileSha1;
1026 }
1027
1028 $fileRec = array();
1029 $fileRec['filesize'] = $fileSize;
1030 $fileRec['filename'] = $file->getProperty('name');
1031 $fileRec['filemtime'] = $file->getProperty('modification_date');
1032
1033 // build unique id based on the storage and the file identifier
1034 $fileId = md5($file->getStorage()->getUid() . ':' . $file->getProperty('identifier_hash'));
1035
1036 // Setting this data in the header
1037 $this->dat['header']['files_fal'][$fileId] = $fileRec;
1038
1039 if (!$this->saveFilesOutsideExportFile) {
1040 // ... and finally add the heavy stuff:
1041 $fileRec['content'] = $fileContent;
1042 } else {
1043 GeneralUtility::upload_copy_move($file->getForLocalProcessing(FALSE), $this->getTemporaryFilesPathForExport() . $file->getProperty('sha1'));
1044 }
1045 $fileRec['content_sha1'] = $fileSha1;
1046
1047 $this->dat['files_fal'][$fileId] = $fileRec;
1048 }
1049
1050
1051 /**
1052 * Adds a files content to the export memory
1053 *
1054 * @param array $fI File information with three keys: "filename" = filename without path, "ID_absFile" = absolute filepath to the file (including the filename), "ID" = md5 hash of "ID_absFile". "relFileName" is optional for files attached to records, but mandatory for soft referenced files (since the relFileName determines where such a file should be stored!)
1055 * @param string $recordRef If the file is related to a record, this is the id on the form [table]:[id]. Information purposes only.
1056 * @param string $fieldname If the file is related to a record, this is the field name it was related to. Information purposes only.
1057 * @return void
1058 */
1059 public function export_addFile($fI, $recordRef = '', $fieldname = '') {
1060 if (!@is_file($fI['ID_absFile'])) {
1061 $this->error($fI['ID_absFile'] . ' was not a file! Skipping.');
1062 return;
1063 }
1064 if (filesize($fI['ID_absFile']) >= $this->maxFileSize) {
1065 $this->error($fI['ID_absFile'] . ' was larger (' . GeneralUtility::formatSize(filesize($fI['ID_absFile'])) . ') than the maxFileSize (' . GeneralUtility::formatSize($this->maxFileSize) . ')! Skipping.');
1066 return;
1067 }
1068 $fileInfo = stat($fI['ID_absFile']);
1069 $fileRec = array();
1070 $fileRec['filesize'] = $fileInfo['size'];
1071 $fileRec['filename'] = PathUtility::basename($fI['ID_absFile']);
1072 $fileRec['filemtime'] = $fileInfo['mtime'];
1073 //for internal type file_reference
1074 $fileRec['relFileRef'] = PathUtility::stripPathSitePrefix($fI['ID_absFile']);
1075 if ($recordRef) {
1076 $fileRec['record_ref'] = $recordRef . '/' . $fieldname;
1077 }
1078 if ($fI['relFileName']) {
1079 $fileRec['relFileName'] = $fI['relFileName'];
1080 }
1081 // Setting this data in the header
1082 $this->dat['header']['files'][$fI['ID']] = $fileRec;
1083 // ... and for the recordlisting, why not let us know WHICH relations there was...
1084 if ($recordRef && $recordRef !== '_SOFTREF_') {
1085 $refParts = explode(':', $recordRef, 2);
1086 if (!is_array($this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'])) {
1087 $this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'] = array();
1088 }
1089 $this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'][] = $fI['ID'];
1090 }
1091 $fileMd5 = md5_file($fI['ID_absFile']);
1092 if (!$this->saveFilesOutsideExportFile) {
1093 // ... and finally add the heavy stuff:
1094 $fileRec['content'] = GeneralUtility::getUrl($fI['ID_absFile']);
1095 } else {
1096 GeneralUtility::upload_copy_move($fI['ID_absFile'], $this->getTemporaryFilesPathForExport() . $fileMd5);
1097 }
1098 $fileRec['content_md5'] = $fileMd5;
1099 $this->dat['files'][$fI['ID']] = $fileRec;
1100 // For soft references, do further processing:
1101 if ($recordRef === '_SOFTREF_') {
1102 // RTE files?
1103 if ($RTEoriginal = $this->getRTEoriginalFilename(PathUtility::basename($fI['ID_absFile']))) {
1104 $RTEoriginal_absPath = PathUtility::dirname($fI['ID_absFile']) . '/' . $RTEoriginal;
1105 if (@is_file($RTEoriginal_absPath)) {
1106 $RTEoriginal_ID = md5($RTEoriginal_absPath);
1107 $fileInfo = stat($RTEoriginal_absPath);
1108 $fileRec = array();
1109 $fileRec['filesize'] = $fileInfo['size'];
1110 $fileRec['filename'] = PathUtility::basename($RTEoriginal_absPath);
1111 $fileRec['filemtime'] = $fileInfo['mtime'];
1112 $fileRec['record_ref'] = '_RTE_COPY_ID:' . $fI['ID'];
1113 $this->dat['header']['files'][$fI['ID']]['RTE_ORIG_ID'] = $RTEoriginal_ID;
1114 // Setting this data in the header
1115 $this->dat['header']['files'][$RTEoriginal_ID] = $fileRec;
1116 $fileMd5 = md5_file($RTEoriginal_absPath);
1117 if (!$this->saveFilesOutsideExportFile) {
1118 // ... and finally add the heavy stuff:
1119 $fileRec['content'] = GeneralUtility::getUrl($RTEoriginal_absPath);
1120 } else {
1121 GeneralUtility::upload_copy_move($RTEoriginal_absPath, $this->getTemporaryFilesPathForExport() . $fileMd5);
1122 }
1123 $fileRec['content_md5'] = $fileMd5;
1124 $this->dat['files'][$RTEoriginal_ID] = $fileRec;
1125 } else {
1126 $this->error('RTE original file "' . PathUtility::stripPathSitePrefix($RTEoriginal_absPath) . '" was not found!');
1127 }
1128 }
1129 // Files with external media?
1130 // This is only done with files grabbed by a softreference parser since it is deemed improbable that hard-referenced files should undergo this treatment.
1131 $html_fI = pathinfo(PathUtility::basename($fI['ID_absFile']));
1132 if ($this->includeExtFileResources && GeneralUtility::inList($this->extFileResourceExtensions, strtolower($html_fI['extension']))) {
1133 $uniquePrefix = '###' . md5($GLOBALS['EXEC_TIME']) . '###';
1134 if (strtolower($html_fI['extension']) === 'css') {
1135 $prefixedMedias = explode($uniquePrefix, preg_replace('/(url[[:space:]]*\\([[:space:]]*["\']?)([^"\')]*)(["\']?[[:space:]]*\\))/i', '\\1' . $uniquePrefix . '\\2' . $uniquePrefix . '\\3', $fileRec['content']));
1136 } else {
1137 // html, htm:
1138 $htmlParser = GeneralUtility::makeInstance(\TYPO3\CMS\Core\Html\HtmlParser::class);
1139 $prefixedMedias = explode($uniquePrefix, $htmlParser->prefixResourcePath($uniquePrefix, $fileRec['content'], array(), $uniquePrefix));
1140 }
1141 $htmlResourceCaptured = FALSE;
1142 foreach ($prefixedMedias as $k => $v) {
1143 if ($k % 2) {
1144 $EXTres_absPath = GeneralUtility::resolveBackPath(PathUtility::dirname($fI['ID_absFile']) . '/' . $v);
1145 $EXTres_absPath = GeneralUtility::getFileAbsFileName($EXTres_absPath);
1146 if ($EXTres_absPath && GeneralUtility::isFirstPartOfStr($EXTres_absPath, PATH_site . $this->fileadminFolderName . '/') && @is_file($EXTres_absPath)) {
1147 $htmlResourceCaptured = TRUE;
1148 $EXTres_ID = md5($EXTres_absPath);
1149 $this->dat['header']['files'][$fI['ID']]['EXT_RES_ID'][] = $EXTres_ID;
1150 $prefixedMedias[$k] = '{EXT_RES_ID:' . $EXTres_ID . '}';
1151 // Add file to memory if it is not set already:
1152 if (!isset($this->dat['header']['files'][$EXTres_ID])) {
1153 $fileInfo = stat($EXTres_absPath);
1154 $fileRec = array();
1155 $fileRec['filesize'] = $fileInfo['size'];
1156 $fileRec['filename'] = PathUtility::basename($EXTres_absPath);
1157 $fileRec['filemtime'] = $fileInfo['mtime'];
1158 $fileRec['record_ref'] = '_EXT_PARENT_:' . $fI['ID'];
1159 // Media relative to the HTML file.
1160 $fileRec['parentRelFileName'] = $v;
1161 // Setting this data in the header
1162 $this->dat['header']['files'][$EXTres_ID] = $fileRec;
1163 // ... and finally add the heavy stuff:
1164 $fileRec['content'] = GeneralUtility::getUrl($EXTres_absPath);
1165 $fileRec['content_md5'] = md5($fileRec['content']);
1166 $this->dat['files'][$EXTres_ID] = $fileRec;
1167 }
1168 }
1169 }
1170 }
1171 if ($htmlResourceCaptured) {
1172 $this->dat['files'][$fI['ID']]['tokenizedContent'] = implode('', $prefixedMedias);
1173 }
1174 }
1175 }
1176 }
1177
1178 /**
1179 * If saveFilesOutsideExportFile is enabled, this function returns the path
1180 * where the files referenced in the export are copied to.
1181 *
1182 * @return string
1183 * @throws \RuntimeException
1184 * @see setSaveFilesOutsideExportFile()
1185 */
1186 public function getTemporaryFilesPathForExport() {
1187 if (!$this->saveFilesOutsideExportFile) {
1188 throw new \RuntimeException('You need to set saveFilesOutsideExportFile to TRUE before you want to get the temporary files path for export.', 1401205213);
1189 }
1190 if ($this->temporaryFilesPathForExport === NULL) {
1191 $temporaryFolderName = $this->getTemporaryFolderName();
1192 $this->temporaryFilesPathForExport = $temporaryFolderName . '/';
1193 }
1194 return $this->temporaryFilesPathForExport;
1195 }
1196
1197 /**
1198 *
1199 * @return string
1200 */
1201 protected function getTemporaryFolderName() {
1202 $temporaryPath = PATH_site . 'typo3temp/';
1203 do {
1204 $temporaryFolderName = $temporaryPath . 'export_temp_files_' . mt_rand(1, PHP_INT_MAX);
1205 } while (is_dir($temporaryFolderName));
1206 GeneralUtility::mkdir($temporaryFolderName);
1207 return $temporaryFolderName;
1208 }
1209
1210 /**
1211 * DB relations flattend to 1-dim array.
1212 * The list will be unique, no table/uid combination will appear twice.
1213 *
1214 * @param array $dbrels 2-dim Array of database relations organized by table key
1215 * @return array 1-dim array where entries are table:uid and keys are array with table/id
1216 */
1217 public function flatDBrels($dbrels) {
1218 $list = array();
1219 foreach ($dbrels as $dat) {
1220 if ($dat['type'] == 'db') {
1221 foreach ($dat['itemArray'] as $i) {
1222 $list[$i['table'] . ':' . $i['id']] = $i;
1223 }
1224 }
1225 if ($dat['type'] == 'flex' && is_array($dat['flexFormRels']['db'])) {
1226 foreach ($dat['flexFormRels']['db'] as $subList) {
1227 foreach ($subList as $i) {
1228 $list[$i['table'] . ':' . $i['id']] = $i;
1229 }
1230 }
1231 }
1232 }
1233 return $list;
1234 }
1235
1236 /**
1237 * Soft References flattend to 1-dim array.
1238 *
1239 * @param array $dbrels 2-dim Array of database relations organized by table key
1240 * @return array 1-dim array where entries are arrays with properties of the soft link found and keys are a unique combination of field, spKey, structure path if applicable and token ID
1241 */
1242 public function flatSoftRefs($dbrels) {
1243 $list = array();
1244 foreach ($dbrels as $field => $dat) {
1245 if (is_array($dat['softrefs']['keys'])) {
1246 foreach ($dat['softrefs']['keys'] as $spKey => $elements) {
1247 if (is_array($elements)) {
1248 foreach ($elements as $subKey => $el) {
1249 $lKey = $field . ':' . $spKey . ':' . $subKey;
1250 $list[$lKey] = array_merge(array('field' => $field, 'spKey' => $spKey), $el);
1251 // Add file_ID key to header - slightly "risky" way of doing this because if the calculation
1252 // changes for the same value in $this->records[...] this will not work anymore!
1253 if ($el['subst'] && $el['subst']['relFileName']) {
1254 $list[$lKey]['file_ID'] = md5(PATH_site . $el['subst']['relFileName']);
1255 }
1256 }
1257 }
1258 }
1259 }
1260 if ($dat['type'] == 'flex' && is_array($dat['flexFormRels']['softrefs'])) {
1261 foreach ($dat['flexFormRels']['softrefs'] as $structurePath => $subSoftrefs) {
1262 if (is_array($subSoftrefs['keys'])) {
1263 foreach ($subSoftrefs['keys'] as $spKey => $elements) {
1264 foreach ($elements as $subKey => $el) {
1265 $lKey = $field . ':' . $structurePath . ':' . $spKey . ':' . $subKey;
1266 $list[$lKey] = array_merge(array('field' => $field, 'spKey' => $spKey, 'structurePath' => $structurePath), $el);
1267 // Add file_ID key to header - slightly "risky" way of doing this because if the calculation
1268 // changes for the same value in $this->records[...] this will not work anymore!
1269 if ($el['subst'] && $el['subst']['relFileName']) {
1270 $list[$lKey]['file_ID'] = md5(PATH_site . $el['subst']['relFileName']);
1271 }
1272 }
1273 }
1274 }
1275 }
1276 }
1277 }
1278 return $list;
1279 }
1280
1281 /**
1282 * If include fields for a specific record type are set, the data
1283 * are filtered out with fields are not included in the fields.
1284 *
1285 * @param string $table The record type to be filtered
1286 * @param array $row The data to be filtered
1287 * @return array The filtered record row
1288 */
1289 protected function filterRecordFields($table, array $row) {
1290 if (isset($this->recordTypesIncludeFields[$table])) {
1291 $includeFields = array_unique(array_merge(
1292 $this->recordTypesIncludeFields[$table],
1293 $this->defaultRecordIncludeFields
1294 ));
1295 $newRow = array();
1296 foreach ($row as $key => $value) {
1297 if (in_array($key, $includeFields)) {
1298 $newRow[$key] = $value;
1299 }
1300 }
1301 } else {
1302 $newRow = $row;
1303 }
1304 return $newRow;
1305 }
1306
1307
1308 /**************************
1309 * File Output
1310 *************************/
1311
1312 /**
1313 * This compiles and returns the data content for an exported file
1314 *
1315 * @param string $type Type of output; "xml" gives xml, otherwise serialized array, possibly compressed.
1316 * @return string The output file stream
1317 */
1318 public function compileMemoryToFileContent($type = '') {
1319 if ($type == 'xml') {
1320 $out = $this->createXML();
1321 } else {
1322 $compress = $this->doOutputCompress();
1323 $out = '';
1324 // adding header:
1325 $out .= $this->addFilePart(serialize($this->dat['header']), $compress);
1326 // adding records:
1327 $out .= $this->addFilePart(serialize($this->dat['records']), $compress);
1328 // adding files:
1329 $out .= $this->addFilePart(serialize($this->dat['files']), $compress);
1330 // adding files_fal:
1331 $out .= $this->addFilePart(serialize($this->dat['files_fal']), $compress);
1332 }
1333 return $out;
1334 }
1335
1336 /**
1337 * Creates XML string from input array
1338 *
1339 * @return string XML content
1340 */
1341 public function createXML() {
1342 // Options:
1343 $options = array(
1344 'alt_options' => array(
1345 '/header' => array(
1346 'disableTypeAttrib' => TRUE,
1347 'clearStackPath' => TRUE,
1348 'parentTagMap' => array(
1349 'files' => 'file',
1350 'files_fal' => 'file',
1351 'records' => 'table',
1352 'table' => 'rec',
1353 'rec:rels' => 'relations',
1354 'relations' => 'element',
1355 'filerefs' => 'file',
1356 'pid_lookup' => 'page_contents',
1357 'header:relStaticTables' => 'static_tables',
1358 'static_tables' => 'tablename',
1359 'excludeMap' => 'item',
1360 'softrefCfg' => 'softrefExportMode',
1361 'extensionDependencies' => 'extkey',
1362 'softrefs' => 'softref_element'
1363 ),
1364 'alt_options' => array(
1365 '/pagetree' => array(
1366 'disableTypeAttrib' => TRUE,
1367 'useIndexTagForNum' => 'node',
1368 'parentTagMap' => array(
1369 'node:subrow' => 'node'
1370 )
1371 ),
1372 '/pid_lookup/page_contents' => array(
1373 'disableTypeAttrib' => TRUE,
1374 'parentTagMap' => array(
1375 'page_contents' => 'table'
1376 ),
1377 'grandParentTagMap' => array(
1378 'page_contents/table' => 'item'
1379 )
1380 )
1381 )
1382 ),
1383 '/records' => array(
1384 'disableTypeAttrib' => TRUE,
1385 'parentTagMap' => array(
1386 'records' => 'tablerow',
1387 'tablerow:data' => 'fieldlist',
1388 'tablerow:rels' => 'related',
1389 'related' => 'field',
1390 'field:itemArray' => 'relations',
1391 'field:newValueFiles' => 'filerefs',
1392 'field:flexFormRels' => 'flexform',
1393 'relations' => 'element',
1394 'filerefs' => 'file',
1395 'flexform:db' => 'db_relations',
1396 'flexform:file' => 'file_relations',
1397 'flexform:softrefs' => 'softref_relations',
1398 'softref_relations' => 'structurePath',
1399 'db_relations' => 'path',
1400 'file_relations' => 'path',
1401 'path' => 'element',
1402 'keys' => 'softref_key',
1403 'softref_key' => 'softref_element'
1404 ),
1405 'alt_options' => array(
1406 '/records/tablerow/fieldlist' => array(
1407 'useIndexTagForAssoc' => 'field'
1408 )
1409 )
1410 ),
1411 '/files' => array(
1412 'disableTypeAttrib' => TRUE,
1413 'parentTagMap' => array(
1414 'files' => 'file'
1415 )
1416 ),
1417 '/files_fal' => array(
1418 'disableTypeAttrib' => TRUE,
1419 'parentTagMap' => array(
1420 'files_fal' => 'file'
1421 )
1422 )
1423 )
1424 );
1425 // Creating XML file from $outputArray:
1426 $charset = $this->dat['header']['charset'] ?: 'utf-8';
1427 $XML = '<?xml version="1.0" encoding="' . $charset . '" standalone="yes" ?>' . LF;
1428 $XML .= GeneralUtility::array2xml($this->dat, '', 0, 'T3RecordDocument', 0, $options);
1429 return $XML;
1430 }
1431
1432 /**
1433 * Returns TRUE if the output should be compressed.
1434 *
1435 * @return bool TRUE if compression is possible AND requested.
1436 */
1437 public function doOutputCompress() {
1438 return $this->compress && !$this->dontCompress;
1439 }
1440
1441 /**
1442 * Returns a content part for a filename being build.
1443 *
1444 * @param array $data Data to store in part
1445 * @param bool $compress Compress file?
1446 * @return string Content stream.
1447 */
1448 public function addFilePart($data, $compress = FALSE) {
1449 if ($compress) {
1450 $data = gzcompress($data);
1451 }
1452 return md5($data) . ':' . ($compress ? '1' : '0') . ':' . str_pad(strlen($data), 10, '0', STR_PAD_LEFT) . ':' . $data . ':';
1453 }
1454
1455 /***********************
1456 * Import
1457 ***********************/
1458
1459 /**
1460 * Initialize all settings for the import
1461 *
1462 * @return void
1463 */
1464 protected function initializeImport() {
1465 // Set this flag to indicate that an import is being/has been done.
1466 $this->doesImport = 1;
1467 // Initialize:
1468 // These vars MUST last for the whole section not being cleared. They are used by the method setRelations() which are called at the end of the import session.
1469 $this->import_mapId = array();
1470 $this->import_newId = array();
1471 $this->import_newId_pids = array();
1472 // Temporary files stack initialized:
1473 $this->unlinkFiles = array();
1474 $this->alternativeFileName = array();
1475 $this->alternativeFilePath = array();
1476
1477 $this->initializeStorageObjects();
1478 }
1479
1480 /**
1481 * Initialize the all present storage objects
1482 *
1483 * @return void
1484 */
1485 protected function initializeStorageObjects() {
1486 /** @var $storageRepository \TYPO3\CMS\Core\Resource\StorageRepository */
1487 $storageRepository = GeneralUtility::makeInstance(\TYPO3\CMS\Core\Resource\StorageRepository::class);
1488 $this->storageObjects = $storageRepository->findAll();
1489 }
1490
1491 /**
1492 * Imports the internal data array to $pid.
1493 *
1494 * @param int $pid Page ID in which to import the content
1495 * @return void
1496 */
1497 public function importData($pid) {
1498
1499 $this->initializeImport();
1500
1501 // Write sys_file_storages first
1502 $this->writeSysFileStorageRecords();
1503 // Write sys_file records and write the binary file data
1504 $this->writeSysFileRecords();
1505 // Write records, first pages, then the rest
1506 // Fields with "hard" relations to database, files and flexform fields are kept empty during this run
1507 $this->writeRecords_pages($pid);
1508 $this->writeRecords_records($pid);
1509 // Finally all the file and DB record references must be fixed. This is done after all records have supposedly been written to database:
1510 // $this->import_mapId will indicate two things: 1) that a record WAS written to db and 2) that it has got a new id-number.
1511 $this->setRelations();
1512 // And when all DB relations are in place, we can fix file and DB relations in flexform fields (since data structures often depends on relations to a DS record):
1513 $this->setFlexFormRelations();
1514 // Unlink temporary files:
1515 $this->unlinkTempFiles();
1516 // Finally, traverse all records and process softreferences with substitution attributes.
1517 $this->processSoftReferences();
1518 // After all migrate records using sys_file_reference now
1519 if ($this->legacyImport) {
1520 $this->migrateLegacyImportRecords();
1521 }
1522 }
1523
1524 /**
1525 * Imports the sys_file_storage records from internal data array.
1526 *
1527 * @return void
1528 */
1529 protected function writeSysFileStorageRecords() {
1530 if (!isset($this->dat['header']['records']['sys_file_storage'])) {
1531 return;
1532 }
1533 $sysFileStorageUidsToBeResetToDefaultStorage = array();
1534 foreach ($this->dat['header']['records']['sys_file_storage'] as $sysFileStorageUid => $_) {
1535 $storageRecord = $this->dat['records']['sys_file_storage:' . $sysFileStorageUid]['data'];
1536 // continue with Local, writable and online storage only
1537 if ($storageRecord['driver'] === 'Local' && $storageRecord['is_writable'] && $storageRecord['is_online']) {
1538 $useThisStorageUidInsteadOfTheOneInImport = 0;
1539 /** @var $localStorage \TYPO3\CMS\Core\Resource\ResourceStorage */
1540 foreach ($this->storageObjects as $localStorage) {
1541 // check the available storage for Local, writable and online ones
1542 if ($localStorage->getDriverType() === 'Local' && $localStorage->isWritable() && $localStorage->isOnline()) {
1543 // check if there is already a identical storage present (same pathType and basePath)
1544 $storageRecordConfiguration = ResourceFactory::getInstance()->convertFlexFormDataToConfigurationArray($storageRecord['configuration']);
1545 $localStorageRecordConfiguration = $localStorage->getConfiguration();
1546 if (
1547 $storageRecordConfiguration['pathType'] === $localStorageRecordConfiguration['pathType']
1548 && $storageRecordConfiguration['basePath'] === $localStorageRecordConfiguration['basePath']
1549 ) {
1550 // same storage is already present
1551 $useThisStorageUidInsteadOfTheOneInImport = $localStorage->getUid();
1552 break;
1553 }
1554 }
1555 }
1556 if ($useThisStorageUidInsteadOfTheOneInImport > 0) {
1557 // same storage is already present; map the to be imported one to the present one
1558 $this->import_mapId['sys_file_storage'][$sysFileStorageUid] = $useThisStorageUidInsteadOfTheOneInImport;
1559 } else {
1560 // Local, writable and online storage. Is allowed to be used to later write files in.
1561 $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);
1562 }
1563 } else {
1564 // Storage with non Local drivers could be imported but must not be used to saves files in, because you
1565 // could not be sure, that this is supported. The default storage will be used in this case.
1566 // It could happen that non writable and non online storage will be created as dupes because you could not
1567 // check the detailed configuration options at this point
1568 $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);
1569 $sysFileStorageUidsToBeResetToDefaultStorage[] = $sysFileStorageUid;
1570 }
1571
1572 }
1573
1574 // Importing the added ones
1575 $tce = $this->getNewTCE();
1576 // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.
1577 $tce->reverseOrder = 1;
1578 $tce->isImporting = TRUE;
1579 $tce->start($this->import_data, array());
1580 $tce->process_datamap();
1581 $this->addToMapId($tce->substNEWwithIDs);
1582
1583 $defaultStorageUid = NULL;
1584 // get default storage
1585 $defaultStorage = ResourceFactory::getInstance()->getDefaultStorage();
1586 if ($defaultStorage !== NULL) {
1587 $defaultStorageUid = $defaultStorage->getUid();
1588 }
1589 foreach ($sysFileStorageUidsToBeResetToDefaultStorage as $sysFileStorageUidToBeResetToDefaultStorage) {
1590 $this->import_mapId['sys_file_storage'][$sysFileStorageUidToBeResetToDefaultStorage] = $defaultStorageUid;
1591 }
1592
1593 // unset the sys_file_storage records to prevent a import in writeRecords_records
1594 unset($this->dat['header']['records']['sys_file_storage']);
1595 }
1596
1597 /**
1598 * Imports the sys_file records and the binary files data from internal data array.
1599 *
1600 * @return void
1601 */
1602 protected function writeSysFileRecords() {
1603 if (!isset($this->dat['header']['records']['sys_file'])) {
1604 return;
1605 }
1606 $this->addGeneralErrorsByTable('sys_file');
1607
1608 // fetch fresh storage records from database
1609 $storageRecords = $this->fetchStorageRecords();
1610
1611 $defaultStorage = ResourceFactory::getInstance()->getDefaultStorage();
1612
1613 $sanitizedFolderMappings = array();
1614
1615 foreach ($this->dat['header']['records']['sys_file'] as $sysFileUid => $_) {
1616 $fileRecord = $this->dat['records']['sys_file:' . $sysFileUid]['data'];
1617
1618 $temporaryFile = NULL;
1619 // check if there is the right file already in the local folder
1620 if ($this->filesPathForImport !== NULL) {
1621 if (is_file($this->filesPathForImport . '/' . $fileRecord['sha1']) && sha1_file($this->filesPathForImport . '/' . $fileRecord['sha1']) === $fileRecord['sha1']) {
1622 $temporaryFile = $this->filesPathForImport . '/' . $fileRecord['sha1'];
1623 }
1624 }
1625
1626 // save file to disk
1627 if ($temporaryFile === NULL) {
1628 $fileId = md5($fileRecord['storage'] . ':' . $fileRecord['identifier_hash']);
1629 $temporaryFile = $this->writeTemporaryFileFromData($fileId);
1630 if ($temporaryFile === NULL) {
1631 // error on writing the file. Error message was already added
1632 continue;
1633 }
1634 }
1635
1636 $originalStorageUid = $fileRecord['storage'];
1637 $useStorageFromStorageRecords = FALSE;
1638
1639 // replace storage id, if a alternative one was registered
1640 if (isset($this->import_mapId['sys_file_storage'][$fileRecord['storage']])) {
1641 $fileRecord['storage'] = $this->import_mapId['sys_file_storage'][$fileRecord['storage']];
1642 $useStorageFromStorageRecords = TRUE;
1643 }
1644
1645 if (empty($fileRecord['storage']) && !$this->isFallbackStorage($fileRecord['storage'])) {
1646 // no storage for the file is defined, mostly because of a missing default storage.
1647 $this->error('Error: No storage for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $originalStorageUid . '"');
1648 continue;
1649 }
1650
1651 // using a storage from the local storage is only allowed, if the uid is present in the
1652 // mapping. Only in this case we could be sure, that it's a local, online and writable storage.
1653 if ($useStorageFromStorageRecords && isset($storageRecords[$fileRecord['storage']])) {
1654 /** @var $storage \TYPO3\CMS\Core\Resource\ResourceStorage */
1655 $storage = ResourceFactory::getInstance()->getStorageObject($fileRecord['storage'], $storageRecords[$fileRecord['storage']]);
1656 } elseif ($this->isFallbackStorage($fileRecord['storage'])) {
1657 $storage = ResourceFactory::getInstance()->getStorageObject(0);
1658 } elseif ($defaultStorage !== NULL) {
1659 $storage = $defaultStorage;
1660 } else {
1661 $this->error('Error: No storage available for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1662 continue;
1663 }
1664
1665 $newFile = NULL;
1666
1667 // check, if there is a identical file
1668 try {
1669 if ($storage->hasFile($fileRecord['identifier'])) {
1670 $file = $storage->getFile($fileRecord['identifier']);
1671 if ($file->getSha1() === $fileRecord['sha1']) {
1672 $newFile = $file;
1673 }
1674 }
1675 } catch (Exception $e) {}
1676
1677 if ($newFile === NULL) {
1678
1679 $folderName = PathUtility::dirname(ltrim($fileRecord['identifier'], '/'));
1680 if (in_array($folderName, $sanitizedFolderMappings)) {
1681 $folderName = $sanitizedFolderMappings[$folderName];
1682 }
1683 if (!$storage->hasFolder($folderName)) {
1684 try {
1685 $importFolder = $storage->createFolder($folderName);
1686 if ($importFolder->getIdentifier() !== $folderName && !in_array($folderName, $sanitizedFolderMappings)) {
1687 $sanitizedFolderMappings[$folderName] = $importFolder->getIdentifier();
1688 }
1689 } catch (Exception $e) {
1690 $this->error('Error: Folder could not be created for file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1691 continue;
1692 }
1693 } else {
1694 $importFolder = $storage->getFolder($folderName);
1695 }
1696
1697 try {
1698 /** @var $newFile \TYPO3\CMS\Core\Resource\File */
1699 $newFile = $storage->addFile($temporaryFile, $importFolder, $fileRecord['name']);
1700 } catch (Exception $e) {
1701 $this->error('Error: File could not be added to the storage: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1702 continue;
1703 }
1704
1705 if ($newFile->getSha1() !== $fileRecord['sha1']) {
1706 $this->error('Error: The hash of the written file is not identical to the import data! File could be corrupted! File: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1707 }
1708 }
1709
1710 // save the new uid in the import id map
1711 $this->import_mapId['sys_file'][$fileRecord['uid']] = $newFile->getUid();
1712 $this->fixUidLocalInSysFileReferenceRecords($fileRecord['uid'], $newFile->getUid());
1713
1714 }
1715
1716 // unset the sys_file records to prevent a import in writeRecords_records
1717 unset($this->dat['header']['records']['sys_file']);
1718 }
1719
1720 /**
1721 * Checks if the $storageId is the id of the fallback storage
1722 *
1723 * @param int|string $storageId
1724 * @return bool
1725 */
1726 protected function isFallbackStorage($storageId) {
1727 return $storageId === 0 || $storageId === '0';
1728 }
1729
1730 /**
1731 * Normally the importer works like the following:
1732 * Step 1: import the records with cleared field values of relation fields (see addSingle())
1733 * Step 2: update the records with the right relation ids (see setRelations())
1734 *
1735 * In step 2 the saving fields of type "relation to sys_file_reference" checks the related sys_file_reference
1736 * record (created in step 1) with the FileExtensionFilter for matching file extensions of the related file.
1737 * To make this work correct, the uid_local of sys_file_reference records has to be not empty AND has to
1738 * relate to the correct (imported) sys_file record uid!!!
1739 *
1740 * This is fixed here.
1741 *
1742 * @param int $oldFileUid
1743 * @param int $newFileUid
1744 * @return void
1745 */
1746 protected function fixUidLocalInSysFileReferenceRecords($oldFileUid, $newFileUid) {
1747 if (!isset($this->dat['header']['records']['sys_file_reference'])) {
1748 return;
1749 }
1750
1751 foreach ($this->dat['header']['records']['sys_file_reference'] as $sysFileReferenceUid => $_) {
1752 $fileReferenceRecord = $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'];
1753 if ($fileReferenceRecord['uid_local'] == $oldFileUid) {
1754 $fileReferenceRecord['uid_local'] = $newFileUid;
1755 $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'] = $fileReferenceRecord;
1756 }
1757 }
1758 }
1759
1760 /**
1761 * Initializes the folder for legacy imports as subfolder of backend users default upload folder
1762 *
1763 * @return void
1764 */
1765 protected function initializeLegacyImportFolder() {
1766 /** @var \TYPO3\CMS\Core\Resource\Folder $folder */
1767 $folder = $GLOBALS['BE_USER']->getDefaultUploadFolder();
1768 if ($folder === FALSE) {
1769 $this->error('Error: the backend users default upload folder is missing! No files will be imported!');
1770 }
1771 if (!$folder->hasFolder($this->legacyImportTargetPath)) {
1772 try {
1773 $this->legacyImportFolder = $folder->createFolder($this->legacyImportTargetPath);
1774 } catch (\TYPO3\CMS\Core\Exception $e) {
1775 $this->error('Error: the import folder in the default upload folder could not be created! No files will be imported!');
1776 }
1777 } else {
1778 $this->legacyImportFolder = $folder->getSubFolder($this->legacyImportTargetPath);
1779 }
1780
1781 }
1782
1783 /**
1784 * Fetched fresh storage records from database because the new imported
1785 * ones are not in cached data of the StorageRepository
1786 *
1787 * @return bool|array
1788 */
1789 protected function fetchStorageRecords() {
1790 $whereClause = BackendUtility::BEenableFields('sys_file_storage');
1791 $whereClause .= BackendUtility::deleteClause('sys_file_storage');
1792
1793 $rows = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows(
1794 '*',
1795 'sys_file_storage',
1796 '1=1' . $whereClause,
1797 '',
1798 '',
1799 '',
1800 'uid'
1801 );
1802
1803 return $rows;
1804 }
1805
1806 /**
1807 * Writes the file from import array to temp dir and returns the filename of it.
1808 *
1809 * @param string $fileId
1810 * @param string $dataKey
1811 * @return string Absolute filename of the temporary filename of the file
1812 */
1813 protected function writeTemporaryFileFromData($fileId, $dataKey = 'files_fal') {
1814 $temporaryFilePath = NULL;
1815 if (is_array($this->dat[$dataKey][$fileId])) {
1816 $temporaryFilePathInternal = GeneralUtility::tempnam('import_temp_');
1817 GeneralUtility::writeFile($temporaryFilePathInternal, $this->dat[$dataKey][$fileId]['content']);
1818 clearstatcache();
1819 if (@is_file($temporaryFilePathInternal)) {
1820 $this->unlinkFiles[] = $temporaryFilePathInternal;
1821 if (filesize($temporaryFilePathInternal) == $this->dat[$dataKey][$fileId]['filesize']) {
1822 $temporaryFilePath = $temporaryFilePathInternal;
1823 } else {
1824 $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' had a size (' . filesize($temporaryFilePathInternal) . ') different from the original (' . $this->dat[$dataKey][$fileId]['filesize'] . ')', 1);
1825 }
1826 } else {
1827 $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' was not written as it should have been!', 1);
1828 }
1829 } else {
1830 $this->error('Error: No file found for ID ' . $fileId, 1);
1831 }
1832 return $temporaryFilePath;
1833 }
1834
1835 /**
1836 * Writing pagetree/pages to database:
1837 *
1838 * @param int $pid PID in which to import. If the operation is an update operation, the root of the page tree inside will be moved to this PID unless it is the same as the root page from the import
1839 * @return void
1840 * @see writeRecords_records()
1841 */
1842 public function writeRecords_pages($pid) {
1843 // First, write page structure if any:
1844 if (is_array($this->dat['header']['records']['pages'])) {
1845 $this->addGeneralErrorsByTable('pages');
1846 // $pageRecords is a copy of the pages array in the imported file. Records here are unset one by one when the addSingle function is called.
1847 $pageRecords = $this->dat['header']['records']['pages'];
1848 $this->import_data = array();
1849 // First add page tree if any
1850 if (is_array($this->dat['header']['pagetree'])) {
1851 $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);
1852 foreach ($pagesFromTree as $uid) {
1853 $thisRec = $this->dat['header']['records']['pages'][$uid];
1854 // PID: Set the main $pid, unless a NEW-id is found
1855 $setPid = isset($this->import_newId_pids[$thisRec['pid']]) ? $this->import_newId_pids[$thisRec['pid']] : $pid;
1856 $this->addSingle('pages', $uid, $setPid);
1857 unset($pageRecords[$uid]);
1858 }
1859 }
1860 // Then add all remaining pages not in tree on root level:
1861 if (count($pageRecords)) {
1862 $remainingPageUids = array_keys($pageRecords);
1863 foreach ($remainingPageUids as $pUid) {
1864 $this->addSingle('pages', $pUid, $pid);
1865 }
1866 }
1867 // Now write to database:
1868 $tce = $this->getNewTCE();
1869 $tce->isImporting = TRUE;
1870 $this->callHook('before_writeRecordsPages', array(
1871 'tce' => &$tce,
1872 'data' => &$this->import_data
1873 ));
1874 $tce->suggestedInsertUids = $this->suggestedInsertUids;
1875 $tce->start($this->import_data, array());
1876 $tce->process_datamap();
1877 $this->callHook('after_writeRecordsPages', array(
1878 'tce' => &$tce
1879 ));
1880 // post-processing: Registering new ids (end all tcemain sessions with this)
1881 $this->addToMapId($tce->substNEWwithIDs);
1882 // In case of an update, order pages from the page tree correctly:
1883 if ($this->update && is_array($this->dat['header']['pagetree'])) {
1884 $this->writeRecords_pages_order($pid);
1885 }
1886 }
1887 }
1888
1889 /**
1890 * Organize all updated pages in page tree so they are related like in the import file
1891 * Only used for updates and when $this->dat['header']['pagetree'] is an array.
1892 *
1893 * @param int $pid Page id in which to import
1894 * @return void
1895 * @access private
1896 * @see writeRecords_pages(), writeRecords_records_order()
1897 */
1898 public function writeRecords_pages_order($pid) {
1899 $cmd_data = array();
1900 // Get uid-pid relations and traverse them in order to map to possible new IDs
1901 $pidsFromTree = $this->flatInversePageTree_pid($this->dat['header']['pagetree']);
1902 foreach ($pidsFromTree as $origPid => $newPid) {
1903 if ($newPid >= 0 && $this->dontIgnorePid('pages', $origPid)) {
1904 // If the page had a new id (because it was created) use that instead!
1905 if (substr($this->import_newId_pids[$origPid], 0, 3) === 'NEW') {
1906 if ($this->import_mapId['pages'][$origPid]) {
1907 $mappedPid = $this->import_mapId['pages'][$origPid];
1908 $cmd_data['pages'][$mappedPid]['move'] = $newPid;
1909 }
1910 } else {
1911 $cmd_data['pages'][$origPid]['move'] = $newPid;
1912 }
1913 }
1914 }
1915 // Execute the move commands if any:
1916 if (count($cmd_data)) {
1917 $tce = $this->getNewTCE();
1918 $this->callHook('before_writeRecordsPagesOrder', array(
1919 'tce' => &$tce,
1920 'data' => &$cmd_data
1921 ));
1922 $tce->start(array(), $cmd_data);
1923 $tce->process_cmdmap();
1924 $this->callHook('after_writeRecordsPagesOrder', array(
1925 'tce' => &$tce
1926 ));
1927 }
1928 }
1929
1930 /**
1931 * Write all database records except pages (writtein in writeRecords_pages())
1932 *
1933 * @param int $pid Page id in which to import
1934 * @return void
1935 * @see writeRecords_pages()
1936 */
1937 public function writeRecords_records($pid) {
1938 // Write the rest of the records
1939 $this->import_data = array();
1940 if (is_array($this->dat['header']['records'])) {
1941 foreach ($this->dat['header']['records'] as $table => $recs) {
1942 $this->addGeneralErrorsByTable($table);
1943 if ($table != 'pages') {
1944 foreach ($recs as $uid => $thisRec) {
1945 // PID: Set the main $pid, unless a NEW-id is found
1946 $setPid = isset($this->import_mapId['pages'][$thisRec['pid']]) ? $this->import_mapId['pages'][$thisRec['pid']] : $pid;
1947 if (is_array($GLOBALS['TCA'][$table]) && $GLOBALS['TCA'][$table]['ctrl']['rootLevel']) {
1948 $setPid = 0;
1949 }
1950 // Add record:
1951 $this->addSingle($table, $uid, $setPid);
1952 }
1953 }
1954 }
1955 } else {
1956 $this->error('Error: No records defined in internal data array.');
1957 }
1958 // Now write to database:
1959 $tce = $this->getNewTCE();
1960 $this->callHook('before_writeRecordsRecords', array(
1961 'tce' => &$tce,
1962 'data' => &$this->import_data
1963 ));
1964 $tce->suggestedInsertUids = $this->suggestedInsertUids;
1965 // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.
1966 $tce->reverseOrder = 1;
1967 $tce->isImporting = TRUE;
1968 $tce->start($this->import_data, array());
1969 $tce->process_datamap();
1970 $this->callHook('after_writeRecordsRecords', array(
1971 'tce' => &$tce
1972 ));
1973 // post-processing: Removing files and registering new ids (end all tcemain sessions with this)
1974 $this->addToMapId($tce->substNEWwithIDs);
1975 // In case of an update, order pages from the page tree correctly:
1976 if ($this->update) {
1977 $this->writeRecords_records_order($pid);
1978 }
1979 }
1980
1981 /**
1982 * Organize all updated record to their new positions.
1983 * Only used for updates
1984 *
1985 * @param int $mainPid Main PID into which we import.
1986 * @return void
1987 * @access private
1988 * @see writeRecords_records(), writeRecords_pages_order()
1989 */
1990 public function writeRecords_records_order($mainPid) {
1991 $cmd_data = array();
1992 if (is_array($this->dat['header']['pagetree'])) {
1993 $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);
1994 } else {
1995 $pagesFromTree = array();
1996 }
1997 if (is_array($this->dat['header']['pid_lookup'])) {
1998 foreach ($this->dat['header']['pid_lookup'] as $pid => $recList) {
1999 $newPid = isset($this->import_mapId['pages'][$pid]) ? $this->import_mapId['pages'][$pid] : $mainPid;
2000 if (\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($newPid)) {
2001 foreach ($recList as $tableName => $uidList) {
2002 // If $mainPid===$newPid then we are on root level and we can consider to move pages as well!
2003 // (they will not be in the page tree!)
2004 if (($tableName != 'pages' || !$pagesFromTree[$pid]) && is_array($uidList)) {
2005 $uidList = array_reverse(array_keys($uidList));
2006 foreach ($uidList as $uid) {
2007 if ($this->dontIgnorePid($tableName, $uid)) {
2008 $cmd_data[$tableName][$uid]['move'] = $newPid;
2009 } else {
2010
2011 }
2012 }
2013 }
2014 }
2015 }
2016 }
2017 }
2018 // Execute the move commands if any:
2019 if (count($cmd_data)) {
2020 $tce = $this->getNewTCE();
2021 $this->callHook('before_writeRecordsRecordsOrder', array(
2022 'tce' => &$tce,
2023 'data' => &$cmd_data
2024 ));
2025 $tce->start(array(), $cmd_data);
2026 $tce->process_cmdmap();
2027 $this->callHook('after_writeRecordsRecordsOrder', array(
2028 'tce' => &$tce
2029 ));
2030 }
2031 }
2032
2033 /**
2034 * Adds a single record to the $importData array. Also copies files to tempfolder.
2035 * However all File/DB-references and flexform field contents are set to blank for now!
2036 * That is done with setRelations() later
2037 *
2038 * @param string $table Table name (from import memory)
2039 * @param int $uid Record UID (from import memory)
2040 * @param int $pid Page id
2041 * @return void
2042 * @see writeRecords()
2043 */
2044 public function addSingle($table, $uid, $pid) {
2045 if ($this->import_mode[$table . ':' . $uid] === 'exclude') {
2046 return;
2047 }
2048 $record = $this->dat['records'][$table . ':' . $uid]['data'];
2049 if (is_array($record)) {
2050 if ($this->update && $this->doesRecordExist($table, $uid) && $this->import_mode[$table . ':' . $uid] !== 'as_new') {
2051 $ID = $uid;
2052 } elseif ($table === 'sys_file_metadata' && $record['sys_language_uid'] == '0' && $this->import_mapId['sys_file'][$record['file']]) {
2053 // on adding sys_file records the belonging sys_file_metadata record was also created
2054 // if there is one the record need to be overwritten instead of creating a new one.
2055 $recordInDatabase = $GLOBALS['TYPO3_DB']->exec_SELECTgetSingleRow(
2056 'uid',
2057 'sys_file_metadata',
2058 'file = ' . $this->import_mapId['sys_file'][$record['file']] . ' AND sys_language_uid = 0 AND pid = 0'
2059 );
2060 // if no record could be found, $this->import_mapId['sys_file'][$record['file']] is pointing
2061 // to a file, that was already there, thus a new metadata record should be created
2062 if (is_array($recordInDatabase)) {
2063 $this->import_mapId['sys_file_metadata'][$record['uid']] = $recordInDatabase['uid'];
2064 $ID = $recordInDatabase['uid'];
2065 } else {
2066 $ID = uniqid('NEW', TRUE);
2067 }
2068
2069 } else {
2070 $ID = uniqid('NEW', TRUE);
2071 }
2072 $this->import_newId[$table . ':' . $ID] = array('table' => $table, 'uid' => $uid);
2073 if ($table == 'pages') {
2074 $this->import_newId_pids[$uid] = $ID;
2075 }
2076 // Set main record data:
2077 $this->import_data[$table][$ID] = $record;
2078 $this->import_data[$table][$ID]['tx_impexp_origuid'] = $this->import_data[$table][$ID]['uid'];
2079 // Reset permission data:
2080 if ($table === 'pages') {
2081 // Have to reset the user/group IDs so pages are owned by importing user. Otherwise strange things may happen for non-admins!
2082 unset($this->import_data[$table][$ID]['perms_userid']);
2083 unset($this->import_data[$table][$ID]['perms_groupid']);
2084 }
2085 // PID and UID:
2086 unset($this->import_data[$table][$ID]['uid']);
2087 // Updates:
2088 if (\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($ID)) {
2089 unset($this->import_data[$table][$ID]['pid']);
2090 } else {
2091 // Inserts:
2092 $this->import_data[$table][$ID]['pid'] = $pid;
2093 if (($this->import_mode[$table . ':' . $uid] === 'force_uid' && $this->update || $this->force_all_UIDS) && $GLOBALS['BE_USER']->isAdmin()) {
2094 $this->import_data[$table][$ID]['uid'] = $uid;
2095 $this->suggestedInsertUids[$table . ':' . $uid] = 'DELETE';
2096 }
2097 }
2098 // Setting db/file blank:
2099 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
2100 switch ((string)$config['type']) {
2101 case 'db':
2102
2103 case 'file':
2104 // Fixed later in ->setRelations() [because we need to know ALL newly created IDs before we can map relations!]
2105 // In the meantime we set NO values for relations.
2106 //
2107 // BUT for field uid_local of table sys_file_reference the relation MUST not be cleared here,
2108 // because the value is already the uid of the right imported sys_file record.
2109 // @see fixUidLocalInSysFileReferenceRecords()
2110 // If it's empty or a uid to another record the FileExtensionFilter will throw an exception or
2111 // delete the reference record if the file extension of the related record doesn't match.
2112 if ($table !== 'sys_file_reference' && $field !== 'uid_local') {
2113 $this->import_data[$table][$ID][$field] = '';
2114 }
2115 break;
2116 case 'flex':
2117 // Fixed later in setFlexFormRelations()
2118 // In the meantime we set NO value for flexforms - this is mainly because file references
2119 // inside will not be processed properly; In fact references will point to no file
2120 // or existing files (in which case there will be double-references which is a big problem of course!)
2121 $this->import_data[$table][$ID][$field] = '';
2122 break;
2123 }
2124 }
2125 } elseif ($table . ':' . $uid != 'pages:0') {
2126 // On root level we don't want this error message.
2127 $this->error('Error: no record was found in data array!', 1);
2128 }
2129 }
2130
2131 /**
2132 * Registers the substNEWids in memory.
2133 *
2134 * @param array $substNEWwithIDs From tcemain to be merged into internal mapping variable in this object
2135 * @return void
2136 * @see writeRecords()
2137 */
2138 public function addToMapId($substNEWwithIDs) {
2139 foreach ($this->import_data as $table => $recs) {
2140 foreach ($recs as $id => $value) {
2141 $old_uid = $this->import_newId[$table . ':' . $id]['uid'];
2142 if (isset($substNEWwithIDs[$id])) {
2143 $this->import_mapId[$table][$old_uid] = $substNEWwithIDs[$id];
2144 } elseif ($this->update) {
2145 // Map same ID to same ID....
2146 $this->import_mapId[$table][$old_uid] = $id;
2147 } else {
2148 // if $this->import_mapId contains already the right mapping, skip the error msg.
2149 // See special handling of sys_file_metadata in addSingle() => nothing to do
2150 if (!($table === 'sys_file_metadata' && isset($this->import_mapId[$table][$old_uid]) && $this->import_mapId[$table][$old_uid] == $id)) {
2151 $this->error('Possible error: ' . $table . ':' . $old_uid . ' had no new id assigned to it. This indicates that the record was not added to database during import. Please check changelog!', 1);
2152 }
2153
2154 }
2155 }
2156 }
2157 }
2158
2159 /**
2160 * Returns a new $TCE object
2161 *
2162 * @return DataHandler $TCE object
2163 */
2164 public function getNewTCE() {
2165 $tce = GeneralUtility::makeInstance(\TYPO3\CMS\Core\DataHandling\DataHandler::class);
2166 $tce->stripslashes_values = 0;
2167 $tce->dontProcessTransformations = 1;
2168 $tce->enableLogging = $this->enableLogging;
2169 $tce->alternativeFileName = $this->alternativeFileName;
2170 $tce->alternativeFilePath = $this->alternativeFilePath;
2171 return $tce;
2172 }
2173
2174 /**
2175 * Cleaning up all the temporary files stored in typo3temp/ folder
2176 *
2177 * @return void
2178 */
2179 public function unlinkTempFiles() {
2180 foreach ($this->unlinkFiles as $fileName) {
2181 if (GeneralUtility::isFirstPartOfStr($fileName, PATH_site . 'typo3temp/')) {
2182 GeneralUtility::unlink_tempfile($fileName);
2183 clearstatcache();
2184 if (is_file($fileName)) {
2185 $this->error('Error: ' . $fileName . ' was NOT unlinked as it should have been!', 1);
2186 }
2187 } else {
2188 $this->error('Error: ' . $fileName . ' was not in temp-path. Not removed!', 1);
2189 }
2190 }
2191 $this->unlinkFiles = array();
2192 }
2193
2194 /***************************
2195 * Import / Relations setting
2196 ***************************/
2197
2198 /**
2199 * At the end of the import process all file and DB relations should be set properly (that is relations
2200 * to imported records are all re-created so imported records are correctly related again)
2201 * Relations in flexform fields are processed in setFlexFormRelations() after this function
2202 *
2203 * @return void
2204 * @see setFlexFormRelations()
2205 */
2206 public function setRelations() {
2207 $updateData = array();
2208 // import_newId contains a register of all records that was in the import memorys "records" key
2209 foreach ($this->import_newId as $nId => $dat) {
2210 $table = $dat['table'];
2211 $uid = $dat['uid'];
2212 // original UID - NOT the new one!
2213 // If the record has been written and received a new id, then proceed:
2214 if (is_array($this->import_mapId[$table]) && isset($this->import_mapId[$table][$uid])) {
2215 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
2216 if (is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {
2217 $thisNewPageUid = 0;
2218 if ($this->legacyImport) {
2219 if ($table != 'pages') {
2220 $oldPid = $this->dat['records'][$table . ':' . $uid]['data']['pid'];
2221 $thisNewPageUid = BackendUtility::wsMapId($table, $this->import_mapId['pages'][$oldPid]);
2222 } else {
2223 $thisNewPageUid = $thisNewUid;
2224 }
2225 }
2226 // Traverse relation fields of each record
2227 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
2228 // uid_local of sys_file_reference needs no update because the correct reference uid was already written
2229 // @see ImportExport::fixUidLocalInSysFileReferenceRecords()
2230 if ($table === 'sys_file_reference' && $field === 'uid_local') {
2231 continue;
2232 }
2233 switch ((string)$config['type']) {
2234 case 'db':
2235 if (is_array($config['itemArray']) && count($config['itemArray'])) {
2236 $itemConfig = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
2237 $valArray = $this->setRelations_db($config['itemArray'], $itemConfig);
2238 $updateData[$table][$thisNewUid][$field] = implode(',', $valArray);
2239 }
2240 break;
2241 case 'file':
2242 if (is_array($config['newValueFiles']) && count($config['newValueFiles'])) {
2243 $valArr = array();
2244 foreach ($config['newValueFiles'] as $fI) {
2245 $valArr[] = $this->import_addFileNameToBeCopied($fI);
2246 }
2247 if ($this->legacyImport && $this->legacyImportFolder === NULL && isset($this->legacyImportMigrationTables[$table][$field])) {
2248 // Do nothing - the legacy import folder is missing
2249 } elseif ($this->legacyImport && $this->legacyImportFolder !== NULL && isset($this->legacyImportMigrationTables[$table][$field])) {
2250 $refIds = array();
2251 foreach ($valArr as $tempFile) {
2252 $fileName = $this->alternativeFileName[$tempFile];
2253 $fileObject = NULL;
2254
2255 try {
2256 // check, if there is alreay the same file in the folder
2257 if ($this->legacyImportFolder->hasFile($fileName)) {
2258 $fileStorage = $this->legacyImportFolder->getStorage();
2259 $file = $fileStorage->getFile($this->legacyImportFolder->getIdentifier() . $fileName);
2260 if ($file->getSha1() === sha1_file($tempFile)) {
2261 $fileObject = $file;
2262 }
2263 }
2264 } catch (Exception $e) {}
2265
2266 if ($fileObject === NULL) {
2267 try {
2268 $fileObject = $this->legacyImportFolder->addFile($tempFile, $fileName, 'changeName');
2269 } catch (\TYPO3\CMS\Core\Exception $e) {
2270 $this->error('Error: no file could be added to the storage for file name' . $this->alternativeFileName[$tempFile]);
2271 }
2272 }
2273 if ($fileObject !== NULL) {
2274 $refId = uniqid('NEW', TRUE);
2275 $refIds[] = $refId;
2276 $updateData['sys_file_reference'][$refId] = array(
2277 'uid_local' => $fileObject->getUid(),
2278 'uid_foreign' => $thisNewUid, // uid of your content record
2279 'tablenames' => $table,
2280 'fieldname' => $field,
2281 'pid' => $thisNewPageUid, // parent id of the parent page
2282 'table_local' => 'sys_file',
2283 );
2284 }
2285 }
2286 $updateData[$table][$thisNewUid][$field] = implode(',', $refIds);
2287 if (!empty($this->legacyImportMigrationTables[$table][$field])) {
2288 $this->legacyImportMigrationRecords[$table][$thisNewUid][$field] = $refIds;
2289 }
2290 } else {
2291 $updateData[$table][$thisNewUid][$field] = implode(',', $valArr);
2292 }
2293 }
2294 break;
2295 }
2296 }
2297 } else {
2298 $this->error('Error: no record was found in data array!', 1);
2299 }
2300 } else {
2301 $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')', 1);
2302 }
2303 }
2304 if (count($updateData)) {
2305 $tce = $this->getNewTCE();
2306 $tce->isImporting = TRUE;
2307 $this->callHook('before_setRelation', array(
2308 'tce' => &$tce,
2309 'data' => &$updateData
2310 ));
2311 $tce->start($updateData, array());
2312 $tce->process_datamap();
2313 // Replace the temporary "NEW" ids with the final ones.
2314 foreach ($this->legacyImportMigrationRecords as $table => $records) {
2315 foreach ($records as $uid => $fields) {
2316 foreach ($fields as $field => $referenceIds) {
2317 foreach ($referenceIds as $key => $referenceId) {
2318 $this->legacyImportMigrationRecords[$table][$uid][$field][$key] = $tce->substNEWwithIDs[$referenceId];
2319 }
2320 }
2321 }
2322 }
2323 $this->callHook('after_setRelations', array(
2324 'tce' => &$tce
2325 ));
2326 }
2327 }
2328
2329 /**
2330 * Maps relations for database
2331 *
2332 * @param array $itemArray Array of item sets (table/uid) from a dbAnalysis object
2333 * @param array $itemConfig Array of TCA config of the field the relation to be set on
2334 * @return array Array with values [table]_[uid] or [uid] for field of type group / internal_type file_reference. These values have the regular tcemain-input group/select type which means they will automatically be processed into a uid-list or MM relations.
2335 */
2336 public function setRelations_db($itemArray, $itemConfig) {
2337 $valArray = array();
2338 foreach ($itemArray as $relDat) {
2339 if (is_array($this->import_mapId[$relDat['table']]) && isset($this->import_mapId[$relDat['table']][$relDat['id']])) {
2340 // Since non FAL file relation type group internal_type file_reference are handled as reference to
2341 // sys_file records Datahandler requires the value as uid of the the related sys_file record only
2342 if ($itemConfig['type'] === 'group' && $itemConfig['internal_type'] === 'file_reference') {
2343 $value = $this->import_mapId[$relDat['table']][$relDat['id']];
2344 } else {
2345 $value = $relDat['table'] . '_' . $this->import_mapId[$relDat['table']][$relDat['id']];
2346 }
2347 $valArray[] = $value;
2348 } elseif ($this->isTableStatic($relDat['table']) || $this->isExcluded($relDat['table'], $relDat['id']) || $relDat['id'] < 0) {
2349 // Checking for less than zero because some select types could contain negative values,
2350 // eg. fe_groups (-1, -2) and sys_language (-1 = ALL languages). This must be handled on both export and import.
2351 $valArray[] = $relDat['table'] . '_' . $relDat['id'];
2352 } else {
2353 $this->error('Lost relation: ' . $relDat['table'] . ':' . $relDat['id'], 1);
2354 }
2355 }
2356 return $valArray;
2357 }
2358
2359 /**
2360 * Writes the file from import array to temp dir and returns the filename of it.
2361 *
2362 * @param array $fI File information with three keys: "filename" = filename without path, "ID_absFile" = absolute filepath to the file (including the filename), "ID" = md5 hash of "ID_absFile
2363 * @return string|NULL Absolute filename of the temporary filename of the file. In ->alternativeFileName the original name is set.
2364 */
2365 public function import_addFileNameToBeCopied($fI) {
2366 if (is_array($this->dat['files'][$fI['ID']])) {
2367 $tmpFile = NULL;
2368 // check if there is the right file already in the local folder
2369 if ($this->filesPathForImport !== NULL) {
2370 if (is_file($this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5']) &&
2371 md5_file($this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5']) === $this->dat['files'][$fI['ID']]['content_md5']) {
2372 $tmpFile = $this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5'];
2373 }
2374 }
2375 if ($tmpFile === NULL) {
2376 $tmpFile = GeneralUtility::tempnam('import_temp_');
2377 GeneralUtility::writeFile($tmpFile, $this->dat['files'][$fI['ID']]['content']);
2378 }
2379 clearstatcache();
2380 if (@is_file($tmpFile)) {
2381 $this->unlinkFiles[] = $tmpFile;
2382 if (filesize($tmpFile) == $this->dat['files'][$fI['ID']]['filesize']) {
2383 $this->alternativeFileName[$tmpFile] = $fI['filename'];
2384 $this->alternativeFilePath[$tmpFile] = $this->dat['files'][$fI['ID']]['relFileRef'];
2385 return $tmpFile;
2386 } else {
2387 $this->error('Error: temporary file ' . $tmpFile . ' had a size (' . filesize($tmpFile) . ') different from the original (' . $this->dat['files'][$fI['ID']]['filesize'] . ')', 1);
2388 }
2389 } else {
2390 $this->error('Error: temporary file ' . $tmpFile . ' was not written as it should have been!', 1);
2391 }
2392 } else {
2393 $this->error('Error: No file found for ID ' . $fI['ID'], 1);
2394 }
2395 return NULL;
2396 }
2397
2398 /**
2399 * After all DB relations has been set in the end of the import (see setRelations()) then it is time to correct all relations inside of FlexForm fields.
2400 * The reason for doing this after is that the setting of relations may affect (quite often!) which data structure is used for the flexforms field!
2401 *
2402 * @return void
2403 * @see setRelations()
2404 */
2405 public function setFlexFormRelations() {
2406 $updateData = array();
2407 // import_newId contains a register of all records that was in the import memorys "records" key
2408 foreach ($this->import_newId as $nId => $dat) {
2409 $table = $dat['table'];
2410 $uid = $dat['uid'];
2411 // original UID - NOT the new one!
2412 // If the record has been written and received a new id, then proceed:
2413 if (!isset($this->import_mapId[$table][$uid])) {
2414 $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')', 1);
2415 continue;
2416 }
2417
2418 if (!is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {
2419 $this->error('Error: no record was found in data array!', 1);
2420 continue;
2421 }
2422 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
2423 // Traverse relation fields of each record
2424 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
2425 switch ((string)$config['type']) {
2426 case 'flex':
2427 // Get XML content and set as default value (string, non-processed):
2428 $updateData[$table][$thisNewUid][$field] = $this->dat['records'][$table . ':' . $uid]['data'][$field];
2429 // If there has been registered relations inside the flex form field, run processing on the content:
2430 if (count($config['flexFormRels']['db']) || count($config['flexFormRels']['file'])) {
2431 $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');
2432 // This will fetch the new row for the element (which should be updated with any references to data structures etc.)
2433 $conf = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
2434 if (is_array($origRecordRow) && is_array($conf) && $conf['type'] === 'flex') {
2435 // Get current data structure and value array:
2436 $dataStructArray = BackendUtility::getFlexFormDS($conf, $origRecordRow, $table, $field);
2437 $currentValueArray = GeneralUtility::xml2array($updateData[$table][$thisNewUid][$field]);
2438 // Do recursive processing of the XML data:
2439 $iteratorObj = GeneralUtility::makeInstance(\TYPO3\CMS\Core\DataHandling\DataHandler::class);
2440 $iteratorObj->callBackObj = $this;
2441 $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData(
2442 $currentValueArray['data'],
2443 array(),
2444 array(),
2445 $dataStructArray,
2446 array($table, $thisNewUid, $field, $config),
2447 'remapListedDBRecords_flexFormCallBack'
2448 );
2449 // The return value is set as an array which means it will be processed by tcemain for file and DB references!
2450 if (is_array($currentValueArray['data'])) {
2451 $updateData[$table][$thisNewUid][$field] = $currentValueArray;
2452 }
2453 }
2454 }
2455 break;
2456 }
2457 }
2458 }
2459 if (count($updateData)) {
2460 $tce = $this->getNewTCE();
2461 $tce->isImporting = TRUE;
2462 $this->callHook('before_setFlexFormRelations', array(
2463 'tce' => &$tce,
2464 'data' => &$updateData
2465 ));
2466 $tce->start($updateData, array());
2467 $tce->process_datamap();
2468 $this->callHook('after_setFlexFormRelations', array(
2469 'tce' => &$tce
2470 ));
2471 }
2472 }
2473
2474 /**
2475 * Callback function for traversing the FlexForm structure in relation to remapping database relations
2476 *
2477 * @param array $pParams Set of parameters in numeric array: table, uid, field
2478 * @param array $dsConf TCA config for field (from Data Structure of course)
2479 * @param string $dataValue Field value (from FlexForm XML)
2480 * @param string $dataValue_ext1 Not used
2481 * @param string $dataValue_ext2 Not used
2482 * @param string $path Path of where the data structure of the element is found
2483 * @return array Array where the "value" key carries the value.
2484 * @see setFlexFormRelations()
2485 */
2486 public function remapListedDBRecords_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path) {
2487 // Extract parameters:
2488 list($table, $uid, $field, $config) = $pParams;
2489 // In case the $path is used as index without a trailing slash we will remove that
2490 if (!is_array($config['flexFormRels']['db'][$path]) && is_array($config['flexFormRels']['db'][rtrim($path, '/')])) {
2491 $path = rtrim($path, '/');
2492 }
2493 if (is_array($config['flexFormRels']['db'][$path])) {
2494 $valArray = $this->setRelations_db($config['flexFormRels']['db'][$path], $dsConf);
2495 $dataValue = implode(',', $valArray);
2496 }
2497 if (is_array($config['flexFormRels']['file'][$path])) {
2498 $valArr = array();
2499 foreach ($config['flexFormRels']['file'][$path] as $fI) {
2500 $valArr[] = $this->import_addFileNameToBeCopied($fI);
2501 }
2502 $dataValue = implode(',', $valArr);
2503 }
2504 return array('value' => $dataValue);
2505 }
2506
2507 /**************************
2508 * Import / Soft References
2509 *************************/
2510
2511 /**
2512 * Processing of soft references
2513 *
2514 * @return void
2515 */
2516 public function processSoftReferences() {
2517 // Initialize:
2518 $inData = array();
2519 // Traverse records:
2520 if (is_array($this->dat['header']['records'])) {
2521 foreach ($this->dat['header']['records'] as $table => $recs) {
2522 foreach ($recs as $uid => $thisRec) {
2523 // If there are soft references defined, traverse those:
2524 if (isset($GLOBALS['TCA'][$table]) && is_array($thisRec['softrefs'])) {
2525 // First traversal is to collect softref configuration and split them up based on fields.
2526 // This could probably also have been done with the "records" key instead of the header.
2527 $fieldsIndex = array();
2528 foreach ($thisRec['softrefs'] as $softrefDef) {
2529 // If a substitution token is set:
2530 if ($softrefDef['field'] && is_array($softrefDef['subst']) && $softrefDef['subst']['tokenID']) {
2531 $fieldsIndex[$softrefDef['field']][$softrefDef['subst']['tokenID']] = $softrefDef;
2532 }
2533 }
2534 // The new id:
2535 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
2536 // Now, if there are any fields that require substitution to be done, lets go for that:
2537 foreach ($fieldsIndex as $field => $softRefCfgs) {
2538 if (is_array($GLOBALS['TCA'][$table]['columns'][$field])) {
2539 $conf = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
2540 if ($conf['type'] === 'flex') {
2541 // This will fetch the new row for the element (which should be updated with any references to data structures etc.)
2542 $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');
2543 if (is_array($origRecordRow)) {
2544 // Get current data structure and value array:
2545 $dataStructArray = BackendUtility::getFlexFormDS($conf, $origRecordRow, $table, $field);
2546 $currentValueArray = GeneralUtility::xml2array($origRecordRow[$field]);
2547 // Do recursive processing of the XML data:
2548 /** @var $iteratorObj \TYPO3\CMS\Core\DataHandling\DataHandler */
2549 $iteratorObj = GeneralUtility::makeInstance(\TYPO3\CMS\Core\DataHandling\DataHandler::class);
2550 $iteratorObj->callBackObj = $this;
2551 $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData($currentValueArray['data'], array(), array(), $dataStructArray, array($table, $uid, $field, $softRefCfgs), 'processSoftReferences_flexFormCallBack');
2552 // The return value is set as an array which means it will be processed by tcemain for file and DB references!
2553 if (is_array($currentValueArray['data'])) {
2554 $inData[$table][$thisNewUid][$field] = $currentValueArray;
2555 }
2556 }
2557 } else {
2558 // Get tokenizedContent string and proceed only if that is not blank:
2559 $tokenizedContent = $this->dat['records'][$table . ':' . $uid]['rels'][$field]['softrefs']['tokenizedContent'];
2560 if (strlen($tokenizedContent) && is_array($softRefCfgs)) {
2561 $inData[$table][$thisNewUid][$field] = $this->processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid);
2562 }
2563 }
2564 }
2565 }
2566 }
2567 }
2568 }
2569 }
2570 // Now write to database:
2571 $tce = $this->getNewTCE();
2572 $tce->isImporting = TRUE;
2573 $this->callHook('before_processSoftReferences', array(
2574 'tce' => $tce,
2575 'data' => &$inData
2576 ));
2577 $tce->enableLogging = TRUE;
2578 $tce->start($inData, array());
2579 $tce->process_datamap();
2580 $this->callHook('after_processSoftReferences', array(
2581 'tce' => $tce
2582 ));
2583 }
2584
2585 /**
2586 * Callback function for traversing the FlexForm structure in relation to remapping softreference relations
2587 *
2588 * @param array $pParams Set of parameters in numeric array: table, uid, field
2589 * @param array $dsConf TCA config for field (from Data Structure of course)
2590 * @param string $dataValue Field value (from FlexForm XML)
2591 * @param string $dataValue_ext1 Not used
2592 * @param string $dataValue_ext2 Not used
2593 * @param string $path Path of where the data structure where the element is found
2594 * @return array Array where the "value" key carries the value.
2595 * @see setFlexFormRelations()
2596 */
2597 public function processSoftReferences_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path) {
2598 // Extract parameters:
2599 list($table, $origUid, $field, $softRefCfgs) = $pParams;
2600 if (is_array($softRefCfgs)) {
2601 // First, find all soft reference configurations for this structure path (they are listed flat in the header):
2602 $thisSoftRefCfgList = array();
2603 foreach ($softRefCfgs as $sK => $sV) {
2604 if ($sV['structurePath'] === $path) {
2605 $thisSoftRefCfgList[$sK] = $sV;
2606 }
2607 }
2608 // If any was found, do processing:
2609 if (count($thisSoftRefCfgList)) {
2610 // Get tokenizedContent string and proceed only if that is not blank:
2611 $tokenizedContent = $this->dat['records'][$table . ':' . $origUid]['rels'][$field]['flexFormRels']['softrefs'][$path]['tokenizedContent'];
2612 if (strlen($tokenizedContent)) {
2613 $dataValue = $this->processSoftReferences_substTokens($tokenizedContent, $thisSoftRefCfgList, $table, $origUid);
2614 }
2615 }
2616 }
2617 // Return
2618 return array('value' => $dataValue);
2619 }
2620
2621 /**
2622 * Substition of softreference tokens
2623 *
2624 * @param string $tokenizedContent Content of field with soft reference tokens in.
2625 * @param array $softRefCfgs Soft reference configurations
2626 * @param string $table Table for which the processing occurs
2627 * @param string $uid UID of record from table
2628 * @return string The input content with tokens substituted according to entries in softRefCfgs
2629 */
2630 public function processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid) {
2631 // traverse each softref type for this field:
2632 foreach ($softRefCfgs as $cfg) {
2633 // Get token ID:
2634 $tokenID = $cfg['subst']['tokenID'];
2635 // Default is current token value:
2636 $insertValue = $cfg['subst']['tokenValue'];
2637 // Based on mode:
2638 switch ((string)$this->softrefCfg[$tokenID]['mode']) {
2639 case 'exclude':
2640 // Exclude is a simple passthrough of the value
2641 break;
2642 case 'editable':
2643 // Editable always picks up the value from this input array:
2644 $insertValue = $this->softrefInputValues[$tokenID];
2645 break;
2646 default:
2647 // Mapping IDs/creating files: Based on type, look up new value:
2648 switch ((string)$cfg['subst']['type']) {
2649 case 'file':
2650 // Create / Overwrite file:
2651 $insertValue = $this->processSoftReferences_saveFile($cfg['subst']['relFileName'], $cfg, $table, $uid);
2652 break;
2653 case 'db':
2654 default:
2655 // Trying to map database element if found in the mapID array:
2656 list($tempTable, $tempUid) = explode(':', $cfg['subst']['recordRef']);
2657 if (isset($this->import_mapId[$tempTable][$tempUid])) {
2658 $insertValue = BackendUtility::wsMapId($tempTable, $this->import_mapId[$tempTable][$tempUid]);
2659 // Look if reference is to a page and the original token value was NOT an integer - then we assume is was an alias and try to look up the new one!
2660 if ($tempTable === 'pages' && !\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($cfg['subst']['tokenValue'])) {
2661 $recWithUniqueValue = BackendUtility::getRecord($tempTable, $insertValue, 'alias');
2662 if ($recWithUniqueValue['alias']) {
2663 $insertValue = $recWithUniqueValue['alias'];
2664 }
2665 } elseif (strpos($cfg['subst']['tokenValue'], ':') !== FALSE) {
2666 list($tokenKey, $tokenId) = explode(':', $cfg['subst']['tokenValue']);
2667 $insertValue = $tokenKey . ':' . $insertValue;
2668 }
2669 }
2670 }
2671 }
2672 // Finally, swap the soft reference token in tokenized content with the insert value:
2673 $tokenizedContent = str_replace('{softref:' . $tokenID . '}', $insertValue, $tokenizedContent);
2674 }
2675 return $tokenizedContent;
2676 }
2677
2678 /**
2679 * Process a soft reference file
2680 *
2681 * @param string $relFileName Old Relative filename
2682 * @param array $cfg soft reference configuration array
2683 * @param string $table Table for which the processing occurs
2684 * @param string $uid UID of record from table
2685 * @return string New relative filename (value to insert instead of the softref token)
2686 */
2687 public function processSoftReferences_saveFile($relFileName, $cfg, $table, $uid) {
2688 if ($fileHeaderInfo = $this->dat['header']['files'][$cfg['file_ID']]) {
2689 // Initialize; Get directory prefix for file and find possible RTE filename
2690 $dirPrefix = PathUtility::dirname($relFileName) . '/';
2691 $rteOrigName = $this->getRTEoriginalFilename(PathUtility::basename($relFileName));
2692 // If filename looks like an RTE file, and the directory is in "uploads/", then process as a RTE file!
2693 if ($rteOrigName && GeneralUtility::isFirstPartOfStr($dirPrefix, 'uploads/')) {
2694 // RTE:
2695 // First, find unique RTE file name:
2696 if (@is_dir((PATH_site . $dirPrefix))) {
2697 // From the "original" RTE filename, produce a new "original" destination filename which is unused.
2698 // Even if updated, the image should be unique. Currently the problem with this is that it leaves a lot of unused RTE images...
2699 $fileProcObj = $this->getFileProcObj();
2700 $origDestName = $fileProcObj->getUniqueName($rteOrigName, PATH_site . $dirPrefix);
2701 // Create copy file name:
2702 $pI = pathinfo($relFileName);
2703 $copyDestName = PathUtility::dirname($origDestName) . '/RTEmagicC_' . substr(PathUtility::basename($origDestName), 10) . '.' . $pI['extension'];
2704 if (
2705 !@is_file($copyDestName) && !@is_file($origDestName)
2706 && $origDestName === GeneralUtility::getFileAbsFileName($origDestName)
2707 && $copyDestName === GeneralUtility::getFileAbsFileName($copyDestName)
2708 ) {
2709 if ($this->dat['header']['files'][$fileHeaderInfo['RTE_ORIG_ID']]) {
2710 if ($this->legacyImport) {
2711 $fileName = PathUtility::basename($copyDestName);
2712 $this->writeSysFileResourceForLegacyImport($fileName, $cfg['file_ID']);
2713 $relFileName = $this->filePathMap[$cfg['file_ID']] . '" data-htmlarea-file-uid="' . $fileName . '" data-htmlarea-file-table="sys_file';
2714 // Also save the original file
2715 $originalFileName = PathUtility::basename($origDestName);
2716 $this->writeSysFileResourceForLegacyImport($originalFileName, $fileHeaderInfo['RTE_ORIG_ID']);
2717 } else {
2718 // Write the copy and original RTE file to the respective filenames:
2719 $this->writeFileVerify($copyDestName, $cfg['file_ID'], TRUE);
2720 $this->writeFileVerify($origDestName, $fileHeaderInfo['RTE_ORIG_ID'], TRUE);
2721 // Return the relative path of the copy file name:
2722 return PathUtility::stripPathSitePrefix($copyDestName);
2723 }
2724 } else {
2725 $this->error('ERROR: Could not find original file ID');
2726 }
2727 } else {
2728 $this->error('ERROR: The destination filenames "' . $copyDestName . '" and "' . $origDestName . '" either existed or have non-valid names');
2729 }
2730 } else {
2731 $this->error('ERROR: "' . PATH_site . $dirPrefix . '" was not a directory, so could not process file "' . $relFileName . '"');
2732 }
2733 } elseif (GeneralUtility::isFirstPartOfStr($dirPrefix, $this->fileadminFolderName . '/')) {
2734 // File in fileadmin/ folder:
2735 // Create file (and possible resources)
2736 $newFileName = $this->processSoftReferences_saveFile_createRelFile($dirPrefix, PathUtility::basename($relFileName), $cfg['file_ID'], $table, $uid);
2737 if (strlen($newFileName)) {
2738 $relFileName = $newFileName;
2739 } else {
2740 $this->error('ERROR: No new file created for "' . $relFileName . '"');
2741 }
2742 } else {
2743 $this->error('ERROR: Sorry, cannot operate on non-RTE files which are outside the fileadmin folder.');
2744 }
2745 } else {
2746 $this->error('ERROR: Could not find file ID in header.');
2747 }
2748 // Return (new) filename relative to PATH_site:
2749 return $relFileName;
2750 }
2751
2752 /**
2753 * Create file in directory and return the new (unique) filename
2754 *
2755 * @param string $origDirPrefix Directory prefix, relative, with trailing slash
2756 * @param string $fileName Filename (without path)
2757 * @param string $fileID File ID from import memory
2758 * @param string $table Table for which the processing occurs
2759 * @param string $uid UID of record from table
2760 * @return string|NULL New relative filename, if any
2761 */
2762 public function processSoftReferences_saveFile_createRelFile($origDirPrefix, $fileName, $fileID, $table, $uid) {
2763 // If the fileID map contains an entry for this fileID then just return the relative filename of that entry;
2764 // we don't want to write another unique filename for this one!
2765 if (isset($this->fileIDMap[$fileID])) {
2766 return PathUtility::stripPathSitePrefix($this->fileIDMap[$fileID]);
2767 }
2768 if ($this->legacyImport) {
2769 // set dirPrefix to fileadmin because the right target folder is set and checked for permissions later
2770 $dirPrefix = $this->fileadminFolderName . '/';
2771 } else {
2772 // Verify FileMount access to dir-prefix. Returns the best alternative relative path if any
2773 $dirPrefix = $this->verifyFolderAccess($origDirPrefix);
2774 }
2775 if ($dirPrefix && (!$this->update || $origDirPrefix === $dirPrefix) && $this->checkOrCreateDir($dirPrefix)) {
2776 $fileHeaderInfo = $this->dat['header']['files'][$fileID];
2777 $updMode = $this->update && $this->import_mapId[$table][$uid] === $uid && $this->import_mode[$table . ':' . $uid] !== 'as_new';
2778 // Create new name for file:
2779 // Must have same ID in map array (just for security, is not really needed) and NOT be set "as_new".
2780
2781 // Write main file:
2782 if ($this->legacyImport) {
2783 $fileWritten = $this->writeSysFileResourceForLegacyImport($fileName, $fileID);
2784 if ($fileWritten) {
2785 $newName = 'file:' . $fileName;
2786 return $newName;
2787 // no support for HTML/CSS file resources attached ATM - see below
2788 }
2789 } else {
2790 if ($updMode) {
2791 $newName = PATH_site . $dirPrefix . $fileName;
2792 } else {
2793 // Create unique filename:
2794 $fileProcObj = $this->getFileProcObj();
2795 $newName = $fileProcObj->getUniqueName($fileName, PATH_site . $dirPrefix);
2796 }
2797 if ($this->writeFileVerify($newName, $fileID)) {
2798 // If the resource was an HTML/CSS file with resources attached, we will write those as well!
2799 if (is_array($fileHeaderInfo['EXT_RES_ID'])) {
2800 $tokenizedContent = $this->dat['files'][$fileID]['tokenizedContent'];
2801 $tokenSubstituted = FALSE;
2802 $fileProcObj = $this->getFileProcObj();
2803 if ($updMode) {
2804 foreach ($fileHeaderInfo['EXT_RES_ID'] as $res_fileID) {
2805 if ($this->dat['files'][$res_fileID]['filename']) {
2806 // Resolve original filename:
2807 $relResourceFileName = $this->dat['files'][$res_fileID]['parentRelFileName'];
2808 $absResourceFileName = GeneralUtility::resolveBackPath(PATH_site . $origDirPrefix . $relResourceFileName);
2809 $absResourceFileName = GeneralUtility::getFileAbsFileName($absResourceFileName);
2810 if ($absResourceFileName && GeneralUtility::isFirstPartOfStr($absResourceFileName, PATH_site . $this->fileadminFolderName . '/')) {
2811 $destDir = PathUtility::stripPathSitePrefix(PathUtility::dirname($absResourceFileName) . '/');
2812 if ($this->verifyFolderAccess($destDir, TRUE) && $this->checkOrCreateDir($destDir)) {
2813 $this->writeFileVerify($absResourceFileName, $res_fileID);
2814 } else {
2815 $this->error('ERROR: Could not create file in directory "' . $destDir . '"');
2816 }
2817 } else {
2818 $this->error('ERROR: Could not resolve path for "' . $relResourceFileName . '"');
2819 }
2820 $tokenizedContent = str_replace('{EXT_RES_ID:' . $res_fileID . '}', $relResourceFileName, $tokenizedContent);
2821 $tokenSubstituted = TRUE;
2822 }
2823 }
2824 } else {
2825 // Create the resouces directory name (filename without extension, suffixed "_FILES")
2826 $resourceDir = PathUtility::dirname($newName) . '/' . preg_replace('/\\.[^.]*$/', '', PathUtility::basename($newName)) . '_FILES';
2827 if (GeneralUtility::mkdir($resourceDir)) {
2828 foreach ($fileHeaderInfo['EXT_RES_ID'] as $res_fileID) {
2829 if ($this->dat['files'][$res_fileID]['filename']) {
2830 $absResourceFileName = $fileProcObj->getUniqueName($this->dat['files'][$res_fileID]['filename'], $resourceDir);
2831 $relResourceFileName = substr($absResourceFileName, strlen(PathUtility::dirname($resourceDir)) + 1);
2832 $this->writeFileVerify($absResourceFileName, $res_fileID);
2833 $tokenizedContent = str_replace('{EXT_RES_ID:' . $res_fileID . '}', $relResourceFileName, $tokenizedContent);
2834 $tokenSubstituted = TRUE;
2835 }
2836 }
2837 }
2838 }
2839 // If substitutions has been made, write the content to the file again:
2840 if ($tokenSubstituted) {
2841