55ca715693c107b360dbd522f9b16ea650f0172f
[Packages/TYPO3.CMS.git] / typo3 / sysext / impexp / Classes / ImportExport.php
1 <?php
2 namespace TYPO3\CMS\Impexp;
3
4 /**
5 * This file is part of the TYPO3 CMS project.
6 *
7 * It is free software; you can redistribute it and/or modify it under
8 * the terms of the GNU General Public License, either version 2
9 * of the License, or any later version.
10 *
11 * For the full copyright and license information, please read the
12 * LICENSE.txt file that was distributed with this source code.
13 *
14 * The TYPO3 project - inspiring people to share!
15 */
16
17 use TYPO3\CMS\Backend\Utility\BackendUtility;
18 use TYPO3\CMS\Core\Exception;
19 use TYPO3\CMS\Core\Resource\ResourceFactory;
20 use TYPO3\CMS\Core\Utility\GeneralUtility;
21 use TYPO3\CMS\Core\Utility\PathUtility;
22
23 /**
24 * EXAMPLE for using the impexp-class for exporting stuff:
25 *
26 * Create and initialize:
27 * $this->export = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance('TYPO3\\CMS\\Impexp\\ImportExport');
28 * $this->export->init();
29 * Set which tables relations we will allow:
30 * $this->export->relOnlyTables[]="tt_news"; // exclusively includes. See comment in the class
31 *
32 * Adding records:
33 * $this->export->export_addRecord("pages", $this->pageinfo);
34 * $this->export->export_addRecord("pages", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("pages", 38));
35 * $this->export->export_addRecord("pages", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("pages", 39));
36 * $this->export->export_addRecord("tt_content", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("tt_content", 12));
37 * $this->export->export_addRecord("tt_content", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("tt_content", 74));
38 * $this->export->export_addRecord("sys_template", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("sys_template", 20));
39 *
40 * Adding all the relations (recursively in 5 levels so relations has THEIR relations registered as well)
41 * for($a=0;$a<5;$a++) {
42 * $addR = $this->export->export_addDBRelations($a);
43 * if (!count($addR)) break;
44 * }
45 *
46 * Finally load all the files.
47 * $this->export->export_addFilesFromRelations(); // MUST be after the DBrelations are set so that file from ALL added records are included!
48 *
49 * Write export
50 * $out = $this->export->compileMemoryToFileContent();
51 */
52
53 /**
54 * T3D file Import/Export library (TYPO3 Record Document)
55 *
56 * @author Kasper Skårhøj <kasperYYYY@typo3.com>
57 */
58 class ImportExport {
59
60 /**
61 * If set, static relations (not exported) will be shown in overview as well
62 *
63 * @var bool
64 */
65 public $showStaticRelations = FALSE;
66
67 /**
68 * Name of the "fileadmin" folder where files for export/import should be located
69 *
70 * @var string
71 */
72 public $fileadminFolderName = '';
73
74 /**
75 * Whether "import" or "export" mode of object. Set through init() function
76 *
77 * @var string
78 */
79 public $mode = '';
80
81 /**
82 * Updates all records that has same UID instead of creating new!
83 *
84 * @var bool
85 */
86 public $update = FALSE;
87
88 /**
89 * Is set by importData() when an import has been done.
90 *
91 * @var bool
92 */
93 public $doesImport = FALSE;
94
95 /**
96 * If set to a page-record, then the preview display of the content will expect this page-record to be the target
97 * for the import and accordingly display validation information. This triggers the visual view of the
98 * import/export memory to validate if import is possible
99 *
100 * @var array
101 */
102 public $display_import_pid_record = array();
103
104 /**
105 * Used to register the forged UID values for imported records that we want
106 * to create with the same UIDs as in the import file. Admin-only feature.
107 *
108 * @var array
109 */
110 public $suggestedInsertUids = array();
111
112 /**
113 * Setting import modes during update state: as_new, exclude, force_uid
114 *
115 * @var array
116 */
117 public $import_mode = array();
118
119 /**
120 * If set, PID correct is ignored globally
121 *
122 * @var bool
123 */
124 public $global_ignore_pid = FALSE;
125
126 /**
127 * If set, all UID values are forced! (update or import)
128 *
129 * @var bool
130 */
131 public $force_all_UIDS = FALSE;
132
133 /**
134 * If set, a diff-view column is added to the overview.
135 *
136 * @var bool
137 */
138 public $showDiff = FALSE;
139
140 /**
141 * If set, and if the user is admin, allow the writing of PHP scripts to fileadmin/ area.
142 *
143 * @var bool
144 */
145 public $allowPHPScripts = FALSE;
146
147 /**
148 * Disable logging when importing
149 *
150 * @var bool
151 */
152 public $enableLogging = FALSE;
153
154 /**
155 * Array of values to substitute in editable softreferences.
156 *
157 * @var array
158 */
159 public $softrefInputValues = array();
160
161 /**
162 * Mapping between the fileID from import memory and the final filenames they are written to.
163 *
164 * @var array
165 */
166 public $fileIDMap = array();
167
168 /**
169 * 1MB max file size
170 *
171 * @var int
172 */
173 public $maxFileSize = 1000000;
174
175 /**
176 * 1MB max record size
177 *
178 * @var int
179 */
180 public $maxRecordSize = 1000000;
181
182 /**
183 * 10MB max export size
184 *
185 * @var int
186 */
187 public $maxExportSize = 10000000;
188
189 /**
190 * Add table names here which are THE ONLY ones which will be included
191 * into export if found as relations. '_ALL' will allow all tables.
192 *
193 * @var array
194 */
195 public $relOnlyTables = array();
196
197 /**
198 * Add tables names here which should not be exported with the file.
199 * (Where relations should be mapped to same UIDs in target system).
200 *
201 * @var array
202 */
203 public $relStaticTables = array();
204
205 /**
206 * Exclude map. Keys are table:uid pairs and if set, records are not added to the export.
207 *
208 * @var array
209 */
210 public $excludeMap = array();
211
212 /**
213 * Soft Reference Token ID modes.
214 *
215 * @var array
216 */
217 public $softrefCfg = array();
218
219 /**
220 * Listing extension dependencies.
221 *
222 * @var array
223 */
224 public $extensionDependencies = array();
225
226 /**
227 * Set by user: If set, compression in t3d files is disabled
228 *
229 * @var bool
230 */
231 public $dontCompress = FALSE;
232
233 /**
234 * If set, HTML file resources are included.
235 *
236 * @var bool
237 */
238 public $includeExtFileResources = FALSE;
239
240 /**
241 * Files with external media (HTML/css style references inside)
242 *
243 * @var string
244 */
245 public $extFileResourceExtensions = 'html,htm,css';
246
247 /**
248 * After records are written this array is filled with [table][original_uid] = [new_uid]
249 *
250 * @var array
251 */
252 public $import_mapId = array();
253
254 /**
255 * Keys are [tablename]:[new NEWxxx ids (or when updating it is uids)]
256 * while values are arrays with table/uid of the original record it is based on.
257 * With the array keys the new ids can be looked up inside tcemain
258 *
259 * @var array
260 */
261 public $import_newId = array();
262
263 /**
264 * Page id map for page tree (import)
265 *
266 * @var array
267 */
268 public $import_newId_pids = array();
269
270 /**
271 * Internal data accumulation for writing records during import
272 *
273 * @var array
274 */
275 public $import_data = array();
276
277 /**
278 * Error log.
279 *
280 * @var array
281 */
282 public $errorLog = array();
283
284 /**
285 * Cache for record paths
286 *
287 * @var array
288 */
289 public $cache_getRecordPath = array();
290
291 /**
292 * Cache of checkPID values.
293 *
294 * @var array
295 */
296 public $checkPID_cache = array();
297
298 /**
299 * Set internally if the gzcompress function exists
300 * Used by ImportExportController
301 *
302 * @var bool
303 */
304 public $compress = FALSE;
305
306 /**
307 * Internal import/export memory
308 *
309 * @var array
310 */
311 public $dat = array();
312
313 /**
314 * File processing object
315 *
316 * @var \TYPO3\CMS\Core\Utility\File\ExtendedFileUtility
317 */
318 protected $fileProcObj = NULL;
319
320 /**
321 * Keys are [recordname], values are an array of fields to be included
322 * in the export
323 *
324 * @var array
325 */
326 protected $recordTypesIncludeFields = array();
327
328 /**
329 * Default array of fields to be included in the export
330 *
331 * @var array
332 */
333 protected $defaultRecordIncludeFields = array('uid', 'pid');
334
335 /**
336 * Array of current registered storage objects
337 *
338 * @var array
339 */
340 protected $storageObjects = array();
341
342 /**
343 * Is set, if the import file has a TYPO3 version below 6.0
344 *
345 * @var bool
346 */
347 protected $legacyImport = FALSE;
348
349 /**
350 * @var \TYPO3\CMS\Core\Resource\Folder
351 */
352 protected $legacyImportFolder = NULL;
353
354 /**
355 * Related to the default storage root
356 *
357 * @var string
358 */
359 protected $legacyImportTargetPath = '_imported/';
360
361 /**
362 * Table fields to migrate
363 *
364 * @var array
365 */
366 protected $legacyImportMigrationTables = array(
367 'tt_content' => array(
368 'image' => array(
369 'titleTexts' => 'titleText',
370 'description' => 'imagecaption',
371 'links' => 'image_link',
372 'alternativeTexts' => 'altText'
373 ),
374 'media' => array(
375 'description' => 'imagecaption',
376 )
377 ),
378 'pages' => array(
379 'media' => array()
380 ),
381 'pages_language_overlay' => array(
382 'media' => array()
383 )
384 );
385
386
387 /**
388 * Records to be migrated after all
389 * Multidimensional array [table][uid][field] = array([related sys_file_reference uids])
390 *
391 * @var array
392 */
393 protected $legacyImportMigrationRecords = array();
394
395 /**
396 * @var bool
397 */
398 protected $saveFilesOutsideExportFile = FALSE;
399
400 /**
401 * @var NULL|string
402 */
403 protected $temporaryFilesPathForExport = NULL;
404
405 /**
406 * @var NULL|string
407 */
408 protected $filesPathForImport = NULL;
409
410 /**
411 * @var array
412 */
413 protected $unlinkFiles = array();
414
415 /**
416 * @var array
417 */
418 protected $alternativeFileName = array();
419
420 /**
421 * @var array
422 */
423 protected $alternativeFilePath = array();
424
425 /**
426 * @var array
427 */
428 protected $filePathMap = array();
429
430 /**
431 * @var array
432 */
433 protected $remainHeader = array();
434
435 /**************************
436 * Initialize
437 *************************/
438
439 /**
440 * Init the object, both import and export
441 *
442 * @param bool $dontCompress If set, compression of t3d files is disabled
443 * @param string $mode Mode of usage, either "import" or "export
444 * @return void
445 */
446 public function init($dontCompress = FALSE, $mode = '') {
447 $this->compress = function_exists('gzcompress');
448 $this->dontCompress = $dontCompress;
449 $this->mode = $mode;
450 $this->fileadminFolderName = !empty($GLOBALS['TYPO3_CONF_VARS']['BE']['fileadminDir']) ? rtrim($GLOBALS['TYPO3_CONF_VARS']['BE']['fileadminDir'], '/') : 'fileadmin';
451 }
452
453 /**************************
454 * Export / Init + Meta Data
455 *************************/
456
457 /**
458 * Set header basics
459 *
460 * @return void
461 */
462 public function setHeaderBasics() {
463 // Initializing:
464 if (is_array($this->softrefCfg)) {
465 foreach ($this->softrefCfg as $key => $value) {
466 if (!strlen($value['mode'])) {
467 unset($this->softrefCfg[$key]);
468 }
469 }
470 }
471 // Setting in header memory:
472 // Version of file format
473 $this->dat['header']['XMLversion'] = '1.0';
474 // Initialize meta data array (to put it in top of file)
475 $this->dat['header']['meta'] = array();
476 // Add list of tables to consider static
477 $this->dat['header']['relStaticTables'] = $this->relStaticTables;
478 // The list of excluded records
479 $this->dat['header']['excludeMap'] = $this->excludeMap;
480 // Soft Reference mode for elements
481 $this->dat['header']['softrefCfg'] = $this->softrefCfg;
482 // List of extensions the import depends on.
483 $this->dat['header']['extensionDependencies'] = $this->extensionDependencies;
484 }
485
486 /**
487 * Set charset
488 *
489 * @param string $charset Charset for the content in the export. During import the character set will be converted if the target system uses another charset.
490 * @return void
491 */
492 public function setCharset($charset) {
493 $this->dat['header']['charset'] = $charset;
494 }
495
496 /**
497 * Sets meta data
498 *
499 * @param string $title Title of the export
500 * @param string $description Description of the export
501 * @param string $notes Notes about the contents
502 * @param string $packager_username Backend Username of the packager (the guy making the export)
503 * @param string $packager_name Real name of the packager
504 * @param string $packager_email Email of the packager
505 * @return void
506 */
507 public function setMetaData($title, $description, $notes, $packager_username, $packager_name, $packager_email) {
508 $this->dat['header']['meta'] = array(
509 'title' => $title,
510 'description' => $description,
511 'notes' => $notes,
512 'packager_username' => $packager_username,
513 'packager_name' => $packager_name,
514 'packager_email' => $packager_email,
515 'TYPO3_version' => TYPO3_version,
516 'created' => strftime('%A %e. %B %Y', $GLOBALS['EXEC_TIME'])
517 );
518 }
519
520 /**
521 * Option to enable having the files not included in the export file.
522 * The files are saved to a temporary folder instead.
523 *
524 * @param bool $saveFilesOutsideExportFile
525 * @see getTemporaryFilesPathForExport()
526 */
527 public function setSaveFilesOutsideExportFile($saveFilesOutsideExportFile) {
528 $this->saveFilesOutsideExportFile = $saveFilesOutsideExportFile;
529 }
530
531 /**
532 * Sets a thumbnail image to the exported file
533 *
534 * @param string $imgFilepath Filename reference, gif, jpg, png. Absolute path.
535 * @return void
536 */
537 public function addThumbnail($imgFilepath) {
538 if (@is_file($imgFilepath)) {
539 $imgInfo = @getimagesize($imgFilepath);
540 if (is_array($imgInfo)) {
541 $fileContent = GeneralUtility::getUrl($imgFilepath);
542 $this->dat['header']['thumbnail'] = array(
543 'imgInfo' => $imgInfo,
544 'content' => $fileContent,
545 'filesize' => strlen($fileContent),
546 'filemtime' => filemtime($imgFilepath),
547 'filename' => PathUtility::basename($imgFilepath)
548 );
549 }
550 }
551 }
552
553 /**************************
554 * Export / Init Page tree
555 *************************/
556
557 /**
558 * Sets the page-tree array in the export header and returns the array in a flattened version
559 *
560 * @param array $idH Hierarchy of ids, the page tree: array([uid] => array("uid" => [uid], "subrow" => array(.....)), [uid] => ....)
561 * @return array The hierarchical page tree converted to a one-dimensional list of pages
562 */
563 public function setPageTree($idH) {
564 $this->dat['header']['pagetree'] = $this->unsetExcludedSections($idH);
565 return $this->flatInversePageTree($this->dat['header']['pagetree']);
566 }
567
568 /**
569 * Removes entries in the page tree which are found in ->excludeMap[]
570 *
571 * @param array $idH Page uid hierarchy
572 * @return array Modified input array
573 * @access private
574 * @see setPageTree()
575 */
576 public function unsetExcludedSections($idH) {
577 if (is_array($idH)) {
578 foreach ($idH as $k => $v) {
579 if ($this->excludeMap['pages:' . $idH[$k]['uid']]) {
580 unset($idH[$k]);
581 } elseif (is_array($idH[$k]['subrow'])) {
582 $idH[$k]['subrow'] = $this->unsetExcludedSections($idH[$k]['subrow']);
583 }
584 }
585 }
586 return $idH;
587 }
588
589 /**
590 * Recursively flattening the idH array (for setPageTree() function)
591 *
592 * @param array $idH Page uid hierarchy
593 * @param array $a Accumulation array of pages (internal, don't set from outside)
594 * @return array Array with uid-uid pairs for all pages in the page tree.
595 * @see flatInversePageTree_pid()
596 */
597 public function flatInversePageTree($idH, $a = array()) {
598 if (is_array($idH)) {
599 $idH = array_reverse($idH);
600 foreach ($idH as $k => $v) {
601 $a[$v['uid']] = $v['uid'];
602 if (is_array($v['subrow'])) {
603 $a = $this->flatInversePageTree($v['subrow'], $a);
604 }
605 }
606 }
607 return $a;
608 }
609
610 /**
611 * Recursively flattening the idH array (for setPageTree() function), setting PIDs as values
612 *
613 * @param array $idH Page uid hierarchy
614 * @param array $a Accumulation array of pages (internal, don't set from outside)
615 * @param int $pid PID value (internal)
616 * @return array Array with uid-pid pairs for all pages in the page tree.
617 * @see flatInversePageTree()
618 */
619 public function flatInversePageTree_pid($idH, $a = array(), $pid = -1) {
620 if (is_array($idH)) {
621 $idH = array_reverse($idH);
622 foreach ($idH as $v) {
623 $a[$v['uid']] = $pid;
624 if (is_array($v['subrow'])) {
625 $a = $this->flatInversePageTree_pid($v['subrow'], $a, $v['uid']);
626 }
627 }
628 }
629 return $a;
630 }
631
632 /**************************
633 * Export
634 *************************/
635
636 /**
637 * Sets the fields of record types to be included in the export
638 *
639 * @param array $recordTypesIncludeFields Keys are [recordname], values are an array of fields to be included in the export
640 * @throws \TYPO3\CMS\Core\Exception if an array value is not type of array
641 * @return void
642 */
643 public function setRecordTypesIncludeFields(array $recordTypesIncludeFields) {
644 foreach ($recordTypesIncludeFields as $table => $fields) {
645 if (!is_array($fields)) {
646 throw new \TYPO3\CMS\Core\Exception('The include fields for record type ' . htmlspecialchars($table) . ' are not defined by an array.', 1391440658);
647 }
648 $this->setRecordTypeIncludeFields($table, $fields);
649 }
650 }
651
652 /**
653 * Sets the fields of a record type to be included in the export
654 *
655 * @param string $table The record type
656 * @param array $fields The fields to be included
657 * @return void
658 */
659 public function setRecordTypeIncludeFields($table, array $fields) {
660 $this->recordTypesIncludeFields[$table] = $fields;
661 }
662
663 /**
664 * Adds the record $row from $table.
665 * No checking for relations done here. Pure data.
666 *
667 * @param string $table Table name
668 * @param array $row Record row.
669 * @param int $relationLevel (Internal) if the record is added as a relation, this is set to the "level" it was on.
670 * @return void
671 */
672 public function export_addRecord($table, $row, $relationLevel = 0) {
673 BackendUtility::workspaceOL($table, $row);
674 if ((string)$table !== '' && is_array($row) && $row['uid'] > 0 && !$this->excludeMap[($table . ':' . $row['uid'])]) {
675 if ($this->checkPID($table === 'pages' ? $row['uid'] : $row['pid'])) {
676 if (!isset($this->dat['records'][($table . ':' . $row['uid'])])) {
677 // Prepare header info:
678 $row = $this->filterRecordFields($table, $row);
679 $headerInfo = array();
680 $headerInfo['uid'] = $row['uid'];
681 $headerInfo['pid'] = $row['pid'];
682 $headerInfo['title'] = GeneralUtility::fixed_lgd_cs(BackendUtility::getRecordTitle($table, $row), 40);
683 $headerInfo['size'] = strlen(serialize($row));
684 if ($relationLevel) {
685 $headerInfo['relationLevel'] = $relationLevel;
686 }
687 // If record content is not too large in size, set the header content and add the rest:
688 if ($headerInfo['size'] < $this->maxRecordSize) {
689 // Set the header summary:
690 $this->dat['header']['records'][$table][$row['uid']] = $headerInfo;
691 // Create entry in the PID lookup:
692 $this->dat['header']['pid_lookup'][$row['pid']][$table][$row['uid']] = 1;
693 // Initialize reference index object:
694 $refIndexObj = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\Database\\ReferenceIndex');
695 // Yes to workspace overlays for exporting....
696 $refIndexObj->WSOL = TRUE;
697 $relations = $refIndexObj->getRelations($table, $row);
698 $relations = $this->fixFileIDsInRelations($relations);
699 // Data:
700 $this->dat['records'][$table . ':' . $row['uid']] = array();
701 $this->dat['records'][$table . ':' . $row['uid']]['data'] = $row;
702 $this->dat['records'][$table . ':' . $row['uid']]['rels'] = $relations;
703 $this->errorLog = array_merge($this->errorLog, $refIndexObj->errorLog);
704 // Merge error logs.
705 // Add information about the relations in the record in the header:
706 $this->dat['header']['records'][$table][$row['uid']]['rels'] = $this->flatDBrels($this->dat['records'][$table . ':' . $row['uid']]['rels']);
707 // Add information about the softrefs to header:
708 $this->dat['header']['records'][$table][$row['uid']]['softrefs'] = $this->flatSoftRefs($this->dat['records'][$table . ':' . $row['uid']]['rels']);
709 } else {
710 $this->error('Record ' . $table . ':' . $row['uid'] . ' was larger than maxRecordSize (' . GeneralUtility::formatSize($this->maxRecordSize) . ')');
711 }
712 } else {
713 $this->error('Record ' . $table . ':' . $row['uid'] . ' already added.');
714 }
715 } else {
716 $this->error('Record ' . $table . ':' . $row['uid'] . ' was outside your DB mounts!');
717 }
718 }
719 }
720
721 /**
722 * This changes the file reference ID from a hash based on the absolute file path
723 * (coming from ReferenceIndex) to a hash based on the relative file path.
724 *
725 * @param array $relations
726 * @return array
727 */
728 protected function fixFileIDsInRelations(array $relations) {
729 foreach ($relations as $field => $relation) {
730 if (isset($relation['type']) && $relation['type'] === 'file') {
731 foreach ($relation['newValueFiles'] as $key => $fileRelationData) {
732 $absoluteFilePath = $fileRelationData['ID_absFile'];
733 if (GeneralUtility::isFirstPartOfStr($absoluteFilePath, PATH_site)) {
734 $relatedFilePath = PathUtility::stripPathSitePrefix($absoluteFilePath);
735 $relations[$field]['newValueFiles'][$key]['ID'] = md5($relatedFilePath);
736 }
737 }
738 }
739 if ($relation['type'] === 'flex') {
740 if (is_array($relation['flexFormRels']['file'])) {
741 foreach ($relation['flexFormRels']['file'] as $key => $subList) {
742 foreach ($subList as $subKey => $fileRelationData) {
743 $absoluteFilePath = $fileRelationData['ID_absFile'];
744 if (GeneralUtility::isFirstPartOfStr($absoluteFilePath, PATH_site)) {
745 $relatedFilePath = PathUtility::stripPathSitePrefix($absoluteFilePath);
746 $relations[$field]['flexFormRels']['file'][$key][$subKey]['ID'] = md5($relatedFilePath);
747 }
748 }
749 }
750 }
751 }
752 }
753 return $relations;
754 }
755
756
757
758 /**
759 * This analyses the existing added records, finds all database relations to records and adds these records to the export file.
760 * This function can be called repeatedly until it returns an empty array.
761 * In principle it should not allow to infinite recursivity, but you better set a limit...
762 * Call this BEFORE the ext_addFilesFromRelations (so files from added relations are also included of course)
763 *
764 * @param int $relationLevel Recursion level
765 * @return array overview of relations found and added: Keys [table]:[uid], values array with table and id
766 * @see export_addFilesFromRelations()
767 */
768 public function export_addDBRelations($relationLevel = 0) {
769 // Traverse all "rels" registered for "records"
770 if (!is_array($this->dat['records'])) {
771 $this->error('There were no records available.');
772 return array();
773 }
774 $addR = array();
775 foreach ($this->dat['records'] as $k => $value) {
776 if (!is_array($this->dat['records'][$k])) {
777 continue;
778 }
779 foreach ($this->dat['records'][$k]['rels'] as $fieldname => $vR) {
780 // For all DB types of relations:
781 if ($vR['type'] == 'db') {
782 foreach ($vR['itemArray'] as $fI) {
783 $this->export_addDBRelations_registerRelation($fI, $addR);
784 }
785 }
786 // For all flex/db types of relations:
787 if ($vR['type'] == 'flex') {
788 // DB relations in flex form fields:
789 if (is_array($vR['flexFormRels']['db'])) {
790 foreach ($vR['flexFormRels']['db'] as $subList) {
791 foreach ($subList as $fI) {
792 $this->export_addDBRelations_registerRelation($fI, $addR);
793 }
794 }
795 }
796 // DB oriented soft references in flex form fields:
797 if (is_array($vR['flexFormRels']['softrefs'])) {
798 foreach ($vR['flexFormRels']['softrefs'] as $subList) {
799 foreach ($subList['keys'] as $spKey => $elements) {
800 foreach ($elements as $el) {
801 if ($el['subst']['type'] === 'db' && $this->includeSoftref($el['subst']['tokenID'])) {
802 list($tempTable, $tempUid) = explode(':', $el['subst']['recordRef']);
803 $fI = array(
804 'table' => $tempTable,
805 'id' => $tempUid
806 );
807 $this->export_addDBRelations_registerRelation($fI, $addR, $el['subst']['tokenID']);
808 }
809 }
810 }
811 }
812 }
813 }
814 // In any case, if there are soft refs:
815 if (is_array($vR['softrefs']['keys'])) {
816 foreach ($vR['softrefs']['keys'] as $spKey => $elements) {
817 foreach ($elements as $el) {
818 if ($el['subst']['type'] === 'db' && $this->includeSoftref($el['subst']['tokenID'])) {
819 list($tempTable, $tempUid) = explode(':', $el['subst']['recordRef']);
820 $fI = array(
821 'table' => $tempTable,
822 'id' => $tempUid
823 );
824 $this->export_addDBRelations_registerRelation($fI, $addR, $el['subst']['tokenID']);
825 }
826 }
827 }
828 }
829 }
830 }
831
832 // Now, if there were new records to add, do so:
833 if (!empty($addR)) {
834 foreach ($addR as $fI) {
835 // Get and set record:
836 $row = BackendUtility::getRecord($fI['table'], $fI['id']);
837 if (is_array($row)) {
838 $this->export_addRecord($fI['table'], $row, $relationLevel + 1);
839 }
840 // Set status message
841 // Relation pointers always larger than zero except certain "select" types with
842 // negative values pointing to uids - but that is not supported here.
843 if ($fI['id'] > 0) {
844 $rId = $fI['table'] . ':' . $fI['id'];
845 if (!isset($this->dat['records'][$rId])) {
846 $this->dat['records'][$rId] = 'NOT_FOUND';
847 $this->error('Relation record ' . $rId . ' was not found!');
848 }
849 }
850 }
851 }
852 // Return overview of relations found and added
853 return $addR;
854 }
855
856 /**
857 * Helper function for export_addDBRelations()
858 *
859 * @param array $fI Array with table/id keys to add
860 * @param array $addR Add array, passed by reference to be modified
861 * @param string $tokenID Softref Token ID, if applicable.
862 * @return void
863 * @see export_addDBRelations()
864 */
865 public function export_addDBRelations_registerRelation($fI, &$addR, $tokenID = '') {
866 $rId = $fI['table'] . ':' . $fI['id'];
867 if (
868 isset($GLOBALS['TCA'][$fI['table']]) && !$this->isTableStatic($fI['table']) && !$this->isExcluded($fI['table'], $fI['id'])
869 && (!$tokenID || $this->includeSoftref($tokenID)) && $this->inclRelation($fI['table'])
870 ) {
871 if (!isset($this->dat['records'][$rId])) {
872 // Set this record to be included since it is not already.
873 $addR[$rId] = $fI;
874 }
875 }
876 }
877
878 /**
879 * This adds all files in relations.
880 * Call this method AFTER adding all records including relations.
881 *
882 * @return void
883 * @see export_addDBRelations()
884 */
885 public function export_addFilesFromRelations() {
886 // Traverse all "rels" registered for "records"
887 if (!is_array($this->dat['records'])) {
888 $this->error('There were no records available.');
889 return;
890 }
891 foreach ($this->dat['records'] as $k => $value) {
892 if (!isset($this->dat['records'][$k]['rels']) || !is_array($this->dat['records'][$k]['rels'])) {
893 continue;
894 }
895 foreach ($this->dat['records'][$k]['rels'] as $fieldname => $vR) {
896 // For all file type relations:
897 if ($vR['type'] == 'file') {
898 foreach ($vR['newValueFiles'] as $key => $fI) {
899 $this->export_addFile($fI, $k, $fieldname);
900 // Remove the absolute reference to the file so it doesn't expose absolute paths from source server:
901 unset($this->dat['records'][$k]['rels'][$fieldname]['newValueFiles'][$key]['ID_absFile']);
902 }
903 }
904 // For all flex type relations:
905 if ($vR['type'] == 'flex') {
906 if (is_array($vR['flexFormRels']['file'])) {
907 foreach ($vR['flexFormRels']['file'] as $key => $subList) {
908 foreach ($subList as $subKey => $fI) {
909 $this->export_addFile($fI, $k, $fieldname);
910 // Remove the absolute reference to the file so it doesn't expose absolute paths from source server:
911 unset($this->dat['records'][$k]['rels'][$fieldname]['flexFormRels']['file'][$key][$subKey]['ID_absFile']);
912 }
913 }
914 }
915 // DB oriented soft references in flex form fields:
916 if (is_array($vR['flexFormRels']['softrefs'])) {
917 foreach ($vR['flexFormRels']['softrefs'] as $key => $subList) {
918 foreach ($subList['keys'] as $spKey => $elements) {
919 foreach ($elements as $subKey => $el) {
920 if ($el['subst']['type'] === 'file' && $this->includeSoftref($el['subst']['tokenID'])) {
921 // Create abs path and ID for file:
922 $ID_absFile = GeneralUtility::getFileAbsFileName(PATH_site . $el['subst']['relFileName']);
923 $ID = md5($el['subst']['relFileName']);
924 if ($ID_absFile) {
925 if (!$this->dat['files'][$ID]) {
926 $fI = array(
927 'filename' => PathUtility::basename($ID_absFile),
928 'ID_absFile' => $ID_absFile,
929 'ID' => $ID,
930 'relFileName' => $el['subst']['relFileName']
931 );
932 $this->export_addFile($fI, '_SOFTREF_');
933 }
934 $this->dat['records'][$k]['rels'][$fieldname]['flexFormRels']['softrefs'][$key]['keys'][$spKey][$subKey]['file_ID'] = $ID;
935 }
936 }
937 }
938 }
939 }
940 }
941 }
942 // In any case, if there are soft refs:
943 if (is_array($vR['softrefs']['keys'])) {
944 foreach ($vR['softrefs']['keys'] as $spKey => $elements) {
945 foreach ($elements as $subKey => $el) {
946 if ($el['subst']['type'] === 'file' && $this->includeSoftref($el['subst']['tokenID'])) {
947 // Create abs path and ID for file:
948 $ID_absFile = GeneralUtility::getFileAbsFileName(PATH_site . $el['subst']['relFileName']);
949 $ID = md5($el['subst']['relFileName']);
950 if ($ID_absFile) {
951 if (!$this->dat['files'][$ID]) {
952 $fI = array(
953 'filename' => PathUtility::basename($ID_absFile),
954 'ID_absFile' => $ID_absFile,
955 'ID' => $ID,
956 'relFileName' => $el['subst']['relFileName']
957 );
958 $this->export_addFile($fI, '_SOFTREF_');
959 }
960 $this->dat['records'][$k]['rels'][$fieldname]['softrefs']['keys'][$spKey][$subKey]['file_ID'] = $ID;
961 }
962 }
963 }
964 }
965 }
966 }
967 }
968 }
969
970 /**
971 * This adds all files from sys_file records
972 *
973 * @return void
974 */
975 public function export_addFilesFromSysFilesRecords() {
976 if (!isset($this->dat['header']['records']['sys_file']) || !is_array($this->dat['header']['records']['sys_file'])) {
977 return;
978 }
979 foreach (array_keys($this->dat['header']['records']['sys_file']) as $sysFileUid) {
980 $recordData = $this->dat['records']['sys_file:' . $sysFileUid]['data'];
981 $file = ResourceFactory::getInstance()->createFileObject($recordData);
982 $this->export_addSysFile($file);
983 }
984 }
985
986 /**
987 * Adds a files content from a sys file record to the export memory
988 *
989 * @param \TYPO3\CMS\Core\Resource\File $file
990 * @return void
991 */
992 public function export_addSysFile(\TYPO3\CMS\Core\Resource\File $file) {
993 if ($file->getProperty('size') >= $this->maxFileSize) {
994 $this->error('File ' . $file->getPublicUrl() . ' was larger (' . GeneralUtility::formatSize($file->getProperty('size')) . ') than the maxFileSize (' . GeneralUtility::formatSize($this->maxFileSize) . ')! Skipping.');
995 return;
996 }
997 $fileContent = '';
998 try {
999 if (!$this->saveFilesOutsideExportFile) {
1000 $fileContent = $file->getContents();
1001 } else {
1002 $file->checkActionPermission('read');
1003 }
1004
1005 } catch (\TYPO3\CMS\Core\Resource\Exception\InsufficientFileAccessPermissionsException $e) {
1006 $this->error('File ' . $file->getPublicUrl() . ': ' . $e->getMessage());
1007 return;
1008 } catch (\TYPO3\CMS\Core\Resource\Exception\IllegalFileExtensionException $e) {
1009 $this->error('File ' . $file->getPublicUrl() . ': ' . $e->getMessage());
1010 return;
1011 }
1012 $fileUid = $file->getUid();
1013 $fileInfo = $file->getStorage()->getFileInfo($file);
1014 // we sadly have to cast it to string here, because the size property is also returning a string
1015 $fileSize = (string)$fileInfo['size'];
1016 if ($fileSize !== $file->getProperty('size')) {
1017 $this->error('File size of ' . $file->getCombinedIdentifier() . ' is not up-to-date in index! File added with current size.');
1018 $this->dat['records']['sys_file:' . $fileUid]['data']['size'] = $fileSize;
1019 }
1020 $fileSha1 = $file->getStorage()->hashFile($file, 'sha1');
1021 if ($fileSha1 !== $file->getProperty('sha1')) {
1022 $this->error('File sha1 hash of ' . $file->getCombinedIdentifier() . ' is not up-to-date in index! File added on current sha1.');
1023 $this->dat['records']['sys_file:' . $fileUid]['data']['sha1'] = $fileSha1;
1024 }
1025
1026 $fileRec = array();
1027 $fileRec['filesize'] = $fileSize;
1028 $fileRec['filename'] = $file->getProperty('name');
1029 $fileRec['filemtime'] = $file->getProperty('modification_date');
1030
1031 // build unique id based on the storage and the file identifier
1032 $fileId = md5($file->getStorage()->getUid() . ':' . $file->getProperty('identifier_hash'));
1033
1034 // Setting this data in the header
1035 $this->dat['header']['files_fal'][$fileId] = $fileRec;
1036
1037 if (!$this->saveFilesOutsideExportFile) {
1038 // ... and finally add the heavy stuff:
1039 $fileRec['content'] = $fileContent;
1040 } else {
1041 GeneralUtility::upload_copy_move($file->getForLocalProcessing(FALSE), $this->getTemporaryFilesPathForExport() . $file->getProperty('sha1'));
1042 }
1043 $fileRec['content_sha1'] = $fileSha1;
1044
1045 $this->dat['files_fal'][$fileId] = $fileRec;
1046 }
1047
1048
1049 /**
1050 * Adds a files content to the export memory
1051 *
1052 * @param array $fI File information with three keys: "filename" = filename without path, "ID_absFile" = absolute filepath to the file (including the filename), "ID" = md5 hash of "ID_absFile". "relFileName" is optional for files attached to records, but mandatory for soft referenced files (since the relFileName determines where such a file should be stored!)
1053 * @param string $recordRef If the file is related to a record, this is the id on the form [table]:[id]. Information purposes only.
1054 * @param string $fieldname If the file is related to a record, this is the field name it was related to. Information purposes only.
1055 * @return void
1056 */
1057 public function export_addFile($fI, $recordRef = '', $fieldname = '') {
1058 if (!@is_file($fI['ID_absFile'])) {
1059 $this->error($fI['ID_absFile'] . ' was not a file! Skipping.');
1060 return;
1061 }
1062 if (filesize($fI['ID_absFile']) >= $this->maxFileSize) {
1063 $this->error($fI['ID_absFile'] . ' was larger (' . GeneralUtility::formatSize(filesize($fI['ID_absFile'])) . ') than the maxFileSize (' . GeneralUtility::formatSize($this->maxFileSize) . ')! Skipping.');
1064 return;
1065 }
1066 $fileInfo = stat($fI['ID_absFile']);
1067 $fileRec = array();
1068 $fileRec['filesize'] = $fileInfo['size'];
1069 $fileRec['filename'] = PathUtility::basename($fI['ID_absFile']);
1070 $fileRec['filemtime'] = $fileInfo['mtime'];
1071 //for internal type file_reference
1072 $fileRec['relFileRef'] = PathUtility::stripPathSitePrefix($fI['ID_absFile']);
1073 if ($recordRef) {
1074 $fileRec['record_ref'] = $recordRef . '/' . $fieldname;
1075 }
1076 if ($fI['relFileName']) {
1077 $fileRec['relFileName'] = $fI['relFileName'];
1078 }
1079 // Setting this data in the header
1080 $this->dat['header']['files'][$fI['ID']] = $fileRec;
1081 // ... and for the recordlisting, why not let us know WHICH relations there was...
1082 if ($recordRef && $recordRef !== '_SOFTREF_') {
1083 $refParts = explode(':', $recordRef, 2);
1084 if (!is_array($this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'])) {
1085 $this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'] = array();
1086 }
1087 $this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'][] = $fI['ID'];
1088 }
1089 $fileMd5 = md5_file($fI['ID_absFile']);
1090 if (!$this->saveFilesOutsideExportFile) {
1091 // ... and finally add the heavy stuff:
1092 $fileRec['content'] = GeneralUtility::getUrl($fI['ID_absFile']);
1093 } else {
1094 GeneralUtility::upload_copy_move($fI['ID_absFile'], $this->getTemporaryFilesPathForExport() . $fileMd5);
1095 }
1096 $fileRec['content_md5'] = $fileMd5;
1097 $this->dat['files'][$fI['ID']] = $fileRec;
1098 // For soft references, do further processing:
1099 if ($recordRef === '_SOFTREF_') {
1100 // RTE files?
1101 if ($RTEoriginal = $this->getRTEoriginalFilename(PathUtility::basename($fI['ID_absFile']))) {
1102 $RTEoriginal_absPath = PathUtility::dirname($fI['ID_absFile']) . '/' . $RTEoriginal;
1103 if (@is_file($RTEoriginal_absPath)) {
1104 $RTEoriginal_ID = md5($RTEoriginal_absPath);
1105 $fileInfo = stat($RTEoriginal_absPath);
1106 $fileRec = array();
1107 $fileRec['filesize'] = $fileInfo['size'];
1108 $fileRec['filename'] = PathUtility::basename($RTEoriginal_absPath);
1109 $fileRec['filemtime'] = $fileInfo['mtime'];
1110 $fileRec['record_ref'] = '_RTE_COPY_ID:' . $fI['ID'];
1111 $this->dat['header']['files'][$fI['ID']]['RTE_ORIG_ID'] = $RTEoriginal_ID;
1112 // Setting this data in the header
1113 $this->dat['header']['files'][$RTEoriginal_ID] = $fileRec;
1114 $fileMd5 = md5_file($RTEoriginal_absPath);
1115 if (!$this->saveFilesOutsideExportFile) {
1116 // ... and finally add the heavy stuff:
1117 $fileRec['content'] = GeneralUtility::getUrl($RTEoriginal_absPath);
1118 } else {
1119 GeneralUtility::upload_copy_move($RTEoriginal_absPath, $this->getTemporaryFilesPathForExport() . $fileMd5);
1120 }
1121 $fileRec['content_md5'] = $fileMd5;
1122 $this->dat['files'][$RTEoriginal_ID] = $fileRec;
1123 } else {
1124 $this->error('RTE original file "' . PathUtility::stripPathSitePrefix($RTEoriginal_absPath) . '" was not found!');
1125 }
1126 }
1127 // Files with external media?
1128 // This is only done with files grabbed by a softreference parser since it is deemed improbable that hard-referenced files should undergo this treatment.
1129 $html_fI = pathinfo(PathUtility::basename($fI['ID_absFile']));
1130 if ($this->includeExtFileResources && GeneralUtility::inList($this->extFileResourceExtensions, strtolower($html_fI['extension']))) {
1131 $uniquePrefix = '###' . md5($GLOBALS['EXEC_TIME']) . '###';
1132 if (strtolower($html_fI['extension']) === 'css') {
1133 $prefixedMedias = explode($uniquePrefix, preg_replace('/(url[[:space:]]*\\([[:space:]]*["\']?)([^"\')]*)(["\']?[[:space:]]*\\))/i', '\\1' . $uniquePrefix . '\\2' . $uniquePrefix . '\\3', $fileRec['content']));
1134 } else {
1135 // html, htm:
1136 $htmlParser = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\Html\\HtmlParser');
1137 $prefixedMedias = explode($uniquePrefix, $htmlParser->prefixResourcePath($uniquePrefix, $fileRec['content'], array(), $uniquePrefix));
1138 }
1139 $htmlResourceCaptured = FALSE;
1140 foreach ($prefixedMedias as $k => $v) {
1141 if ($k % 2) {
1142 $EXTres_absPath = GeneralUtility::resolveBackPath(PathUtility::dirname($fI['ID_absFile']) . '/' . $v);
1143 $EXTres_absPath = GeneralUtility::getFileAbsFileName($EXTres_absPath);
1144 if ($EXTres_absPath && GeneralUtility::isFirstPartOfStr($EXTres_absPath, PATH_site . $this->fileadminFolderName . '/') && @is_file($EXTres_absPath)) {
1145 $htmlResourceCaptured = TRUE;
1146 $EXTres_ID = md5($EXTres_absPath);
1147 $this->dat['header']['files'][$fI['ID']]['EXT_RES_ID'][] = $EXTres_ID;
1148 $prefixedMedias[$k] = '{EXT_RES_ID:' . $EXTres_ID . '}';
1149 // Add file to memory if it is not set already:
1150 if (!isset($this->dat['header']['files'][$EXTres_ID])) {
1151 $fileInfo = stat($EXTres_absPath);
1152 $fileRec = array();
1153 $fileRec['filesize'] = $fileInfo['size'];
1154 $fileRec['filename'] = PathUtility::basename($EXTres_absPath);
1155 $fileRec['filemtime'] = $fileInfo['mtime'];
1156 $fileRec['record_ref'] = '_EXT_PARENT_:' . $fI['ID'];
1157 // Media relative to the HTML file.
1158 $fileRec['parentRelFileName'] = $v;
1159 // Setting this data in the header
1160 $this->dat['header']['files'][$EXTres_ID] = $fileRec;
1161 // ... and finally add the heavy stuff:
1162 $fileRec['content'] = GeneralUtility::getUrl($EXTres_absPath);
1163 $fileRec['content_md5'] = md5($fileRec['content']);
1164 $this->dat['files'][$EXTres_ID] = $fileRec;
1165 }
1166 }
1167 }
1168 }
1169 if ($htmlResourceCaptured) {
1170 $this->dat['files'][$fI['ID']]['tokenizedContent'] = implode('', $prefixedMedias);
1171 }
1172 }
1173 }
1174 }
1175
1176 /**
1177 * If saveFilesOutsideExportFile is enabled, this function returns the path
1178 * where the files referenced in the export are copied to.
1179 *
1180 * @return string
1181 * @throws \RuntimeException
1182 * @see setSaveFilesOutsideExportFile()
1183 */
1184 public function getTemporaryFilesPathForExport() {
1185 if (!$this->saveFilesOutsideExportFile) {
1186 throw new \RuntimeException('You need to set saveFilesOutsideExportFile to TRUE before you want to get the temporary files path for export.', 1401205213);
1187 }
1188 if ($this->temporaryFilesPathForExport === NULL) {
1189 $temporaryFolderName = $this->getTemporaryFolderName();
1190 $this->temporaryFilesPathForExport = $temporaryFolderName . '/';
1191 }
1192 return $this->temporaryFilesPathForExport;
1193 }
1194
1195 /**
1196 *
1197 * @return string
1198 */
1199 protected function getTemporaryFolderName() {
1200 $temporaryPath = PATH_site . 'typo3temp/';
1201 do {
1202 $temporaryFolderName = $temporaryPath . 'export_temp_files_' . mt_rand(1, PHP_INT_MAX);
1203 } while (is_dir($temporaryFolderName));
1204 GeneralUtility::mkdir($temporaryFolderName);
1205 return $temporaryFolderName;
1206 }
1207
1208 /**
1209 * DB relations flattend to 1-dim array.
1210 * The list will be unique, no table/uid combination will appear twice.
1211 *
1212 * @param array $dbrels 2-dim Array of database relations organized by table key
1213 * @return array 1-dim array where entries are table:uid and keys are array with table/id
1214 */
1215 public function flatDBrels($dbrels) {
1216 $list = array();
1217 foreach ($dbrels as $dat) {
1218 if ($dat['type'] == 'db') {
1219 foreach ($dat['itemArray'] as $i) {
1220 $list[$i['table'] . ':' . $i['id']] = $i;
1221 }
1222 }
1223 if ($dat['type'] == 'flex' && is_array($dat['flexFormRels']['db'])) {
1224 foreach ($dat['flexFormRels']['db'] as $subList) {
1225 foreach ($subList as $i) {
1226 $list[$i['table'] . ':' . $i['id']] = $i;
1227 }
1228 }
1229 }
1230 }
1231 return $list;
1232 }
1233
1234 /**
1235 * Soft References flattend to 1-dim array.
1236 *
1237 * @param array $dbrels 2-dim Array of database relations organized by table key
1238 * @return array 1-dim array where entries are arrays with properties of the soft link found and keys are a unique combination of field, spKey, structure path if applicable and token ID
1239 */
1240 public function flatSoftRefs($dbrels) {
1241 $list = array();
1242 foreach ($dbrels as $field => $dat) {
1243 if (is_array($dat['softrefs']['keys'])) {
1244 foreach ($dat['softrefs']['keys'] as $spKey => $elements) {
1245 if (is_array($elements)) {
1246 foreach ($elements as $subKey => $el) {
1247 $lKey = $field . ':' . $spKey . ':' . $subKey;
1248 $list[$lKey] = array_merge(array('field' => $field, 'spKey' => $spKey), $el);
1249 // Add file_ID key to header - slightly "risky" way of doing this because if the calculation
1250 // changes for the same value in $this->records[...] this will not work anymore!
1251 if ($el['subst'] && $el['subst']['relFileName']) {
1252 $list[$lKey]['file_ID'] = md5(PATH_site . $el['subst']['relFileName']);
1253 }
1254 }
1255 }
1256 }
1257 }
1258 if ($dat['type'] == 'flex' && is_array($dat['flexFormRels']['softrefs'])) {
1259 foreach ($dat['flexFormRels']['softrefs'] as $structurePath => $subSoftrefs) {
1260 if (is_array($subSoftrefs['keys'])) {
1261 foreach ($subSoftrefs['keys'] as $spKey => $elements) {
1262 foreach ($elements as $subKey => $el) {
1263 $lKey = $field . ':' . $structurePath . ':' . $spKey . ':' . $subKey;
1264 $list[$lKey] = array_merge(array('field' => $field, 'spKey' => $spKey, 'structurePath' => $structurePath), $el);
1265 // Add file_ID key to header - slightly "risky" way of doing this because if the calculation
1266 // changes for the same value in $this->records[...] this will not work anymore!
1267 if ($el['subst'] && $el['subst']['relFileName']) {
1268 $list[$lKey]['file_ID'] = md5(PATH_site . $el['subst']['relFileName']);
1269 }
1270 }
1271 }
1272 }
1273 }
1274 }
1275 }
1276 return $list;
1277 }
1278
1279 /**
1280 * If include fields for a specific record type are set, the data
1281 * are filtered out with fields are not included in the fields.
1282 *
1283 * @param string $table The record type to be filtered
1284 * @param array $row The data to be filtered
1285 * @return array The filtered record row
1286 */
1287 protected function filterRecordFields($table, array $row) {
1288 if (isset($this->recordTypesIncludeFields[$table])) {
1289 $includeFields = array_unique(array_merge(
1290 $this->recordTypesIncludeFields[$table],
1291 $this->defaultRecordIncludeFields
1292 ));
1293 $newRow = array();
1294 foreach ($row as $key => $value) {
1295 if (in_array($key, $includeFields)) {
1296 $newRow[$key] = $value;
1297 }
1298 }
1299 } else {
1300 $newRow = $row;
1301 }
1302 return $newRow;
1303 }
1304
1305
1306 /**************************
1307 * File Output
1308 *************************/
1309
1310 /**
1311 * This compiles and returns the data content for an exported file
1312 *
1313 * @param string $type Type of output; "xml" gives xml, otherwise serialized array, possibly compressed.
1314 * @return string The output file stream
1315 */
1316 public function compileMemoryToFileContent($type = '') {
1317 if ($type == 'xml') {
1318 $out = $this->createXML();
1319 } else {
1320 $compress = $this->doOutputCompress();
1321 $out = '';
1322 // adding header:
1323 $out .= $this->addFilePart(serialize($this->dat['header']), $compress);
1324 // adding records:
1325 $out .= $this->addFilePart(serialize($this->dat['records']), $compress);
1326 // adding files:
1327 $out .= $this->addFilePart(serialize($this->dat['files']), $compress);
1328 // adding files_fal:
1329 $out .= $this->addFilePart(serialize($this->dat['files_fal']), $compress);
1330 }
1331 return $out;
1332 }
1333
1334 /**
1335 * Creates XML string from input array
1336 *
1337 * @return string XML content
1338 */
1339 public function createXML() {
1340 // Options:
1341 $options = array(
1342 'alt_options' => array(
1343 '/header' => array(
1344 'disableTypeAttrib' => TRUE,
1345 'clearStackPath' => TRUE,
1346 'parentTagMap' => array(
1347 'files' => 'file',
1348 'files_fal' => 'file',
1349 'records' => 'table',
1350 'table' => 'rec',
1351 'rec:rels' => 'relations',
1352 'relations' => 'element',
1353 'filerefs' => 'file',
1354 'pid_lookup' => 'page_contents',
1355 'header:relStaticTables' => 'static_tables',
1356 'static_tables' => 'tablename',
1357 'excludeMap' => 'item',
1358 'softrefCfg' => 'softrefExportMode',
1359 'extensionDependencies' => 'extkey',
1360 'softrefs' => 'softref_element'
1361 ),
1362 'alt_options' => array(
1363 '/pagetree' => array(
1364 'disableTypeAttrib' => TRUE,
1365 'useIndexTagForNum' => 'node',
1366 'parentTagMap' => array(
1367 'node:subrow' => 'node'
1368 )
1369 ),
1370 '/pid_lookup/page_contents' => array(
1371 'disableTypeAttrib' => TRUE,
1372 'parentTagMap' => array(
1373 'page_contents' => 'table'
1374 ),
1375 'grandParentTagMap' => array(
1376 'page_contents/table' => 'item'
1377 )
1378 )
1379 )
1380 ),
1381 '/records' => array(
1382 'disableTypeAttrib' => TRUE,
1383 'parentTagMap' => array(
1384 'records' => 'tablerow',
1385 'tablerow:data' => 'fieldlist',
1386 'tablerow:rels' => 'related',
1387 'related' => 'field',
1388 'field:itemArray' => 'relations',
1389 'field:newValueFiles' => 'filerefs',
1390 'field:flexFormRels' => 'flexform',
1391 'relations' => 'element',
1392 'filerefs' => 'file',
1393 'flexform:db' => 'db_relations',
1394 'flexform:file' => 'file_relations',
1395 'flexform:softrefs' => 'softref_relations',
1396 'softref_relations' => 'structurePath',
1397 'db_relations' => 'path',
1398 'file_relations' => 'path',
1399 'path' => 'element',
1400 'keys' => 'softref_key',
1401 'softref_key' => 'softref_element'
1402 ),
1403 'alt_options' => array(
1404 '/records/tablerow/fieldlist' => array(
1405 'useIndexTagForAssoc' => 'field'
1406 )
1407 )
1408 ),
1409 '/files' => array(
1410 'disableTypeAttrib' => TRUE,
1411 'parentTagMap' => array(
1412 'files' => 'file'
1413 )
1414 ),
1415 '/files_fal' => array(
1416 'disableTypeAttrib' => TRUE,
1417 'parentTagMap' => array(
1418 'files_fal' => 'file'
1419 )
1420 )
1421 )
1422 );
1423 // Creating XML file from $outputArray:
1424 $charset = $this->dat['header']['charset'] ?: 'utf-8';
1425 $XML = '<?xml version="1.0" encoding="' . $charset . '" standalone="yes" ?>' . LF;
1426 $XML .= GeneralUtility::array2xml($this->dat, '', 0, 'T3RecordDocument', 0, $options);
1427 return $XML;
1428 }
1429
1430 /**
1431 * Returns TRUE if the output should be compressed.
1432 *
1433 * @return bool TRUE if compression is possible AND requested.
1434 */
1435 public function doOutputCompress() {
1436 return $this->compress && !$this->dontCompress;
1437 }
1438
1439 /**
1440 * Returns a content part for a filename being build.
1441 *
1442 * @param array $data Data to store in part
1443 * @param bool $compress Compress file?
1444 * @return string Content stream.
1445 */
1446 public function addFilePart($data, $compress = FALSE) {
1447 if ($compress) {
1448 $data = gzcompress($data);
1449 }
1450 return md5($data) . ':' . ($compress ? '1' : '0') . ':' . str_pad(strlen($data), 10, '0', STR_PAD_LEFT) . ':' . $data . ':';
1451 }
1452
1453 /***********************
1454 * Import
1455 ***********************/
1456
1457 /**
1458 * Initialize all settings for the import
1459 *
1460 * @return void
1461 */
1462 protected function initializeImport() {
1463 // Set this flag to indicate that an import is being/has been done.
1464 $this->doesImport = 1;
1465 // Initialize:
1466 // These vars MUST last for the whole section not being cleared. They are used by the method setRelations() which are called at the end of the import session.
1467 $this->import_mapId = array();
1468 $this->import_newId = array();
1469 $this->import_newId_pids = array();
1470 // Temporary files stack initialized:
1471 $this->unlinkFiles = array();
1472 $this->alternativeFileName = array();
1473 $this->alternativeFilePath = array();
1474
1475 $this->initializeStorageObjects();
1476 }
1477
1478 /**
1479 * Initialize the all present storage objects
1480 *
1481 * @return void
1482 */
1483 protected function initializeStorageObjects() {
1484 /** @var $storageRepository \TYPO3\CMS\Core\Resource\StorageRepository */
1485 $storageRepository = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\Resource\\StorageRepository');
1486 $this->storageObjects = $storageRepository->findAll();
1487 }
1488
1489 /**
1490 * Imports the internal data array to $pid.
1491 *
1492 * @param int $pid Page ID in which to import the content
1493 * @return void
1494 */
1495 public function importData($pid) {
1496
1497 $this->initializeImport();
1498
1499 // Write sys_file_storages first
1500 $this->writeSysFileStorageRecords();
1501 // Write sys_file records and write the binary file data
1502 $this->writeSysFileRecords();
1503 // Write records, first pages, then the rest
1504 // Fields with "hard" relations to database, files and flexform fields are kept empty during this run
1505 $this->writeRecords_pages($pid);
1506 $this->writeRecords_records($pid);
1507 // Finally all the file and DB record references must be fixed. This is done after all records have supposedly been written to database:
1508 // $this->import_mapId will indicate two things: 1) that a record WAS written to db and 2) that it has got a new id-number.
1509 $this->setRelations();
1510 // And when all DB relations are in place, we can fix file and DB relations in flexform fields (since data structures often depends on relations to a DS record):
1511 $this->setFlexFormRelations();
1512 // Unlink temporary files:
1513 $this->unlinkTempFiles();
1514 // Finally, traverse all records and process softreferences with substitution attributes.
1515 $this->processSoftReferences();
1516 // After all migrate records using sys_file_reference now
1517 if ($this->legacyImport) {
1518 $this->migrateLegacyImportRecords();
1519 }
1520 }
1521
1522 /**
1523 * Imports the sys_file_storage records from internal data array.
1524 *
1525 * @return void
1526 */
1527 protected function writeSysFileStorageRecords() {
1528 if (!isset($this->dat['header']['records']['sys_file_storage'])) {
1529 return;
1530 }
1531 $sysFileStorageUidsToBeResetToDefaultStorage = array();
1532 foreach (array_keys($this->dat['header']['records']['sys_file_storage']) as $sysFileStorageUid) {
1533 $storageRecord = $this->dat['records']['sys_file_storage:' . $sysFileStorageUid]['data'];
1534 // continue with Local, writable and online storage only
1535 if ($storageRecord['driver'] === 'Local' && $storageRecord['is_writable'] && $storageRecord['is_online']) {
1536 $useThisStorageUidInsteadOfTheOneInImport = 0;
1537 /** @var $localStorage \TYPO3\CMS\Core\Resource\ResourceStorage */
1538 foreach ($this->storageObjects as $localStorage) {
1539 // check the available storage for Local, writable and online ones
1540 if ($localStorage->getDriverType() === 'Local' && $localStorage->isWritable() && $localStorage->isOnline()) {
1541 // check if there is already a identical storage present (same pathType and basePath)
1542 $storageRecordConfiguration = ResourceFactory::getInstance()->convertFlexFormDataToConfigurationArray($storageRecord['configuration']);
1543 $localStorageRecordConfiguration = $localStorage->getConfiguration();
1544 if (
1545 $storageRecordConfiguration['pathType'] === $localStorageRecordConfiguration['pathType']
1546 && $storageRecordConfiguration['basePath'] === $localStorageRecordConfiguration['basePath']
1547 ) {
1548 // same storage is already present
1549 $useThisStorageUidInsteadOfTheOneInImport = $localStorage->getUid();
1550 break;
1551 }
1552 }
1553 }
1554 if ($useThisStorageUidInsteadOfTheOneInImport > 0) {
1555 // same storage is already present; map the to be imported one to the present one
1556 $this->import_mapId['sys_file_storage'][$sysFileStorageUid] = $useThisStorageUidInsteadOfTheOneInImport;
1557 } else {
1558 // Local, writable and online storage. Is allowed to be used to later write files in.
1559 $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);
1560 }
1561 } else {
1562 // Storage with non Local drivers could be imported but must not be used to saves files in, because you
1563 // could not be sure, that this is supported. The default storage will be used in this case.
1564 // It could happen that non writable and non online storage will be created as dupes because you could not
1565 // check the detailed configuration options at this point
1566 $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);
1567 $sysFileStorageUidsToBeResetToDefaultStorage[] = $sysFileStorageUid;
1568 }
1569
1570 }
1571
1572 // Importing the added ones
1573 $tce = $this->getNewTCE();
1574 // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.
1575 $tce->reverseOrder = 1;
1576 $tce->isImporting = TRUE;
1577 $tce->start($this->import_data, array());
1578 $tce->process_datamap();
1579 $this->addToMapId($tce->substNEWwithIDs);
1580
1581 $defaultStorageUid = NULL;
1582 // get default storage
1583 $defaultStorage = ResourceFactory::getInstance()->getDefaultStorage();
1584 if ($defaultStorage !== NULL) {
1585 $defaultStorageUid = $defaultStorage->getUid();
1586 }
1587 foreach ($sysFileStorageUidsToBeResetToDefaultStorage as $sysFileStorageUidToBeResetToDefaultStorage) {
1588 $this->import_mapId['sys_file_storage'][$sysFileStorageUidToBeResetToDefaultStorage] = $defaultStorageUid;
1589 }
1590
1591 // unset the sys_file_storage records to prevent a import in writeRecords_records
1592 unset($this->dat['header']['records']['sys_file_storage']);
1593 }
1594
1595 /**
1596 * Imports the sys_file records and the binary files data from internal data array.
1597 *
1598 * @return void
1599 */
1600 protected function writeSysFileRecords() {
1601 if (!isset($this->dat['header']['records']['sys_file'])) {
1602 return;
1603 }
1604
1605 // fetch fresh storage records from database
1606 $storageRecords = $this->fetchStorageRecords();
1607
1608 $defaultStorage = ResourceFactory::getInstance()->getDefaultStorage();
1609
1610 $sanitizedFolderMappings = array();
1611
1612 foreach (array_keys($this->dat['header']['records']['sys_file']) as $sysFileUid) {
1613 $fileRecord = $this->dat['records']['sys_file:' . $sysFileUid]['data'];
1614
1615 $temporaryFile = NULL;
1616 // check if there is the right file already in the local folder
1617 if ($this->filesPathForImport !== NULL) {
1618 if (is_file($this->filesPathForImport . '/' . $fileRecord['sha1']) && sha1_file($this->filesPathForImport . '/' . $fileRecord['sha1']) === $fileRecord['sha1']) {
1619 $temporaryFile = $this->filesPathForImport . '/' . $fileRecord['sha1'];
1620 }
1621 }
1622
1623 // save file to disk
1624 if ($temporaryFile === NULL) {
1625 $fileId = md5($fileRecord['storage'] . ':' . $fileRecord['identifier_hash']);
1626 $temporaryFile = $this->writeTemporaryFileFromData($fileId);
1627 if ($temporaryFile === NULL) {
1628 // error on writing the file. Error message was already added
1629 continue;
1630 }
1631 }
1632
1633 $originalStorageUid = $fileRecord['storage'];
1634 $useStorageFromStorageRecords = FALSE;
1635
1636 // replace storage id, if a alternative one was registered
1637 if (isset($this->import_mapId['sys_file_storage'][$fileRecord['storage']])) {
1638 $fileRecord['storage'] = $this->import_mapId['sys_file_storage'][$fileRecord['storage']];
1639 $useStorageFromStorageRecords = TRUE;
1640 }
1641
1642 if (empty($fileRecord['storage']) && !$this->isFallbackStorage($fileRecord['storage'])) {
1643 // no storage for the file is defined, mostly because of a missing default storage.
1644 $this->error('Error: No storage for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $originalStorageUid . '"');
1645 continue;
1646 }
1647
1648 // using a storage from the local storage is only allowed, if the uid is present in the
1649 // mapping. Only in this case we could be sure, that it's a local, online and writable storage.
1650 if ($useStorageFromStorageRecords && isset($storageRecords[$fileRecord['storage']])) {
1651 /** @var $storage \TYPO3\CMS\Core\Resource\ResourceStorage */
1652 $storage = ResourceFactory::getInstance()->getStorageObject($fileRecord['storage'], $storageRecords[$fileRecord['storage']]);
1653 } elseif ($this->isFallbackStorage($fileRecord['storage'])) {
1654 $storage = ResourceFactory::getInstance()->getStorageObject(0);
1655 } elseif ($defaultStorage !== NULL) {
1656 $storage = $defaultStorage;
1657 } else {
1658 $this->error('Error: No storage available for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1659 continue;
1660 }
1661
1662 $newFile = NULL;
1663
1664 // check, if there is a identical file
1665 try {
1666 if ($storage->hasFile($fileRecord['identifier'])) {
1667 $file = $storage->getFile($fileRecord['identifier']);
1668 if ($file->getSha1() === $fileRecord['sha1']) {
1669 $newFile = $file;
1670 }
1671 }
1672 } catch (Exception $e) {}
1673
1674 if ($newFile === NULL) {
1675
1676 $folderName = PathUtility::dirname(ltrim($fileRecord['identifier'], '/'));
1677 if (in_array($folderName, $sanitizedFolderMappings)) {
1678 $folderName = $sanitizedFolderMappings[$folderName];
1679 }
1680 if (!$storage->hasFolder($folderName)) {
1681 try {
1682 $importFolder = $storage->createFolder($folderName);
1683 if ($importFolder->getIdentifier() !== $folderName && !in_array($folderName, $sanitizedFolderMappings)) {
1684 $sanitizedFolderMappings[$folderName] = $importFolder->getIdentifier();
1685 }
1686 } catch (Exception $e) {
1687 $this->error('Error: Folder could not be created for file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1688 continue;
1689 }
1690 } else {
1691 $importFolder = $storage->getFolder($folderName);
1692 }
1693
1694 try {
1695 /** @var $newFile \TYPO3\CMS\Core\Resource\File */
1696 $newFile = $storage->addFile($temporaryFile, $importFolder, $fileRecord['name']);
1697 } catch (Exception $e) {
1698 $this->error('Error: File could not be added to the storage: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1699 continue;
1700 }
1701
1702 if ($newFile->getSha1() !== $fileRecord['sha1']) {
1703 $this->error('Error: The hash of the written file is not identical to the import data! File could be corrupted! File: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1704 }
1705 }
1706
1707 // save the new uid in the import id map
1708 $this->import_mapId['sys_file'][$fileRecord['uid']] = $newFile->getUid();
1709 $this->fixUidLocalInSysFileReferenceRecords($fileRecord['uid'], $newFile->getUid());
1710
1711 }
1712
1713 // unset the sys_file records to prevent a import in writeRecords_records
1714 unset($this->dat['header']['records']['sys_file']);
1715 }
1716
1717 /**
1718 * Checks if the $storageId is the id of the fallback storage
1719 *
1720 * @param int|string $storageId
1721 * @return bool
1722 */
1723 protected function isFallbackStorage($storageId) {
1724 return $storageId === 0 || $storageId === '0';
1725 }
1726
1727 /**
1728 * Normally the importer works like the following:
1729 * Step 1: import the records with cleared field values of relation fields (see addSingle())
1730 * Step 2: update the records with the right relation ids (see setRelations())
1731 *
1732 * In step 2 the saving fields of type "relation to sys_file_reference" checks the related sys_file_reference
1733 * record (created in step 1) with the FileExtensionFilter for matching file extensions of the related file.
1734 * To make this work correct, the uid_local of sys_file_reference records has to be not empty AND has to
1735 * relate to the correct (imported) sys_file record uid!!!
1736 *
1737 * This is fixed here.
1738 *
1739 * @param int $oldFileUid
1740 * @param int $newFileUid
1741 * @return void
1742 */
1743 protected function fixUidLocalInSysFileReferenceRecords($oldFileUid, $newFileUid) {
1744 if (!isset($this->dat['header']['records']['sys_file_reference'])) {
1745 return;
1746 }
1747
1748 foreach (array_keys($this->dat['header']['records']['sys_file_reference']) as $sysFileReferenceUid) {
1749 $fileReferenceRecord = $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'];
1750 if ($fileReferenceRecord['uid_local'] == $oldFileUid) {
1751 $fileReferenceRecord['uid_local'] = $newFileUid;
1752 $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'] = $fileReferenceRecord;
1753 }
1754 }
1755 }
1756
1757 /**
1758 * Initializes the folder for legacy imports as subfolder of backend users default upload folder
1759 *
1760 * @return void
1761 */
1762 protected function initializeLegacyImportFolder() {
1763 /** @var \TYPO3\CMS\Core\Resource\Folder $folder */
1764 $folder = $GLOBALS['BE_USER']->getDefaultUploadFolder();
1765 if ($folder === FALSE) {
1766 $this->error('Error: the backend users default upload folder is missing! No files will be imported!');
1767 }
1768 if (!$folder->hasFolder($this->legacyImportTargetPath)) {
1769 try {
1770 $this->legacyImportFolder = $folder->createFolder($this->legacyImportTargetPath);
1771 } catch (\TYPO3\CMS\Core\Exception $e) {
1772 $this->error('Error: the import folder in the default upload folder could not be created! No files will be imported!');
1773 }
1774 } else {
1775 $this->legacyImportFolder = $folder->getSubFolder($this->legacyImportTargetPath);
1776 }
1777
1778 }
1779
1780 /**
1781 * Fetched fresh storage records from database because the new imported
1782 * ones are not in cached data of the StorageRepository
1783 *
1784 * @return bool|array
1785 */
1786 protected function fetchStorageRecords() {
1787 $whereClause = BackendUtility::BEenableFields('sys_file_storage');
1788 $whereClause .= BackendUtility::deleteClause('sys_file_storage');
1789
1790 $rows = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows(
1791 '*',
1792 'sys_file_storage',
1793 '1=1' . $whereClause,
1794 '',
1795 '',
1796 '',
1797 'uid'
1798 );
1799
1800 return $rows;
1801 }
1802
1803 /**
1804 * Writes the file from import array to temp dir and returns the filename of it.
1805 *
1806 * @param string $fileId
1807 * @param string $dataKey
1808 * @return string Absolute filename of the temporary filename of the file
1809 */
1810 protected function writeTemporaryFileFromData($fileId, $dataKey = 'files_fal') {
1811 $temporaryFilePath = NULL;
1812 if (is_array($this->dat[$dataKey][$fileId])) {
1813 $temporaryFilePathInternal = GeneralUtility::tempnam('import_temp_');
1814 GeneralUtility::writeFile($temporaryFilePathInternal, $this->dat[$dataKey][$fileId]['content']);
1815 clearstatcache();
1816 if (@is_file($temporaryFilePathInternal)) {
1817 $this->unlinkFiles[] = $temporaryFilePathInternal;
1818 if (filesize($temporaryFilePathInternal) == $this->dat[$dataKey][$fileId]['filesize']) {
1819 $temporaryFilePath = $temporaryFilePathInternal;
1820 } else {
1821 $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' had a size (' . filesize($temporaryFilePathInternal) . ') different from the original (' . $this->dat[$dataKey][$fileId]['filesize'] . ')', 1);
1822 }
1823 } else {
1824 $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' was not written as it should have been!', 1);
1825 }
1826 } else {
1827 $this->error('Error: No file found for ID ' . $fileId, 1);
1828 }
1829 return $temporaryFilePath;
1830 }
1831
1832 /**
1833 * Writing pagetree/pages to database:
1834 *
1835 * @param int $pid PID in which to import. If the operation is an update operation, the root of the page tree inside will be moved to this PID unless it is the same as the root page from the import
1836 * @return void
1837 * @see writeRecords_records()
1838 */
1839 public function writeRecords_pages($pid) {
1840 // First, write page structure if any:
1841 if (is_array($this->dat['header']['records']['pages'])) {
1842 // $pageRecords is a copy of the pages array in the imported file. Records here are unset one by one when the addSingle function is called.
1843 $pageRecords = $this->dat['header']['records']['pages'];
1844 $this->import_data = array();
1845 // First add page tree if any
1846 if (is_array($this->dat['header']['pagetree'])) {
1847 $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);
1848 foreach ($pagesFromTree as $uid) {
1849 $thisRec = $this->dat['header']['records']['pages'][$uid];
1850 // PID: Set the main $pid, unless a NEW-id is found
1851 $setPid = isset($this->import_newId_pids[$thisRec['pid']]) ? $this->import_newId_pids[$thisRec['pid']] : $pid;
1852 $this->addSingle('pages', $uid, $setPid);
1853 unset($pageRecords[$uid]);
1854 }
1855 }
1856 // Then add all remaining pages not in tree on root level:
1857 if (count($pageRecords)) {
1858 $remainingPageUids = array_keys($pageRecords);
1859 foreach ($remainingPageUids as $pUid) {
1860 $this->addSingle('pages', $pUid, $pid);
1861 }
1862 }
1863 // Now write to database:
1864 $tce = $this->getNewTCE();
1865 $tce->isImporting = TRUE;
1866 $this->callHook('before_writeRecordsPages', array(
1867 'tce' => &$tce,
1868 'data' => &$this->import_data
1869 ));
1870 $tce->suggestedInsertUids = $this->suggestedInsertUids;
1871 $tce->start($this->import_data, array());
1872 $tce->process_datamap();
1873 $this->callHook('after_writeRecordsPages', array(
1874 'tce' => &$tce
1875 ));
1876 // post-processing: Registering new ids (end all tcemain sessions with this)
1877 $this->addToMapId($tce->substNEWwithIDs);
1878 // In case of an update, order pages from the page tree correctly:
1879 if ($this->update && is_array($this->dat['header']['pagetree'])) {
1880 $this->writeRecords_pages_order($pid);
1881 }
1882 }
1883 }
1884
1885 /**
1886 * Organize all updated pages in page tree so they are related like in the import file
1887 * Only used for updates and when $this->dat['header']['pagetree'] is an array.
1888 *
1889 * @param int $pid Page id in which to import
1890 * @return void
1891 * @access private
1892 * @see writeRecords_pages(), writeRecords_records_order()
1893 */
1894 public function writeRecords_pages_order($pid) {
1895 $cmd_data = array();
1896 // Get uid-pid relations and traverse them in order to map to possible new IDs
1897 $pidsFromTree = $this->flatInversePageTree_pid($this->dat['header']['pagetree']);
1898 foreach ($pidsFromTree as $origPid => $newPid) {
1899 if ($newPid >= 0 && $this->dontIgnorePid('pages', $origPid)) {
1900 // If the page had a new id (because it was created) use that instead!
1901 if (substr($this->import_newId_pids[$origPid], 0, 3) === 'NEW') {
1902 if ($this->import_mapId['pages'][$origPid]) {
1903 $mappedPid = $this->import_mapId['pages'][$origPid];
1904 $cmd_data['pages'][$mappedPid]['move'] = $newPid;
1905 }
1906 } else {
1907 $cmd_data['pages'][$origPid]['move'] = $newPid;
1908 }
1909 }
1910 }
1911 // Execute the move commands if any:
1912 if (count($cmd_data)) {
1913 $tce = $this->getNewTCE();
1914 $this->callHook('before_writeRecordsPagesOrder', array(
1915 'tce' => &$tce,
1916 'data' => &$cmd_data
1917 ));
1918 $tce->start(array(), $cmd_data);
1919 $tce->process_cmdmap();
1920 $this->callHook('after_writeRecordsPagesOrder', array(
1921 'tce' => &$tce
1922 ));
1923 }
1924 }
1925
1926 /**
1927 * Write all database records except pages (writtein in writeRecords_pages())
1928 *
1929 * @param int $pid Page id in which to import
1930 * @return void
1931 * @see writeRecords_pages()
1932 */
1933 public function writeRecords_records($pid) {
1934 // Write the rest of the records
1935 $this->import_data = array();
1936 if (is_array($this->dat['header']['records'])) {
1937 foreach ($this->dat['header']['records'] as $table => $recs) {
1938 if ($table != 'pages') {
1939 foreach ($recs as $uid => $thisRec) {
1940 // PID: Set the main $pid, unless a NEW-id is found
1941 $setPid = isset($this->import_mapId['pages'][$thisRec['pid']]) ? $this->import_mapId['pages'][$thisRec['pid']] : $pid;
1942 if (is_array($GLOBALS['TCA'][$table]) && $GLOBALS['TCA'][$table]['ctrl']['rootLevel']) {
1943 $setPid = 0;
1944 }
1945 // Add record:
1946 $this->addSingle($table, $uid, $setPid);
1947 }
1948 }
1949 }
1950 } else {
1951 $this->error('Error: No records defined in internal data array.');
1952 }
1953 // Now write to database:
1954 $tce = $this->getNewTCE();
1955 $this->callHook('before_writeRecordsRecords', array(
1956 'tce' => &$tce,
1957 'data' => &$this->import_data
1958 ));
1959 $tce->suggestedInsertUids = $this->suggestedInsertUids;
1960 // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.
1961 $tce->reverseOrder = 1;
1962 $tce->isImporting = TRUE;
1963 $tce->start($this->import_data, array());
1964 $tce->process_datamap();
1965 $this->callHook('after_writeRecordsRecords', array(
1966 'tce' => &$tce
1967 ));
1968 // post-processing: Removing files and registering new ids (end all tcemain sessions with this)
1969 $this->addToMapId($tce->substNEWwithIDs);
1970 // In case of an update, order pages from the page tree correctly:
1971 if ($this->update) {
1972 $this->writeRecords_records_order($pid);
1973 }
1974 }
1975
1976 /**
1977 * Organize all updated record to their new positions.
1978 * Only used for updates
1979 *
1980 * @param int $mainPid Main PID into which we import.
1981 * @return void
1982 * @access private
1983 * @see writeRecords_records(), writeRecords_pages_order()
1984 */
1985 public function writeRecords_records_order($mainPid) {
1986 $cmd_data = array();
1987 if (is_array($this->dat['header']['pagetree'])) {
1988 $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);
1989 } else {
1990 $pagesFromTree = array();
1991 }
1992 if (is_array($this->dat['header']['pid_lookup'])) {
1993 foreach ($this->dat['header']['pid_lookup'] as $pid => $recList) {
1994 $newPid = isset($this->import_mapId['pages'][$pid]) ? $this->import_mapId['pages'][$pid] : $mainPid;
1995 if (\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($newPid)) {
1996 foreach ($recList as $tableName => $uidList) {
1997 // If $mainPid===$newPid then we are on root level and we can consider to move pages as well!
1998 // (they will not be in the page tree!)
1999 if (($tableName != 'pages' || !$pagesFromTree[$pid]) && is_array($uidList)) {
2000 $uidList = array_reverse(array_keys($uidList));
2001 foreach ($uidList as $uid) {
2002 if ($this->dontIgnorePid($tableName, $uid)) {
2003 $cmd_data[$tableName][$uid]['move'] = $newPid;
2004 } else {
2005
2006 }
2007 }
2008 }
2009 }
2010 }
2011 }
2012 }
2013 // Execute the move commands if any:
2014 if (count($cmd_data)) {
2015 $tce = $this->getNewTCE();
2016 $this->callHook('before_writeRecordsRecordsOrder', array(
2017 'tce' => &$tce,
2018 'data' => &$cmd_data
2019 ));
2020 $tce->start(array(), $cmd_data);
2021 $tce->process_cmdmap();
2022 $this->callHook('after_writeRecordsRecordsOrder', array(
2023 'tce' => &$tce
2024 ));
2025 }
2026 }
2027
2028 /**
2029 * Adds a single record to the $importData array. Also copies files to tempfolder.
2030 * However all File/DB-references and flexform field contents are set to blank for now!
2031 * That is done with setRelations() later
2032 *
2033 * @param string $table Table name (from import memory)
2034 * @param int $uid Record UID (from import memory)
2035 * @param int $pid Page id
2036 * @return void
2037 * @see writeRecords()
2038 */
2039 public function addSingle($table, $uid, $pid) {
2040 if ($this->import_mode[$table . ':' . $uid] === 'exclude') {
2041 return;
2042 }
2043 $record = $this->dat['records'][$table . ':' . $uid]['data'];
2044 if (is_array($record)) {
2045 if ($this->update && $this->doesRecordExist($table, $uid) && $this->import_mode[$table . ':' . $uid] !== 'as_new') {
2046 $ID = $uid;
2047 } elseif ($table === 'sys_file_metadata' && $record['sys_language_uid'] == '0' && $this->import_mapId['sys_file'][$record['file']]) {
2048 // on adding sys_file records the belonging sys_file_metadata record was also created
2049 // if there is one the record need to be overwritten instead of creating a new one.
2050 $recordInDatabase = $GLOBALS['TYPO3_DB']->exec_SELECTgetSingleRow(
2051 'uid',
2052 'sys_file_metadata',
2053 'file = ' . $this->import_mapId['sys_file'][$record['file']] . ' AND sys_language_uid = 0 AND pid = 0'
2054 );
2055 // if no record could be found, $this->import_mapId['sys_file'][$record['file']] is pointing
2056 // to a file, that was already there, thus a new metadata record should be created
2057 if (is_array($recordInDatabase)) {
2058 $this->import_mapId['sys_file_metadata'][$record['uid']] = $recordInDatabase['uid'];
2059 $ID = $recordInDatabase['uid'];
2060 } else {
2061 $ID = uniqid('NEW', TRUE);
2062 }
2063
2064 } else {
2065 $ID = uniqid('NEW', TRUE);
2066 }
2067 $this->import_newId[$table . ':' . $ID] = array('table' => $table, 'uid' => $uid);
2068 if ($table == 'pages') {
2069 $this->import_newId_pids[$uid] = $ID;
2070 }
2071 // Set main record data:
2072 $this->import_data[$table][$ID] = $record;
2073 $this->import_data[$table][$ID]['tx_impexp_origuid'] = $this->import_data[$table][$ID]['uid'];
2074 // Reset permission data:
2075 if ($table === 'pages') {
2076 // Have to reset the user/group IDs so pages are owned by importing user. Otherwise strange things may happen for non-admins!
2077 unset($this->import_data[$table][$ID]['perms_userid']);
2078 unset($this->import_data[$table][$ID]['perms_groupid']);
2079 }
2080 // PID and UID:
2081 unset($this->import_data[$table][$ID]['uid']);
2082 // Updates:
2083 if (\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($ID)) {
2084 unset($this->import_data[$table][$ID]['pid']);
2085 } else {
2086 // Inserts:
2087 $this->import_data[$table][$ID]['pid'] = $pid;
2088 if (($this->import_mode[$table . ':' . $uid] === 'force_uid' && $this->update || $this->force_all_UIDS) && $GLOBALS['BE_USER']->isAdmin()) {
2089 $this->import_data[$table][$ID]['uid'] = $uid;
2090 $this->suggestedInsertUids[$table . ':' . $uid] = 'DELETE';
2091 }
2092 }
2093 // Setting db/file blank:
2094 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
2095 switch ((string)$config['type']) {
2096 case 'db':
2097
2098 case 'file':
2099 // Fixed later in ->setRelations() [because we need to know ALL newly created IDs before we can map relations!]
2100 // In the meantime we set NO values for relations.
2101 //
2102 // BUT for field uid_local of table sys_file_reference the relation MUST not be cleared here,
2103 // because the value is already the uid of the right imported sys_file record.
2104 // @see fixUidLocalInSysFileReferenceRecords()
2105 // If it's empty or a uid to another record the FileExtensionFilter will throw an exception or
2106 // delete the reference record if the file extension of the related record doesn't match.
2107 if ($table !== 'sys_file_reference' && $field !== 'uid_local') {
2108 $this->import_data[$table][$ID][$field] = '';
2109 }
2110 break;
2111 case 'flex':
2112 // Fixed later in setFlexFormRelations()
2113 // In the meantime we set NO value for flexforms - this is mainly because file references
2114 // inside will not be processed properly; In fact references will point to no file
2115 // or existing files (in which case there will be double-references which is a big problem of course!)
2116 $this->import_data[$table][$ID][$field] = '';
2117 break;
2118 }
2119 }
2120 } elseif ($table . ':' . $uid != 'pages:0') {
2121 // On root level we don't want this error message.
2122 $this->error('Error: no record was found in data array!', 1);
2123 }
2124 }
2125
2126 /**
2127 * Registers the substNEWids in memory.
2128 *
2129 * @param array $substNEWwithIDs From tcemain to be merged into internal mapping variable in this object
2130 * @return void
2131 * @see writeRecords()
2132 */
2133 public function addToMapId($substNEWwithIDs) {
2134 foreach ($this->import_data as $table => $recs) {
2135 foreach ($recs as $id => $value) {
2136 $old_uid = $this->import_newId[$table . ':' . $id]['uid'];
2137 if (isset($substNEWwithIDs[$id])) {
2138 $this->import_mapId[$table][$old_uid] = $substNEWwithIDs[$id];
2139 } elseif ($this->update) {
2140 // Map same ID to same ID....
2141 $this->import_mapId[$table][$old_uid] = $id;
2142 } else {
2143 // if $this->import_mapId contains already the right mapping, skip the error msg.
2144 // See special handling of sys_file_metadata in addSingle() => nothing to do
2145 if (!($table === 'sys_file_metadata' && isset($this->import_mapId[$table][$old_uid]) && $this->import_mapId[$table][$old_uid] == $id)) {
2146 $this->error('Possible error: ' . $table . ':' . $old_uid . ' had no new id assigned to it. This indicates that the record was not added to database during import. Please check changelog!', 1);
2147 }
2148
2149 }
2150 }
2151 }
2152 }
2153
2154 /**
2155 * Returns a new $TCE object
2156 *
2157 * @return object $TCE object
2158 */
2159 public function getNewTCE() {
2160 $tce = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\DataHandling\\DataHandler');
2161 $tce->stripslashes_values = 0;
2162 $tce->dontProcessTransformations = 1;
2163 $tce->enableLogging = $this->enableLogging;
2164 $tce->alternativeFileName = $this->alternativeFileName;
2165 $tce->alternativeFilePath = $this->alternativeFilePath;
2166 return $tce;
2167 }
2168
2169 /**
2170 * Cleaning up all the temporary files stored in typo3temp/ folder
2171 *
2172 * @return void
2173 */
2174 public function unlinkTempFiles() {
2175 foreach ($this->unlinkFiles as $fileName) {
2176 if (GeneralUtility::isFirstPartOfStr($fileName, PATH_site . 'typo3temp/')) {
2177 GeneralUtility::unlink_tempfile($fileName);
2178 clearstatcache();
2179 if (is_file($fileName)) {
2180 $this->error('Error: ' . $fileName . ' was NOT unlinked as it should have been!', 1);
2181 }
2182 } else {
2183 $this->error('Error: ' . $fileName . ' was not in temp-path. Not removed!', 1);
2184 }
2185 }
2186 $this->unlinkFiles = array();
2187 }
2188
2189 /***************************
2190 * Import / Relations setting
2191 ***************************/
2192
2193 /**
2194 * At the end of the import process all file and DB relations should be set properly (that is relations
2195 * to imported records are all re-created so imported records are correctly related again)
2196 * Relations in flexform fields are processed in setFlexFormRelations() after this function
2197 *
2198 * @return void
2199 * @see setFlexFormRelations()
2200 */
2201 public function setRelations() {
2202 $updateData = array();
2203 // import_newId contains a register of all records that was in the import memorys "records" key
2204 foreach ($this->import_newId as $nId => $dat) {
2205 $table = $dat['table'];
2206 $uid = $dat['uid'];
2207 // original UID - NOT the new one!
2208 // If the record has been written and received a new id, then proceed:
2209 if (is_array($this->import_mapId[$table]) && isset($this->import_mapId[$table][$uid])) {
2210 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
2211 if (is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {
2212 $thisNewPageUid = 0;
2213 if ($this->legacyImport) {
2214 if ($table != 'pages') {
2215 $oldPid = $this->dat['records'][$table . ':' . $uid]['data']['pid'];
2216 $thisNewPageUid = BackendUtility::wsMapId($table, $this->import_mapId['pages'][$oldPid]);
2217 } else {
2218 $thisNewPageUid = $thisNewUid;
2219 }
2220 }
2221 // Traverse relation fields of each record
2222 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
2223 // uid_local of sys_file_reference needs no update because the correct reference uid was already written
2224 // @see ImportExport::fixUidLocalInSysFileReferenceRecords()
2225 if ($table === 'sys_file_reference' && $field === 'uid_local') {
2226 continue;
2227 }
2228 switch ((string)$config['type']) {
2229 case 'db':
2230 if (is_array($config['itemArray']) && count($config['itemArray'])) {
2231 $itemConfig = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
2232 $valArray = $this->setRelations_db($config['itemArray'], $itemConfig);
2233 $updateData[$table][$thisNewUid][$field] = implode(',', $valArray);
2234 }
2235 break;
2236 case 'file':
2237 if (is_array($config['newValueFiles']) && count($config['newValueFiles'])) {
2238 $valArr = array();
2239 foreach ($config['newValueFiles'] as $fI) {
2240 $valArr[] = $this->import_addFileNameToBeCopied($fI);
2241 }
2242 if ($this->legacyImport && $this->legacyImportFolder === NULL && isset($this->legacyImportMigrationTables[$table][$field])) {
2243 // Do nothing - the legacy import folder is missing
2244 } elseif ($this->legacyImport && $this->legacyImportFolder !== NULL && isset($this->legacyImportMigrationTables[$table][$field])) {
2245 $refIds = array();
2246 foreach ($valArr as $tempFile) {
2247 $fileName = $this->alternativeFileName[$tempFile];
2248 $fileObject = NULL;
2249
2250 try {
2251 // check, if there is alreay the same file in the folder
2252 if ($this->legacyImportFolder->hasFile($fileName)) {
2253 $fileStorage = $this->legacyImportFolder->getStorage();
2254 $file = $fileStorage->getFile($this->legacyImportFolder->getIdentifier() . $fileName);
2255 if ($file->getSha1() === sha1_file($tempFile)) {
2256 $fileObject = $file;
2257 }
2258 }
2259 } catch (Exception $e) {}
2260
2261 if ($fileObject === NULL) {
2262 try {
2263 $fileObject = $this->legacyImportFolder->addFile($tempFile, $fileName, 'changeName');
2264 } catch (\TYPO3\CMS\Core\Exception $e) {
2265 $this->error('Error: no file could be added to the storage for file name' . $this->alternativeFileName[$tempFile]);
2266 }
2267 }
2268 if ($fileObject !== NULL) {
2269 $refId = uniqid('NEW', TRUE);
2270 $refIds[] = $refId;
2271 $updateData['sys_file_reference'][$refId] = array(
2272 'uid_local' => $fileObject->getUid(),
2273 'uid_foreign' => $thisNewUid, // uid of your content record
2274 'tablenames' => $table,
2275 'fieldname' => $field,
2276 'pid' => $thisNewPageUid, // parent id of the parent page
2277 'table_local' => 'sys_file',
2278 );
2279 }
2280 }
2281 $updateData[$table][$thisNewUid][$field] = implode(',', $refIds);
2282 if (!empty($this->legacyImportMigrationTables[$table][$field])) {
2283 $this->legacyImportMigrationRecords[$table][$thisNewUid][$field] = $refIds;
2284 }
2285 } else {
2286 $updateData[$table][$thisNewUid][$field] = implode(',', $valArr);
2287 }
2288 }
2289 break;
2290 }
2291 }
2292 } else {
2293 $this->error('Error: no record was found in data array!', 1);
2294 }
2295 } else {
2296 $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')', 1);
2297 }
2298 }
2299 if (count($updateData)) {
2300 $tce = $this->getNewTCE();
2301 $tce->isImporting = TRUE;
2302 $this->callHook('before_setRelation', array(
2303 'tce' => &$tce,
2304 'data' => &$updateData
2305 ));
2306 $tce->start($updateData, array());
2307 $tce->process_datamap();
2308 // Replace the temporary "NEW" ids with the final ones.
2309 foreach ($this->legacyImportMigrationRecords as $table => $records) {
2310 foreach ($records as $uid => $fields) {
2311 foreach ($fields as $field => $referenceIds) {
2312 foreach ($referenceIds as $key => $referenceId) {
2313 $this->legacyImportMigrationRecords[$table][$uid][$field][$key] = $tce->substNEWwithIDs[$referenceId];
2314 }
2315 }
2316 }
2317 }
2318 $this->callHook('after_setRelations', array(
2319 'tce' => &$tce
2320 ));
2321 }
2322 }
2323
2324 /**
2325 * Maps relations for database
2326 *
2327 * @param array $itemArray Array of item sets (table/uid) from a dbAnalysis object
2328 * @param array $itemConfig Array of TCA config of the field the relation to be set on
2329 * @return array Array with values [table]_[uid] or [uid] for field of type group / internal_type file_reference. These values have the regular tcemain-input group/select type which means they will automatically be processed into a uid-list or MM relations.
2330 */
2331 public function setRelations_db($itemArray, $itemConfig) {
2332 $valArray = array();
2333 foreach ($itemArray as $relDat) {
2334 if (is_array($this->import_mapId[$relDat['table']]) && isset($this->import_mapId[$relDat['table']][$relDat['id']])) {
2335 // Since non FAL file relation type group internal_type file_reference are handled as reference to
2336 // sys_file records Datahandler requires the value as uid of the the related sys_file record only
2337 if ($itemConfig['type'] === 'group' && $itemConfig['internal_type'] === 'file_reference') {
2338 $value = $this->import_mapId[$relDat['table']][$relDat['id']];
2339 } else {
2340 $value = $relDat['table'] . '_' . $this->import_mapId[$relDat['table']][$relDat['id']];
2341 }
2342 $valArray[] = $value;
2343 } elseif ($this->isTableStatic($relDat['table']) || $this->isExcluded($relDat['table'], $relDat['id']) || $relDat['id'] < 0) {
2344 // Checking for less than zero because some select types could contain negative values,
2345 // eg. fe_groups (-1, -2) and sys_language (-1 = ALL languages). This must be handled on both export and import.
2346 $valArray[] = $relDat['table'] . '_' . $relDat['id'];
2347 } else {
2348 $this->error('Lost relation: ' . $relDat['table'] . ':' . $relDat['id'], 1);
2349 }
2350 }
2351 return $valArray;
2352 }
2353
2354 /**
2355 * Writes the file from import array to temp dir and returns the filename of it.
2356 *
2357 * @param array $fI File information with three keys: "filename" = filename without path, "ID_absFile" = absolute filepath to the file (including the filename), "ID" = md5 hash of "ID_absFile
2358 * @return string|NULL Absolute filename of the temporary filename of the file. In ->alternativeFileName the original name is set.
2359 */
2360 public function import_addFileNameToBeCopied($fI) {
2361 if (is_array($this->dat['files'][$fI['ID']])) {
2362 $tmpFile = NULL;
2363 // check if there is the right file already in the local folder
2364 if ($this->filesPathForImport !== NULL) {
2365 if (is_file($this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5']) &&
2366 md5_file($this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5']) === $this->dat['files'][$fI['ID']]['content_md5']) {
2367 $tmpFile = $this->filesPathForImport . '/' . $this->dat['files'][$fI['ID']]['content_md5'];
2368 }
2369 }
2370 if ($tmpFile === NULL) {
2371 $tmpFile = GeneralUtility::tempnam('import_temp_');
2372 GeneralUtility::writeFile($tmpFile, $this->dat['files'][$fI['ID']]['content']);
2373 }
2374 clearstatcache();
2375 if (@is_file($tmpFile)) {
2376 $this->unlinkFiles[] = $tmpFile;
2377 if (filesize($tmpFile) == $this->dat['files'][$fI['ID']]['filesize']) {
2378 $this->alternativeFileName[$tmpFile] = $fI['filename'];
2379 $this->alternativeFilePath[$tmpFile] = $this->dat['files'][$fI['ID']]['relFileRef'];
2380 return $tmpFile;
2381 } else {
2382 $this->error('Error: temporary file ' . $tmpFile . ' had a size (' . filesize($tmpFile) . ') different from the original (' . $this->dat['files'][$fI['ID']]['filesize'] . ')', 1);
2383 }
2384 } else {
2385 $this->error('Error: temporary file ' . $tmpFile . ' was not written as it should have been!', 1);
2386 }
2387 } else {
2388 $this->error('Error: No file found for ID ' . $fI['ID'], 1);
2389 }
2390 return NULL;
2391 }
2392
2393 /**
2394 * After all DB relations has been set in the end of the import (see setRelations()) then it is time to correct all relations inside of FlexForm fields.
2395 * The reason for doing this after is that the setting of relations may affect (quite often!) which data structure is used for the flexforms field!
2396 *
2397 * @return void
2398 * @see setRelations()
2399 */
2400 public function setFlexFormRelations() {
2401 $updateData = array();
2402 // import_newId contains a register of all records that was in the import memorys "records" key
2403 foreach ($this->import_newId as $nId => $dat) {
2404 $table = $dat['table'];
2405 $uid = $dat['uid'];
2406 // original UID - NOT the new one!
2407 // If the record has been written and received a new id, then proceed:
2408 if (!isset($this->import_mapId[$table][$uid])) {
2409 $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')', 1);
2410 continue;
2411 }
2412
2413 if (!is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {
2414 $this->error('Error: no record was found in data array!', 1);
2415 continue;
2416 }
2417 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
2418 // Traverse relation fields of each record
2419 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
2420 switch ((string)$config['type']) {
2421 case 'flex':
2422 // Get XML content and set as default value (string, non-processed):
2423 $updateData[$table][$thisNewUid][$field] = $this->dat['records'][$table . ':' . $uid]['data'][$field];
2424 // If there has been registered relations inside the flex form field, run processing on the content:
2425 if (count($config['flexFormRels']['db']) || count($config['flexFormRels']['file'])) {
2426 $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');
2427 // This will fetch the new row for the element (which should be updated with any references to data structures etc.)
2428 $conf = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
2429 if (is_array($origRecordRow) && is_array($conf) && $conf['type'] === 'flex') {
2430 // Get current data structure and value array:
2431 $dataStructArray = BackendUtility::getFlexFormDS($conf, $origRecordRow, $table, $field);
2432 $currentValueArray = GeneralUtility::xml2array($updateData[$table][$thisNewUid][$field]);
2433 // Do recursive processing of the XML data:
2434 $iteratorObj = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\DataHandling\\DataHandler');
2435 $iteratorObj->callBackObj = $this;
2436 $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData(
2437 $currentValueArray['data'],
2438 array(),
2439 array(),
2440 $dataStructArray,
2441 array($table, $thisNewUid, $field, $config),
2442 'remapListedDBRecords_flexFormCallBack'
2443 );
2444 // The return value is set as an array which means it will be processed by tcemain for file and DB references!
2445 if (is_array($currentValueArray['data'])) {
2446 $updateData[$table][$thisNewUid][$field] = $currentValueArray;
2447 }
2448 }
2449 }
2450 break;
2451 }
2452 }
2453 }
2454 if (count($updateData)) {
2455 $tce = $this->getNewTCE();
2456 $tce->isImporting = TRUE;
2457 $this->callHook('before_setFlexFormRelations', array(
2458 'tce' => &$tce,
2459 'data' => &$updateData
2460 ));
2461 $tce->start($updateData, array());
2462 $tce->process_datamap();
2463 $this->callHook('after_setFlexFormRelations', array(
2464 'tce' => &$tce
2465 ));
2466 }
2467 }
2468
2469 /**
2470 * Callback function for traversing the FlexForm structure in relation to remapping database relations
2471 *
2472 * @param array $pParams Set of parameters in numeric array: table, uid, field
2473 * @param array $dsConf TCA config for field (from Data Structure of course)
2474 * @param string $dataValue Field value (from FlexForm XML)
2475 * @param string $dataValue_ext1 Not used
2476 * @param string $dataValue_ext2 Not used
2477 * @param string $path Path of where the data structure of the element is found
2478 * @return array Array where the "value" key carries the value.
2479 * @see setFlexFormRelations()
2480 */
2481 public function remapListedDBRecords_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path) {
2482 // Extract parameters:
2483 list($table, $uid, $field, $config) = $pParams;
2484 // In case the $path is used as index without a trailing slash we will remove that
2485 if (!is_array($config['flexFormRels']['db'][$path]) && is_array($config['flexFormRels']['db'][rtrim($path, '/')])) {
2486 $path = rtrim($path, '/');
2487 }
2488 if (is_array($config['flexFormRels']['db'][$path])) {
2489 $valArray = $this->setRelations_db($config['flexFormRels']['db'][$path], $dsConf);
2490 $dataValue = implode(',', $valArray);
2491 }
2492 if (is_array($config['flexFormRels']['file'][$path])) {
2493 $valArr = array();
2494 foreach ($config['flexFormRels']['file'][$path] as $fI) {
2495 $valArr[] = $this->import_addFileNameToBeCopied($fI);
2496 }
2497 $dataValue = implode(',', $valArr);
2498 }
2499 return array('value' => $dataValue);
2500 }
2501
2502 /**************************
2503 * Import / Soft References
2504 *************************/
2505
2506 /**
2507 * Processing of soft references
2508 *
2509 * @return void
2510 */
2511 public function processSoftReferences() {
2512 // Initialize:
2513 $inData = array();
2514 // Traverse records:
2515 if (is_array($this->dat['header']['records'])) {
2516 foreach ($this->dat['header']['records'] as $table => $recs) {
2517 foreach ($recs as $uid => $thisRec) {
2518 // If there are soft references defined, traverse those:
2519 if (isset($GLOBALS['TCA'][$table]) && is_array($thisRec['softrefs'])) {
2520 // First traversal is to collect softref configuration and split them up based on fields.
2521 // This could probably also have been done with the "records" key instead of the header.
2522 $fieldsIndex = array();
2523 foreach ($thisRec['softrefs'] as $softrefDef) {
2524 // If a substitution token is set:
2525 if ($softrefDef['field'] && is_array($softrefDef['subst']) && $softrefDef['subst']['tokenID']) {
2526 $fieldsIndex[$softrefDef['field']][$softrefDef['subst']['tokenID']] = $softrefDef;
2527 }
2528 }
2529 // The new id:
2530 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
2531 // Now, if there are any fields that require substitution to be done, lets go for that:
2532 foreach ($fieldsIndex as $field => $softRefCfgs) {
2533 if (is_array($GLOBALS['TCA'][$table]['columns'][$field])) {
2534 $conf = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
2535 if ($conf['type'] === 'flex') {
2536 // This will fetch the new row for the element (which should be updated with any references to data structures etc.)
2537 $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');
2538 if (is_array($origRecordRow)) {
2539 // Get current data structure and value array:
2540 $dataStructArray = BackendUtility::getFlexFormDS($conf, $origRecordRow, $table, $field);
2541 $currentValueArray = GeneralUtility::xml2array($origRecordRow[$field]);
2542 // Do recursive processing of the XML data:
2543 /** @var $iteratorObj \TYPO3\CMS\Core\DataHandling\DataHandler */
2544 $iteratorObj = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\DataHandling\\DataHandler');
2545 $iteratorObj->callBackObj = $this;
2546 $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData($currentValueArray['data'], array(), array(), $dataStructArray, array($table, $uid, $field, $softRefCfgs), 'processSoftReferences_flexFormCallBack');
2547 // The return value is set as an array which means it will be processed by tcemain for file and DB references!
2548 if (is_array($currentValueArray['data'])) {
2549 $inData[$table][$thisNewUid][$field] = $currentValueArray;
2550 }
2551 }
2552 } else {
2553 // Get tokenizedContent string and proceed only if that is not blank:
2554 $tokenizedContent = $this->dat['records'][$table . ':' . $uid]['rels'][$field]['softrefs']['tokenizedContent'];
2555 if (strlen($tokenizedContent) && is_array($softRefCfgs)) {
2556 $inData[$table][$thisNewUid][$field] = $this->processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid);
2557 }
2558 }
2559 }
2560 }
2561 }
2562 }
2563 }
2564 }
2565 // Now write to database:
2566 $tce = $this->getNewTCE();
2567 $tce->isImporting = TRUE;
2568 $this->callHook('before_processSoftReferences', array(
2569 'tce' => $tce,
2570 'data' => &$inData
2571 ));
2572 $tce->enableLogging = TRUE;
2573 $tce->start($inData, array());
2574 $tce->process_datamap();
2575 $this->callHook('after_processSoftReferences', array(
2576 'tce' => $tce
2577 ));
2578 }
2579
2580 /**
2581 * Callback function for traversing the FlexForm structure in relation to remapping softreference relations
2582 *
2583 * @param array $pParams Set of parameters in numeric array: table, uid, field
2584 * @param array $dsConf TCA config for field (from Data Structure of course)
2585 * @param string $dataValue Field value (from FlexForm XML)
2586 * @param string $dataValue_ext1 Not used
2587 * @param string $dataValue_ext2 Not used
2588 * @param string $path Path of where the data structure where the element is found
2589 * @return array Array where the "value" key carries the value.
2590 * @see setFlexFormRelations()
2591 */
2592 public function processSoftReferences_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path) {
2593 // Extract parameters:
2594 list($table, $origUid, $field, $softRefCfgs) = $pParams;
2595 if (is_array($softRefCfgs)) {
2596 // First, find all soft reference configurations for this structure path (they are listed flat in the header):
2597 $thisSoftRefCfgList = array();
2598 foreach ($softRefCfgs as $sK => $sV) {
2599 if ($sV['structurePath'] === $path) {
2600 $thisSoftRefCfgList[$sK] = $sV;
2601 }
2602 }
2603 // If any was found, do processing:
2604 if (count($thisSoftRefCfgList)) {
2605 // Get tokenizedContent string and proceed only if that is not blank:
2606 $tokenizedContent = $this->dat['records'][$table . ':' . $origUid]['rels'][$field]['flexFormRels']['softrefs'][$path]['tokenizedContent'];
2607 if (strlen($tokenizedContent)) {
2608 $dataValue = $this->processSoftReferences_substTokens($tokenizedContent, $thisSoftRefCfgList, $table, $origUid);
2609 }
2610 }
2611 }
2612 // Return
2613 return array('value' => $dataValue);
2614 }
2615
2616 /**
2617 * Substition of softreference tokens
2618 *
2619 * @param string $tokenizedContent Content of field with soft reference tokens in.
2620 * @param array $softRefCfgs Soft reference configurations
2621 * @param string $table Table for which the processing occurs
2622 * @param string $uid UID of record from table
2623 * @return string The input content with tokens substituted according to entries in softRefCfgs
2624 */
2625 public function processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid) {
2626 // traverse each softref type for this field:
2627 foreach ($softRefCfgs as $cfg) {
2628 // Get token ID:
2629 $tokenID = $cfg['subst']['tokenID'];
2630 // Default is current token value:
2631 $insertValue = $cfg['subst']['tokenValue'];
2632 // Based on mode:
2633 switch ((string)$this->softrefCfg[$tokenID]['mode']) {
2634 case 'exclude':
2635 // Exclude is a simple passthrough of the value
2636 break;
2637 case 'editable':
2638 // Editable always picks up the value from this input array:
2639 $insertValue = $this->softrefInputValues[$tokenID];
2640 break;
2641 default:
2642 // Mapping IDs/creating files: Based on type, look up new value:
2643 switch ((string)$cfg['subst']['type']) {
2644 case 'file':
2645 // Create / Overwrite file:
2646 $insertValue = $this->processSoftReferences_saveFile($cfg['subst']['relFileName'], $cfg, $table, $uid);
2647 break;
2648 case 'db':
2649 default:
2650 // Trying to map database element if found in the mapID array:
2651 list($tempTable, $tempUid) = explode(':', $cfg['subst']['recordRef']);
2652 if (isset($this->import_mapId[$tempTable][$tempUid])) {
2653 $insertValue = BackendUtility::wsMapId($tempTable, $this->import_mapId[$tempTable][$tempUid]);
2654 // Look if reference is to a page and the original token value was NOT an integer - then we assume is was an alias and try to look up the new one!
2655 if ($tempTable === 'pages' && !\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($cfg['subst']['tokenValue'])) {
2656 $recWithUniqueValue = BackendUtility::getRecord($tempTable, $insertValue, 'alias');
2657 if ($recWithUniqueValue['alias']) {
2658 $insertValue = $recWithUniqueValue['alias'];
2659 }
2660 } elseif (strpos($cfg['subst']['tokenValue'], ':') !== FALSE) {
2661 list($tokenKey, $tokenId) = explode(':', $cfg['subst']['tokenValue']);
2662 $insertValue = $tokenKey . ':' . $insertValue;
2663 }
2664 }
2665 }
2666 }
2667 // Finally, swap the soft reference token in tokenized content with the insert value:
2668 $tokenizedContent = str_replace('{softref:' . $tokenID . '}', $insertValue, $tokenizedContent);
2669 }
2670 return $tokenizedContent;
2671 }
2672
2673 /**
2674 * Process a soft reference file
2675 *
2676 * @param string $relFileName Old Relative filename
2677 * @param array $cfg soft reference configuration array
2678 * @param string $table Table for which the processing occurs
2679 * @param string $uid UID of record from table
2680 * @return string New relative filename (value to insert instead of the softref token)
2681 */
2682 public function processSoftReferences_saveFile($relFileName, $cfg, $table, $uid) {
2683 if ($fileHeaderInfo = $this->dat['header']['files'][$cfg['file_ID']]) {
2684 // Initialize; Get directory prefix for file and find possible RTE filename
2685 $dirPrefix = PathUtility::dirname($relFileName) . '/';
2686 $rteOrigName = $this->getRTEoriginalFilename(PathUtility::basename($relFileName));
2687 // If filename looks like an RTE file, and the directory is in "uploads/", then process as a RTE file!
2688 if ($rteOrigName && GeneralUtility::isFirstPartOfStr($dirPrefix, 'uploads/')) {
2689 // RTE:
2690 // First, find unique RTE file name:
2691 if (@is_dir((PATH_site . $dirPrefix))) {
2692 // From the "original" RTE filename, produce a new "original" destination filename which is unused.
2693 // Even if updated, the image should be unique. Currently the problem with this is that it leaves a lot of unused RTE images...
2694 $fileProcObj = $this->getFileProcObj();
2695 $origDestName = $fileProcObj->getUniqueName($rteOrigName, PATH_site . $dirPrefix);
2696 // Create copy file name:
2697 $pI = pathinfo($relFileName);
2698 $copyDestName = PathUtility::dirname($origDestName) . '/RTEmagicC_' . substr(PathUtility::basename($origDestName), 10) . '.' . $pI['extension'];
2699 if (
2700 !@is_file($copyDestName) && !@is_file($origDestName)
2701 && $origDestName === GeneralUtility::getFileAbsFileName($origDestName)
2702 && $copyDestName === GeneralUtility::getFileAbsFileName($copyDestName)
2703 ) {
2704 if ($this->dat['header']['files'][$fileHeaderInfo['RTE_ORIG_ID']]) {
2705 if ($this->legacyImport) {
2706 $fileName = PathUtility::basename($copyDestName);
2707 $this->writeSysFileResourceForLegacyImport($fileName, $cfg['file_ID']);
2708 $relFileName = $this->filePathMap[$cfg['file_ID']] . '" data-htmlarea-file-uid="' . $fileName . '" data-htmlarea-file-table="sys_file';
2709 // Also save the original file
2710 $originalFileName = PathUtility::basename($origDestName);
2711 $this->writeSysFileResourceForLegacyImport($originalFileName, $fileHeaderInfo['RTE_ORIG_ID']);
2712 } else {
2713 // Write the copy and original RTE file to the respective filenames:
2714 $this->writeFileVerify($copyDestName, $cfg['file_ID'], TRUE);
2715 $this->writeFileVerify($origDestName, $fileHeaderInfo['RTE_ORIG_ID'], TRUE);
2716 // Return the relative path of the copy file name:
2717 return PathUtility::stripPathSitePrefix($copyDestName);
2718 }
2719 } else {
2720 $this->error('ERROR: Could not find original file ID');
2721 }
2722 } else {
2723 $this->error('ERROR: The destination filenames "' . $copyDestName . '" and "' . $origDestName . '" either existed or have non-valid names');
2724 }
2725 } else {
2726 $this->error('ERROR: "' . PATH_site . $dirPrefix . '" was not a directory, so could not process file "' . $relFileName . '"');
2727 }
2728 } elseif (GeneralUtility::isFirstPartOfStr($dirPrefix, $this->fileadminFolderName . '/')) {
2729 // File in fileadmin/ folder:
2730 // Create file (and possible resources)
2731 $newFileName = $this->processSoftReferences_saveFile_createRelFile($dirPrefix, PathUtility::basename($relFileName), $cfg['file_ID'], $table, $uid);
2732 if (strlen($newFileName)) {
2733 $relFileName = $newFileName;
2734 } else {
2735 $this->error('ERROR: No new file created for "' . $relFileName . '"');
2736 }
2737 } else {
2738 $this->error('ERROR: Sorry, cannot operate on non-RTE files which are outside the fileadmin folder.');
2739 }
2740 } else {
2741 $this->error('ERROR: Could not find file ID in header.');
2742 }
2743 // Return (new) filename relative to PATH_site:
2744 return $relFileName;
2745 }
2746
2747 /**
2748 * Create file in directory and return the new (unique) filename
2749 *
2750 * @param string $origDirPrefix Directory prefix, relative, with trailing slash
2751 * @param string $fileName Filename (without path)
2752 * @param string $fileID File ID from import memory
2753 * @param string $table Table for which the processing occurs
2754 * @param string $uid UID of record from table
2755 * @return string|NULL New relative filename, if any
2756 */
2757 public function processSoftReferences_saveFile_createRelFile($origDirPrefix, $fileName, $fileID, $table, $uid) {
2758 // If the fileID map contains an entry for this fileID then just return the relative filename of that entry;
2759 // we don't want to write another unique filename for this one!
2760 if (isset($this->fileIDMap[$fileID])) {
2761 return PathUtility::stripPathSitePrefix($this->fileIDMap[$fileID]);
2762 }
2763 if ($this->legacyImport) {
2764 // set dirPrefix to fileadmin because the right target folder is set and checked for permissions later
2765 $dirPrefix = $this->fileadminFolderName . '/';
2766 } else {
2767 // Verify FileMount access to dir-prefix. Returns the best alternative relative path if any
2768 $dirPrefix = $this->verifyFolderAccess($origDirPrefix);
2769 }
2770 if ($dirPrefix && (!$this->update || $origDirPrefix === $dirPrefix) && $this->checkOrCreateDir($dirPrefix)) {
2771 $fileHeaderInfo = $this->dat['header']['files'][$fileID];
2772 $updMode = $this->update && $this->import_mapId[$table][$uid] === $uid && $this->import_mode[$table . ':' . $uid] !== 'as_new';
2773 // Create new name for file:
2774 // Must have same ID in map array (just for security, is not really needed) and NOT be set "as_new".
2775
2776 // Write main file:
2777 if ($this->legacyImport) {
2778 $fileWritten = $this->writeSysFileResourceForLegacyImport($fileName, $fileID);
2779 if ($fileWritten) {
2780 $newName = 'file:' . $fileName;
2781 return $newName;
2782 // no support for HTML/CSS file resources attached ATM - see below
2783 }
2784 } else {
2785 if ($updMode) {
2786 $newName = PATH_site . $dirPrefix . $fileName;
2787 } else {
2788 // Create unique filename:
2789 $fileProcObj = $this->getFileProcObj();
2790 $newName = $fileProcObj->getUniqueName($fileName, PATH_site . $dirPrefix);
2791 }
2792 if ($this->writeFileVerify($newName, $fileID)) {
2793 // If the resource was an HTML/CSS file with resources attached, we will write those as well!
2794 if (is_array($fileHeaderInfo['EXT_RES_ID'])) {
2795 $tokenizedContent = $this->dat['files'][$fileID]['tokenizedContent'];
2796 $tokenSubstituted = FALSE;
2797 $fileProcObj = $this->getFileProcObj();
2798 if ($updMode) {
2799 foreach ($fileHeaderInfo['EXT_RES_ID'] as $res_fileID) {
2800 if ($this->dat['files'][$res_fileID]['filename']) {
2801 // Resolve original filename:
2802 $relResourceFileName = $this->dat['files'][$res_fileID]['parentRelFileName'];
2803 $absResourceFileName = GeneralUtility::resolveBackPath(PATH_site . $origDirPrefix . $relResourceFileName);
2804 $absResourceFileName = GeneralUtility::getFileAbsFileName($absResourceFileName);
2805 if ($absResourceFileName && GeneralUtility::isFirstPartOfStr($absResourceFileName, PATH_site . $this->fileadminFolderName .