cecea212549c22fcbf76fbc2e6576bcd791a97b6
[Packages/TYPO3.CMS.git] / typo3 / sysext / impexp / Classes / ImportExport.php
1 <?php
2 namespace TYPO3\CMS\Impexp;
3
4 /***************************************************************
5 * Copyright notice
6 *
7 * (c) 1999-2013 Kasper Skårhøj (kasperYYYY@typo3.com)
8 * All rights reserved
9 *
10 * This script is part of the TYPO3 project. The TYPO3 project is
11 * free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2 of the License, or
14 * (at your option) any later version.
15 *
16 * The GNU General Public License can be found at
17 * http://www.gnu.org/copyleft/gpl.html.
18 * A copy is found in the text file GPL.txt and important notices to the license
19 * from the author is found in LICENSE.txt distributed with these scripts.
20 *
21 *
22 * This script is distributed in the hope that it will be useful,
23 * but WITHOUT ANY WARRANTY; without even the implied warranty of
24 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25 * GNU General Public License for more details.
26 *
27 * This copyright notice MUST APPEAR in all copies of the script!
28 ***************************************************************/
29
30 use TYPO3\CMS\Backend\Utility\BackendUtility;
31 use TYPO3\CMS\Core\Exception;
32 use TYPO3\CMS\Core\Utility\GeneralUtility;
33 use TYPO3\CMS\Core\Utility\PathUtility;
34
35 /**
36 * EXAMPLE for using the impexp-class for exporting stuff:
37 *
38 * Create and initialize:
39 * $this->export = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance('TYPO3\\CMS\\Impexp\\ImportExport');
40 * $this->export->init();
41 * Set which tables relations we will allow:
42 * $this->export->relOnlyTables[]="tt_news"; // exclusively includes. See comment in the class
43 *
44 * Adding records:
45 * $this->export->export_addRecord("pages", $this->pageinfo);
46 * $this->export->export_addRecord("pages", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("pages", 38));
47 * $this->export->export_addRecord("pages", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("pages", 39));
48 * $this->export->export_addRecord("tt_content", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("tt_content", 12));
49 * $this->export->export_addRecord("tt_content", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("tt_content", 74));
50 * $this->export->export_addRecord("sys_template", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("sys_template", 20));
51 *
52 * Adding all the relations (recursively in 5 levels so relations has THEIR relations registered as well)
53 * for($a=0;$a<5;$a++) {
54 * $addR = $this->export->export_addDBRelations($a);
55 * if (!count($addR)) break;
56 * }
57 *
58 * Finally load all the files.
59 * $this->export->export_addFilesFromRelations(); // MUST be after the DBrelations are set so that file from ALL added records are included!
60 *
61 * Write export
62 * $out = $this->export->compileMemoryToFileContent();
63 */
64 /**
65 * T3D file Import/Export library (TYPO3 Record Document)
66 *
67 * @author Kasper Skårhøj <kasperYYYY@typo3.com>
68 */
69 class ImportExport {
70
71 /**
72 * If set, static relations (not exported) will be shown in overview as well
73 *
74 * @todo Define visibility
75 */
76 public $showStaticRelations = FALSE;
77
78 /**
79 * Name of the "fileadmin" folder where files for export/import should be located
80 *
81 * @todo Define visibility
82 */
83 public $fileadminFolderName = '';
84
85 /**
86 * Whether "import" or "export" mode of object. Set through init() function
87 *
88 * @todo Define visibility
89 */
90 public $mode = '';
91
92 /**
93 * Updates all records that has same UID instead of creating new!
94 *
95 * @todo Define visibility
96 */
97 public $update = FALSE;
98
99 /**
100 * Is set by importData() when an import has been done.
101 *
102 * @todo Define visibility
103 */
104 public $doesImport = FALSE;
105
106 /**
107 * If set to a page-record, then the preview display of the content will expect this page-record to be the target
108 * for the import and accordingly display validation information. This triggers the visual view of the
109 * import/export memory to validate if import is possible
110 *
111 * @todo Define visibility
112 */
113 public $display_import_pid_record = '';
114
115 /**
116 * Used to register the forged UID values for imported records that we want to create with the same UIDs as in the import file. Admin-only feature.
117 *
118 * @todo Define visibility
119 */
120 public $suggestedInsertUids = array();
121
122 /**
123 * Setting import modes during update state: as_new, exclude, force_uid
124 *
125 * @todo Define visibility
126 */
127 public $import_mode = array();
128
129 /**
130 * If set, PID correct is ignored globally
131 *
132 * @todo Define visibility
133 */
134 public $global_ignore_pid = FALSE;
135
136 /**
137 * If set, all UID values are forced! (update or import)
138 *
139 * @todo Define visibility
140 */
141 public $force_all_UIDS = FALSE;
142
143 /**
144 * If set, a diff-view column is added to the overview.
145 *
146 * @todo Define visibility
147 */
148 public $showDiff = FALSE;
149
150 /**
151 * If set, and if the user is admin, allow the writing of PHP scripts to fileadmin/ area.
152 *
153 * @todo Define visibility
154 */
155 public $allowPHPScripts = FALSE;
156
157 /**
158 * Disable logging when importing
159 *
160 * @todo Define visibility
161 */
162 public $enableLogging = FALSE;
163
164 /**
165 * Array of values to substitute in editable softreferences.
166 *
167 * @todo Define visibility
168 */
169 public $softrefInputValues = array();
170
171 /**
172 * Mapping between the fileID from import memory and the final filenames they are written to.
173 *
174 * @todo Define visibility
175 */
176 public $fileIDMap = array();
177
178 /**
179 * 1MB max file size
180 *
181 * @todo Define visibility
182 */
183 public $maxFileSize = 1000000;
184
185 /**
186 * 1MB max record size
187 *
188 * @todo Define visibility
189 */
190 public $maxRecordSize = 1000000;
191
192 /**
193 * 10MB max export size
194 *
195 * @todo Define visibility
196 */
197 public $maxExportSize = 10000000;
198
199 /**
200 * Add table names here which are THE ONLY ones which will be included into export if found as relations. '_ALL' will allow all tables.
201 *
202 * @todo Define visibility
203 */
204 public $relOnlyTables = array();
205
206 /**
207 * Add tables names here which should not be exported with the file. (Where relations should be mapped to same UIDs in target system).
208 *
209 * @todo Define visibility
210 */
211 public $relStaticTables = array();
212
213 /**
214 * Exclude map. Keys are table:uid pairs and if set, records are not added to the export.
215 *
216 * @todo Define visibility
217 */
218 public $excludeMap = array();
219
220 /**
221 * Soft Reference Token ID modes.
222 *
223 * @todo Define visibility
224 */
225 public $softrefCfg = array();
226
227 /**
228 * Listing extension dependencies.
229 *
230 * @todo Define visibility
231 */
232 public $extensionDependencies = array();
233
234 /**
235 * Set by user: If set, compression in t3d files is disabled
236 *
237 * @todo Define visibility
238 */
239 public $dontCompress = 0;
240
241 /**
242 * Boolean, if set, HTML file resources are included.
243 *
244 * @todo Define visibility
245 */
246 public $includeExtFileResources = 0;
247
248 /**
249 * Files with external media (HTML/css style references inside)
250 *
251 * @todo Define visibility
252 */
253 public $extFileResourceExtensions = 'html,htm,css';
254
255 /**
256 * After records are written this array is filled with [table][original_uid] = [new_uid]
257 *
258 * @todo Define visibility
259 */
260 public $import_mapId = array();
261
262 /**
263 * Keys are [tablename]:[new NEWxxx ids (or when updating it is uids)] while values are arrays with table/uid of the original record it is based on.
264 * By the array keys the new ids can be looked up inside tcemain
265 *
266 * @todo Define visibility
267 */
268 public $import_newId = array();
269
270 /**
271 * Page id map for page tree (import)
272 *
273 * @todo Define visibility
274 */
275 public $import_newId_pids = array();
276
277 /**
278 * Internal data accumulation for writing records during import
279 *
280 * @todo Define visibility
281 */
282 public $import_data = array();
283
284 /**
285 * Error log.
286 *
287 * @todo Define visibility
288 */
289 public $errorLog = array();
290
291 /**
292 * Cache for record paths
293 *
294 * @todo Define visibility
295 */
296 public $cache_getRecordPath = array();
297
298 /**
299 * Cache of checkPID values.
300 *
301 * @todo Define visibility
302 */
303 public $checkPID_cache = array();
304
305 /**
306 * Set internally if the gzcompress function exists
307 *
308 * @todo Define visibility
309 */
310 public $compress = 0;
311
312 /**
313 * Internal import/export memory
314 *
315 * @todo Define visibility
316 */
317 public $dat = array();
318
319 /**
320 * File processing object
321 *
322 * @var \TYPO3\CMS\Core\Utility\File\ExtendedFileUtility
323 * @todo Define visibility
324 */
325 public $fileProcObj = '';
326
327 /**
328 * Keys are [recordname], values are an array of fields to be included
329 * in the export
330 *
331 * @var array
332 */
333 protected $recordTypesIncludeFields = array();
334
335 /**
336 * Default array of fields to be included in the export
337 *
338 * @var array
339 */
340 protected $defaultRecordIncludeFields = array('uid', 'pid');
341
342 /**
343 * Array of current registered storage objects
344 *
345 * @var array
346 */
347 protected $storageObjects = array();
348
349 /**
350 * Is set, if the import file has a TYPO3 version below 6.0
351 *
352 * @var bool
353 */
354 protected $legacyImport = FALSE;
355
356 /**
357 * @var \TYPO3\CMS\Core\Resource\Folder
358 */
359 protected $legacyImportFolder = NULL;
360
361 /**
362 * Related to the default storage root
363 *
364 * @var string
365 */
366 protected $legacyImportTargetPath = '_imported/';
367
368 /**
369 * Table fields to migrate
370 *
371 * @var array
372 */
373 protected $legacyImportMigrationTables = array(
374 'tt_content' => array(
375 'image' => array(
376 'titleTexts' => 'titleText',
377 'description' => 'imagecaption',
378 'links' => 'image_link',
379 'alternativeTexts' => 'altText'
380 ),
381 'media' => array(
382 'description' => 'imagecaption',
383 )
384 ),
385 'pages' => array(
386 'media' => array()
387 ),
388 'pages_language_overlay' => array(
389 'media' => array()
390 )
391 );
392
393
394 /**
395 * Records to be migrated after all
396 * Multidimensional array [table][uid][field] = array([related sys_file_reference uids])
397 *
398 * @var array
399 */
400 protected $legacyImportMigrationRecords = array();
401
402
403 /**************************
404 * Initialize
405 *************************/
406
407 /**
408 * Init the object, both import and export
409 *
410 * @param boolean $dontCompress If set, compression in t3d files is disabled
411 * @param string $mode Mode of usage, either "import" or "export
412 * @return void
413 * @todo Define visibility
414 */
415 public function init($dontCompress = 0, $mode = '') {
416 $this->compress = function_exists('gzcompress');
417 $this->dontCompress = $dontCompress;
418 $this->mode = $mode;
419 $this->fileadminFolderName = !empty($GLOBALS['TYPO3_CONF_VARS']['BE']['fileadminDir']) ? rtrim($GLOBALS['TYPO3_CONF_VARS']['BE']['fileadminDir'], '/') : 'fileadmin';
420 }
421
422 /**************************
423 * Export / Init + Meta Data
424 *************************/
425
426 /**
427 * Set header basics
428 *
429 * @return void
430 * @todo Define visibility
431 */
432 public function setHeaderBasics() {
433 // Initializing:
434 if (is_array($this->softrefCfg)) {
435 foreach ($this->softrefCfg as $key => $value) {
436 if (!strlen($value['mode'])) {
437 unset($this->softrefCfg[$key]);
438 }
439 }
440 }
441 // Setting in header memory:
442 // Version of file format
443 $this->dat['header']['XMLversion'] = '1.0';
444 // Initialize meta data array (to put it in top of file)
445 $this->dat['header']['meta'] = array();
446 // Add list of tables to consider static
447 $this->dat['header']['relStaticTables'] = $this->relStaticTables;
448 // The list of excluded records
449 $this->dat['header']['excludeMap'] = $this->excludeMap;
450 // Soft Reference mode for elements
451 $this->dat['header']['softrefCfg'] = $this->softrefCfg;
452 // List of extensions the import depends on.
453 $this->dat['header']['extensionDependencies'] = $this->extensionDependencies;
454 }
455
456 /**
457 * Set charset
458 *
459 * @param string $charset Charset for the content in the export. During import the character set will be converted if the target system uses another charset.
460 * @return void
461 * @todo Define visibility
462 */
463 public function setCharset($charset) {
464 $this->dat['header']['charset'] = $charset;
465 }
466
467 /**
468 * Sets meta data
469 *
470 * @param string $title Title of the export
471 * @param string $description Description of the export
472 * @param string $notes Notes about the contents
473 * @param string $packager_username Backend Username of the packager (the guy making the export)
474 * @param string $packager_name Real name of the packager
475 * @param string $packager_email Email of the packager
476 * @return void
477 * @todo Define visibility
478 */
479 public function setMetaData($title, $description, $notes, $packager_username, $packager_name, $packager_email) {
480 $this->dat['header']['meta'] = array(
481 'title' => $title,
482 'description' => $description,
483 'notes' => $notes,
484 'packager_username' => $packager_username,
485 'packager_name' => $packager_name,
486 'packager_email' => $packager_email,
487 'TYPO3_version' => TYPO3_version,
488 'created' => strftime('%A %e. %B %Y', $GLOBALS['EXEC_TIME'])
489 );
490 }
491
492 /**
493 * Sets a thumbnail image to the exported file
494 *
495 * @param string $imgFilepath Filename reference, gif, jpg, png. Absolute path.
496 * @return void
497 * @todo Define visibility
498 */
499 public function addThumbnail($imgFilepath) {
500 if (@is_file($imgFilepath)) {
501 $imgInfo = @getimagesize($imgFilepath);
502 if (is_array($imgInfo)) {
503 $fileContent = GeneralUtility::getUrl($imgFilepath);
504 $this->dat['header']['thumbnail'] = array(
505 'imgInfo' => $imgInfo,
506 'content' => $fileContent,
507 'filesize' => strlen($fileContent),
508 'filemtime' => filemtime($imgFilepath),
509 'filename' => PathUtility::basename($imgFilepath)
510 );
511 }
512 }
513 }
514
515 /**************************
516 * Export / Init Page tree
517 *************************/
518
519 /**
520 * Sets the page-tree array in the export header and returns the array in a flattened version
521 *
522 * @param array $idH Hierarchy of ids, the page tree: array([uid] => array("uid" => [uid], "subrow" => array(.....)), [uid] => ....)
523 * @return array The hierarchical page tree converted to a one-dimensional list of pages
524 * @todo Define visibility
525 */
526 public function setPageTree($idH) {
527 $this->dat['header']['pagetree'] = $this->unsetExcludedSections($idH);
528 return $this->flatInversePageTree($this->dat['header']['pagetree']);
529 }
530
531 /**
532 * Removes entries in the page tree which are found in ->excludeMap[]
533 *
534 * @param array $idH Page uid hierarchy
535 * @return array Modified input array
536 * @access private
537 * @see setPageTree()
538 * @todo Define visibility
539 */
540 public function unsetExcludedSections($idH) {
541 if (is_array($idH)) {
542 foreach ($idH as $k => $v) {
543 if ($this->excludeMap['pages:' . $idH[$k]['uid']]) {
544 unset($idH[$k]);
545 } elseif (is_array($idH[$k]['subrow'])) {
546 $idH[$k]['subrow'] = $this->unsetExcludedSections($idH[$k]['subrow']);
547 }
548 }
549 }
550 return $idH;
551 }
552
553 /**
554 * Recursively flattening the idH array (for setPageTree() function)
555 *
556 * @param array $idH Page uid hierarchy
557 * @param array $a Accumulation array of pages (internal, don't set from outside)
558 * @return array Array with uid-uid pairs for all pages in the page tree.
559 * @see flatInversePageTree_pid()
560 * @todo Define visibility
561 */
562 public function flatInversePageTree($idH, $a = array()) {
563 if (is_array($idH)) {
564 $idH = array_reverse($idH);
565 foreach ($idH as $k => $v) {
566 $a[$v['uid']] = $v['uid'];
567 if (is_array($v['subrow'])) {
568 $a = $this->flatInversePageTree($v['subrow'], $a);
569 }
570 }
571 }
572 return $a;
573 }
574
575 /**
576 * Recursively flattening the idH array (for setPageTree() function), setting PIDs as values
577 *
578 * @param array $idH Page uid hierarchy
579 * @param array $a Accumulation array of pages (internal, don't set from outside)
580 * @param integer $pid PID value (internal)
581 * @return array Array with uid-pid pairs for all pages in the page tree.
582 * @see flatInversePageTree()
583 * @todo Define visibility
584 */
585 public function flatInversePageTree_pid($idH, $a = array(), $pid = -1) {
586 if (is_array($idH)) {
587 $idH = array_reverse($idH);
588 foreach ($idH as $k => $v) {
589 $a[$v['uid']] = $pid;
590 if (is_array($v['subrow'])) {
591 $a = $this->flatInversePageTree_pid($v['subrow'], $a, $v['uid']);
592 }
593 }
594 }
595 return $a;
596 }
597
598 /**************************
599 * Export
600 *************************/
601
602 /**
603 * Sets the fields of record types to be included in the export
604 *
605 * @param array $recordTypesIncludeFields Keys are [recordname], values are an array of fields to be included in the export
606 * @throws \TYPO3\CMS\Core\Exception if an array value is not type of array
607 * @return void
608 */
609 public function setRecordTypesIncludeFields(array $recordTypesIncludeFields) {
610 foreach ($recordTypesIncludeFields as $table => $fields) {
611 if (!is_array($fields)) {
612 throw new \TYPO3\CMS\Core\Exception('The include fields for record type ' . htmlspecialchars($table) . ' are not defined by an array.', 1391440658);
613 }
614 $this->setRecordTypeIncludeFields($table, $fields);
615 }
616 }
617
618 /**
619 * Sets the fields of a record type to be included in the export
620 *
621 * @param string $table The record type
622 * @param array $fields The fields to be included
623 * @return void
624 */
625 public function setRecordTypeIncludeFields($table, array $fields) {
626 $this->recordTypesIncludeFields[$table] = $fields;
627 }
628
629 /**
630 * Adds the record $row from $table.
631 * No checking for relations done here. Pure data.
632 *
633 * @param string $table Table name
634 * @param array $row Record row.
635 * @param integer $relationLevel (Internal) if the record is added as a relation, this is set to the "level" it was on.
636 * @return void
637 * @todo Define visibility
638 */
639 public function export_addRecord($table, $row, $relationLevel = 0) {
640 BackendUtility::workspaceOL($table, $row);
641 if ((string)$table !== '' && is_array($row) && $row['uid'] > 0 && !$this->excludeMap[($table . ':' . $row['uid'])]) {
642 if ($this->checkPID($table === 'pages' ? $row['uid'] : $row['pid'])) {
643 if (!isset($this->dat['records'][($table . ':' . $row['uid'])])) {
644 // Prepare header info:
645 $row = $this->filterRecordFields($table, $row);
646 $headerInfo = array();
647 $headerInfo['uid'] = $row['uid'];
648 $headerInfo['pid'] = $row['pid'];
649 $headerInfo['title'] = GeneralUtility::fixed_lgd_cs(BackendUtility::getRecordTitle($table, $row), 40);
650 $headerInfo['size'] = strlen(serialize($row));
651 if ($relationLevel) {
652 $headerInfo['relationLevel'] = $relationLevel;
653 }
654 // If record content is not too large in size, set the header content and add the rest:
655 if ($headerInfo['size'] < $this->maxRecordSize) {
656 // Set the header summary:
657 $this->dat['header']['records'][$table][$row['uid']] = $headerInfo;
658 // Create entry in the PID lookup:
659 $this->dat['header']['pid_lookup'][$row['pid']][$table][$row['uid']] = 1;
660 // Initialize reference index object:
661 $refIndexObj = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\Database\\ReferenceIndex');
662 // Yes to workspace overlays for exporting....
663 $refIndexObj->WSOL = TRUE;
664 $relations = $refIndexObj->getRelations($table, $row);
665 $relations = $this->fixFileIDsInRelations($relations);
666 // Data:
667 $this->dat['records'][$table . ':' . $row['uid']] = array();
668 $this->dat['records'][$table . ':' . $row['uid']]['data'] = $row;
669 $this->dat['records'][$table . ':' . $row['uid']]['rels'] = $relations;
670 $this->errorLog = array_merge($this->errorLog, $refIndexObj->errorLog);
671 // Merge error logs.
672 // Add information about the relations in the record in the header:
673 $this->dat['header']['records'][$table][$row['uid']]['rels'] = $this->flatDBrels($this->dat['records'][$table . ':' . $row['uid']]['rels']);
674 // Add information about the softrefs to header:
675 $this->dat['header']['records'][$table][$row['uid']]['softrefs'] = $this->flatSoftRefs($this->dat['records'][$table . ':' . $row['uid']]['rels']);
676 } else {
677 $this->error('Record ' . $table . ':' . $row['uid'] . ' was larger than maxRecordSize (' . GeneralUtility::formatSize($this->maxRecordSize) . ')');
678 }
679 } else {
680 $this->error('Record ' . $table . ':' . $row['uid'] . ' already added.');
681 }
682 } else {
683 $this->error('Record ' . $table . ':' . $row['uid'] . ' was outside your DB mounts!');
684 }
685 }
686 }
687
688 /**
689 * This changes the file reference ID from a hash based on the absolute file path
690 * (coming from ReferenceIndex) to a hash based on the relative file path.
691 *
692 * @param array $relations
693 * @return array
694 */
695 protected function fixFileIDsInRelations(array $relations) {
696 foreach ($relations as $field => $relation) {
697 if (isset($relation['type']) && $relation['type'] === 'file') {
698 foreach ($relation['newValueFiles'] as $key => $fileRelationData) {
699 $absoluteFilePath = $fileRelationData['ID_absFile'];
700 if (GeneralUtility::isFirstPartOfStr($absoluteFilePath, PATH_site)) {
701 $relatedFilePath = \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($absoluteFilePath);
702 $relations[$field]['newValueFiles'][$key]['ID'] = md5($relatedFilePath);
703 }
704 }
705 }
706 if ($relation['type'] == 'flex') {
707 if (is_array($relation['flexFormRels']['file'])) {
708 foreach ($relation['flexFormRels']['file'] as $key => $subList) {
709 foreach ($subList as $subKey => $fileRelationData) {
710 $absoluteFilePath = $fileRelationData['ID_absFile'];
711 if (GeneralUtility::isFirstPartOfStr($absoluteFilePath, PATH_site)) {
712 $relatedFilePath = \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($absoluteFilePath);
713 $relations[$field]['flexFormRels']['file'][$key][$subKey]['ID'] = md5($relatedFilePath);
714 }
715 }
716 }
717 }
718 }
719 }
720 return $relations;
721 }
722
723
724
725 /**
726 * This analyses the existing added records, finds all database relations to records and adds these records to the export file.
727 * This function can be called repeatedly until it returns an empty array. In principle it should not allow to infinite recursivity, but you better set a limit...
728 * Call this BEFORE the ext_addFilesFromRelations (so files from added relations are also included of course)
729 *
730 * @param integer $relationLevel Recursion level
731 * @return array overview of relations found and added: Keys [table]:[uid], values array with table and id
732 * @see export_addFilesFromRelations()
733 * @todo Define visibility
734 */
735 public function export_addDBRelations($relationLevel = 0) {
736 // Initialize:
737 $addR = array();
738 // Traverse all "rels" registered for "records"
739 if (is_array($this->dat['records'])) {
740 foreach ($this->dat['records'] as $k => $value) {
741 if (is_array($this->dat['records'][$k])) {
742 foreach ($this->dat['records'][$k]['rels'] as $fieldname => $vR) {
743 // For all DB types of relations:
744 if ($vR['type'] == 'db') {
745 foreach ($vR['itemArray'] as $fI) {
746 $this->export_addDBRelations_registerRelation($fI, $addR);
747 }
748 }
749 // For all flex/db types of relations:
750 if ($vR['type'] == 'flex') {
751 // DB relations in flex form fields:
752 if (is_array($vR['flexFormRels']['db'])) {
753 foreach ($vR['flexFormRels']['db'] as $subList) {
754 foreach ($subList as $fI) {
755 $this->export_addDBRelations_registerRelation($fI, $addR);
756 }
757 }
758 }
759 // DB oriented soft references in flex form fields:
760 if (is_array($vR['flexFormRels']['softrefs'])) {
761 foreach ($vR['flexFormRels']['softrefs'] as $subList) {
762 foreach ($subList['keys'] as $spKey => $elements) {
763 foreach ($elements as $el) {
764 if ($el['subst']['type'] === 'db' && $this->includeSoftref($el['subst']['tokenID'])) {
765 list($tempTable, $tempUid) = explode(':', $el['subst']['recordRef']);
766 $fI = array(
767 'table' => $tempTable,
768 'id' => $tempUid
769 );
770 $this->export_addDBRelations_registerRelation($fI, $addR, $el['subst']['tokenID']);
771 }
772 }
773 }
774 }
775 }
776 }
777 // In any case, if there are soft refs:
778 if (is_array($vR['softrefs']['keys'])) {
779 foreach ($vR['softrefs']['keys'] as $spKey => $elements) {
780 foreach ($elements as $el) {
781 if ($el['subst']['type'] === 'db' && $this->includeSoftref($el['subst']['tokenID'])) {
782 list($tempTable, $tempUid) = explode(':', $el['subst']['recordRef']);
783 $fI = array(
784 'table' => $tempTable,
785 'id' => $tempUid
786 );
787 $this->export_addDBRelations_registerRelation($fI, $addR, $el['subst']['tokenID']);
788 }
789 }
790 }
791 }
792 }
793 }
794 }
795 } else {
796 $this->error('There were no records available.');
797 }
798 // Now, if there were new records to add, do so:
799 if (count($addR)) {
800 foreach ($addR as $fI) {
801 // Get and set record:
802 $row = BackendUtility::getRecord($fI['table'], $fI['id']);
803 if (is_array($row)) {
804 $this->export_addRecord($fI['table'], $row, $relationLevel + 1);
805 }
806 // Set status message
807 // Relation pointers always larger than zero except certain "select" types with negative values pointing to uids - but that is not supported here.
808 if ($fI['id'] > 0) {
809 $rId = $fI['table'] . ':' . $fI['id'];
810 if (!isset($this->dat['records'][$rId])) {
811 $this->dat['records'][$rId] = 'NOT_FOUND';
812 $this->error('Relation record ' . $rId . ' was not found!');
813 }
814 }
815 }
816 }
817 // Return overview of relations found and added
818 return $addR;
819 }
820
821 /**
822 * Helper function for export_addDBRelations()
823 *
824 * @param array $fI Array with table/id keys to add
825 * @param array $addR Add array, passed by reference to be modified
826 * @param string $tokenID Softref Token ID, if applicable.
827 * @return void
828 * @see export_addDBRelations()
829 * @todo Define visibility
830 */
831 public function export_addDBRelations_registerRelation($fI, &$addR, $tokenID = '') {
832 $rId = $fI['table'] . ':' . $fI['id'];
833 if (isset($GLOBALS['TCA'][$fI['table']]) && !$this->isTableStatic($fI['table']) && !$this->isExcluded($fI['table'], $fI['id']) && (!$tokenID || $this->includeSoftref($tokenID)) && $this->inclRelation($fI['table'])) {
834 if (!isset($this->dat['records'][$rId])) {
835 // Set this record to be included since it is not already.
836 $addR[$rId] = $fI;
837 }
838 }
839 }
840
841 /**
842 * This adds all files in relations.
843 * Call this method AFTER adding all records including relations.
844 *
845 * @return void
846 * @see export_addDBRelations()
847 * @todo Define visibility
848 */
849 public function export_addFilesFromRelations() {
850 // Traverse all "rels" registered for "records"
851 if (is_array($this->dat['records'])) {
852 foreach ($this->dat['records'] as $k => $value) {
853 if (isset($this->dat['records'][$k]['rels']) && is_array($this->dat['records'][$k]['rels'])) {
854 foreach ($this->dat['records'][$k]['rels'] as $fieldname => $vR) {
855 // For all file type relations:
856 if ($vR['type'] == 'file') {
857 foreach ($vR['newValueFiles'] as $key => $fI) {
858 $this->export_addFile($fI, $k, $fieldname);
859 // Remove the absolute reference to the file so it doesn't expose absolute paths from source server:
860 unset($this->dat['records'][$k]['rels'][$fieldname]['newValueFiles'][$key]['ID_absFile']);
861 }
862 }
863 // For all flex type relations:
864 if ($vR['type'] == 'flex') {
865 if (is_array($vR['flexFormRels']['file'])) {
866 foreach ($vR['flexFormRels']['file'] as $key => $subList) {
867 foreach ($subList as $subKey => $fI) {
868 $this->export_addFile($fI, $k, $fieldname);
869 // Remove the absolute reference to the file so it doesn't expose absolute paths from source server:
870 unset($this->dat['records'][$k]['rels'][$fieldname]['flexFormRels']['file'][$key][$subKey]['ID_absFile']);
871 }
872 }
873 }
874 // DB oriented soft references in flex form fields:
875 if (is_array($vR['flexFormRels']['softrefs'])) {
876 foreach ($vR['flexFormRels']['softrefs'] as $key => $subList) {
877 foreach ($subList['keys'] as $spKey => $elements) {
878 foreach ($elements as $subKey => $el) {
879 if ($el['subst']['type'] === 'file' && $this->includeSoftref($el['subst']['tokenID'])) {
880 // Create abs path and ID for file:
881 $ID_absFile = GeneralUtility::getFileAbsFileName(PATH_site . $el['subst']['relFileName']);
882 $ID = md5($el['subst']['relFileName']);
883 if ($ID_absFile) {
884 if (!$this->dat['files'][$ID]) {
885 $fI = array(
886 'filename' => PathUtility::basename($ID_absFile),
887 'ID_absFile' => $ID_absFile,
888 'ID' => $ID,
889 'relFileName' => $el['subst']['relFileName']
890 );
891 $this->export_addFile($fI, '_SOFTREF_');
892 }
893 $this->dat['records'][$k]['rels'][$fieldname]['flexFormRels']['softrefs'][$key]['keys'][$spKey][$subKey]['file_ID'] = $ID;
894 }
895 }
896 }
897 }
898 }
899 }
900 }
901 // In any case, if there are soft refs:
902 if (is_array($vR['softrefs']['keys'])) {
903 foreach ($vR['softrefs']['keys'] as $spKey => $elements) {
904 foreach ($elements as $subKey => $el) {
905 if ($el['subst']['type'] === 'file' && $this->includeSoftref($el['subst']['tokenID'])) {
906 // Create abs path and ID for file:
907 $ID_absFile = GeneralUtility::getFileAbsFileName(PATH_site . $el['subst']['relFileName']);
908 $ID = md5($el['subst']['relFileName']);
909 if ($ID_absFile) {
910 if (!$this->dat['files'][$ID]) {
911 $fI = array(
912 'filename' => PathUtility::basename($ID_absFile),
913 'ID_absFile' => $ID_absFile,
914 'ID' => $ID,
915 'relFileName' => $el['subst']['relFileName']
916 );
917 $this->export_addFile($fI, '_SOFTREF_');
918 }
919 $this->dat['records'][$k]['rels'][$fieldname]['softrefs']['keys'][$spKey][$subKey]['file_ID'] = $ID;
920 }
921 }
922 }
923 }
924 }
925 }
926 }
927 }
928 } else {
929 $this->error('There were no records available.');
930 }
931 }
932
933 /**
934 * This adds all files from sys_file records
935 *
936 * @return void
937 */
938 public function export_addFilesFromSysFilesRecords() {
939 if (!isset($this->dat['header']['records']['sys_file']) || !is_array($this->dat['header']['records']['sys_file'])) {
940 return;
941 }
942 foreach (array_keys($this->dat['header']['records']['sys_file']) as $sysFileUid) {
943 $recordData = $this->dat['records']['sys_file:' . $sysFileUid]['data'];
944 $file = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->createFileObject($recordData);
945 $this->export_addSysFile($file);
946 }
947 }
948
949 /**
950 * Adds a files content from a sys file record to the export memory
951 *
952 * @param \TYPO3\CMS\Core\Resource\File $file
953 * @return void
954 */
955 public function export_addSysFile(\TYPO3\CMS\Core\Resource\File $file) {
956 if ($file->getProperty('size') >= $this->maxFileSize) {
957 $this->error('File ' . $file->getPublicUrl() . ' was larger (' . GeneralUtility::formatSize($file->getProperty('size')) . ') than the maxFileSize (' . GeneralUtility::formatSize($this->maxFileSize) . ')! Skipping.');
958 return;
959 }
960 try {
961 $fileContent = $file->getContents();
962 } catch (\TYPO3\CMS\Core\Resource\Exception\InsufficientFileAccessPermissionsException $e) {
963 $this->error('File ' . $file->getPublicUrl() . ': ' . $e->getMessage());
964 return;
965 } catch (\TYPO3\CMS\Core\Resource\Exception\IllegalFileExtensionException $e) {
966 $this->error('File ' . $file->getPublicUrl() . ': ' . $e->getMessage());
967 return;
968 }
969 $fileUid = $file->getUid();
970 $fileInfo = $file->getStorage()->getFileInfo($file);
971 // we sadly have to cast it to string here, because the size property is also returning a string
972 $fileSize = (string)$fileInfo['size'];
973 if ($fileSize !== $file->getProperty('size')) {
974 $this->error('File size of ' . $file->getCombinedIdentifier() . ' is not up-to-date in index! File added with current size.');
975 $this->dat['records']['sys_file:' . $fileUid]['data']['size'] = $fileSize;
976 }
977 $fileSha1 = $file->getStorage()->hashFile($file, 'sha1');
978 if ($fileSha1 !== $file->getProperty('sha1')) {
979 $this->error('File sha1 hash of ' . $file->getCombinedIdentifier() . ' is not up-to-date in index! File added on current sha1.');
980 $this->dat['records']['sys_file:' . $fileUid]['data']['sha1'] = $fileSha1;
981 }
982
983 $fileRec = array();
984 $fileRec['filesize'] = $fileSize;
985 $fileRec['filename'] = $file->getProperty('name');
986 $fileRec['filemtime'] = $file->getProperty('modification_date');
987
988 // build unique id based on the storage and the file identifier
989 $fileId = md5($file->getStorage()->getUid() . ':' . $file->getProperty('identifier_hash'));
990
991 // Setting this data in the header
992 $this->dat['header']['files_fal'][$fileId] = $fileRec;
993
994 // ... and finally add the heavy stuff:
995 $fileRec['content'] = $fileContent;
996 $fileRec['content_sha1'] = $fileSha1;
997
998 $this->dat['files_fal'][$fileId] = $fileRec;
999 }
1000
1001
1002 /**
1003 * Adds a files content to the export memory
1004 *
1005 * @param array $fI File information with three keys: "filename" = filename without path, "ID_absFile" = absolute filepath to the file (including the filename), "ID" = md5 hash of "ID_absFile". "relFileName" is optional for files attached to records, but mandatory for soft referenced files (since the relFileName determines where such a file should be stored!)
1006 * @param string $recordRef If the file is related to a record, this is the id on the form [table]:[id]. Information purposes only.
1007 * @param string $fieldname If the file is related to a record, this is the field name it was related to. Information purposes only.
1008 * @return void
1009 * @todo Define visibility
1010 */
1011 public function export_addFile($fI, $recordRef = '', $fieldname = '') {
1012 if (@is_file($fI['ID_absFile'])) {
1013 if (filesize($fI['ID_absFile']) < $this->maxFileSize) {
1014 $fileInfo = stat($fI['ID_absFile']);
1015 $fileRec = array();
1016 $fileRec['filesize'] = $fileInfo['size'];
1017 $fileRec['filename'] = PathUtility::basename($fI['ID_absFile']);
1018 $fileRec['filemtime'] = $fileInfo['mtime'];
1019 //for internal type file_reference
1020 $fileRec['relFileRef'] = \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($fI['ID_absFile']);
1021 if ($recordRef) {
1022 $fileRec['record_ref'] = $recordRef . '/' . $fieldname;
1023 }
1024 if ($fI['relFileName']) {
1025 $fileRec['relFileName'] = $fI['relFileName'];
1026 }
1027 // Setting this data in the header
1028 $this->dat['header']['files'][$fI['ID']] = $fileRec;
1029 // ... and for the recordlisting, why not let us know WHICH relations there was...
1030 if ($recordRef && $recordRef !== '_SOFTREF_') {
1031 $refParts = explode(':', $recordRef, 2);
1032 if (!is_array($this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'])) {
1033 $this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'] = array();
1034 }
1035 $this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'][] = $fI['ID'];
1036 }
1037 // ... and finally add the heavy stuff:
1038 $fileRec['content'] = GeneralUtility::getUrl($fI['ID_absFile']);
1039 $fileRec['content_md5'] = md5($fileRec['content']);
1040 $this->dat['files'][$fI['ID']] = $fileRec;
1041 // For soft references, do further processing:
1042 if ($recordRef === '_SOFTREF_') {
1043 // RTE files?
1044 if ($RTEoriginal = $this->getRTEoriginalFilename(PathUtility::basename($fI['ID_absFile']))) {
1045 $RTEoriginal_absPath = PathUtility::dirname($fI['ID_absFile']) . '/' . $RTEoriginal;
1046 if (@is_file($RTEoriginal_absPath)) {
1047 $RTEoriginal_ID = md5($RTEoriginal_absPath);
1048 $fileInfo = stat($RTEoriginal_absPath);
1049 $fileRec = array();
1050 $fileRec['filesize'] = $fileInfo['size'];
1051 $fileRec['filename'] = PathUtility::basename($RTEoriginal_absPath);
1052 $fileRec['filemtime'] = $fileInfo['mtime'];
1053 $fileRec['record_ref'] = '_RTE_COPY_ID:' . $fI['ID'];
1054 $this->dat['header']['files'][$fI['ID']]['RTE_ORIG_ID'] = $RTEoriginal_ID;
1055 // Setting this data in the header
1056 $this->dat['header']['files'][$RTEoriginal_ID] = $fileRec;
1057 // ... and finally add the heavy stuff:
1058 $fileRec['content'] = GeneralUtility::getUrl($RTEoriginal_absPath);
1059 $fileRec['content_md5'] = md5($fileRec['content']);
1060 $this->dat['files'][$RTEoriginal_ID] = $fileRec;
1061 } else {
1062 $this->error('RTE original file "' . \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($RTEoriginal_absPath) . '" was not found!');
1063 }
1064 }
1065 // Files with external media?
1066 // This is only done with files grabbed by a softreference parser since it is deemed improbable that hard-referenced files should undergo this treatment.
1067 $html_fI = pathinfo(PathUtility::basename($fI['ID_absFile']));
1068 if ($this->includeExtFileResources && GeneralUtility::inList($this->extFileResourceExtensions, strtolower($html_fI['extension']))) {
1069 $uniquePrefix = '###' . md5($GLOBALS['EXEC_TIME']) . '###';
1070 if (strtolower($html_fI['extension']) === 'css') {
1071 $prefixedMedias = explode($uniquePrefix, preg_replace('/(url[[:space:]]*\\([[:space:]]*["\']?)([^"\')]*)(["\']?[[:space:]]*\\))/i', '\\1' . $uniquePrefix . '\\2' . $uniquePrefix . '\\3', $fileRec['content']));
1072 } else {
1073 // html, htm:
1074 $htmlParser = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\Html\\HtmlParser');
1075 $prefixedMedias = explode($uniquePrefix, $htmlParser->prefixResourcePath($uniquePrefix, $fileRec['content'], array(), $uniquePrefix));
1076 }
1077 $htmlResourceCaptured = FALSE;
1078 foreach ($prefixedMedias as $k => $v) {
1079 if ($k % 2) {
1080 $EXTres_absPath = GeneralUtility::resolveBackPath(PathUtility::dirname($fI['ID_absFile']) . '/' . $v);
1081 $EXTres_absPath = GeneralUtility::getFileAbsFileName($EXTres_absPath);
1082 if ($EXTres_absPath && GeneralUtility::isFirstPartOfStr($EXTres_absPath, PATH_site . $this->fileadminFolderName . '/') && @is_file($EXTres_absPath)) {
1083 $htmlResourceCaptured = TRUE;
1084 $EXTres_ID = md5($EXTres_absPath);
1085 $this->dat['header']['files'][$fI['ID']]['EXT_RES_ID'][] = $EXTres_ID;
1086 $prefixedMedias[$k] = '{EXT_RES_ID:' . $EXTres_ID . '}';
1087 // Add file to memory if it is not set already:
1088 if (!isset($this->dat['header']['files'][$EXTres_ID])) {
1089 $fileInfo = stat($EXTres_absPath);
1090 $fileRec = array();
1091 $fileRec['filesize'] = $fileInfo['size'];
1092 $fileRec['filename'] = PathUtility::basename($EXTres_absPath);
1093 $fileRec['filemtime'] = $fileInfo['mtime'];
1094 $fileRec['record_ref'] = '_EXT_PARENT_:' . $fI['ID'];
1095 // Media relative to the HTML file.
1096 $fileRec['parentRelFileName'] = $v;
1097 // Setting this data in the header
1098 $this->dat['header']['files'][$EXTres_ID] = $fileRec;
1099 // ... and finally add the heavy stuff:
1100 $fileRec['content'] = GeneralUtility::getUrl($EXTres_absPath);
1101 $fileRec['content_md5'] = md5($fileRec['content']);
1102 $this->dat['files'][$EXTres_ID] = $fileRec;
1103 }
1104 }
1105 }
1106 }
1107 if ($htmlResourceCaptured) {
1108 $this->dat['files'][$fI['ID']]['tokenizedContent'] = implode('', $prefixedMedias);
1109 }
1110 }
1111 }
1112 } else {
1113 $this->error($fI['ID_absFile'] . ' was larger (' . GeneralUtility::formatSize(filesize($fI['ID_absFile'])) . ') than the maxFileSize (' . GeneralUtility::formatSize($this->maxFileSize) . ')! Skipping.');
1114 }
1115 } else {
1116 $this->error($fI['ID_absFile'] . ' was not a file! Skipping.');
1117 }
1118 }
1119
1120 /**
1121 * DB relations flattend to 1-dim array.
1122 * The list will be unique, no table/uid combination will appear twice.
1123 *
1124 * @param array $dbrels 2-dim Array of database relations organized by table key
1125 * @return array 1-dim array where entries are table:uid and keys are array with table/id
1126 * @todo Define visibility
1127 */
1128 public function flatDBrels($dbrels) {
1129 $list = array();
1130 foreach ($dbrels as $dat) {
1131 if ($dat['type'] == 'db') {
1132 foreach ($dat['itemArray'] as $i) {
1133 $list[$i['table'] . ':' . $i['id']] = $i;
1134 }
1135 }
1136 if ($dat['type'] == 'flex' && is_array($dat['flexFormRels']['db'])) {
1137 foreach ($dat['flexFormRels']['db'] as $subList) {
1138 foreach ($subList as $i) {
1139 $list[$i['table'] . ':' . $i['id']] = $i;
1140 }
1141 }
1142 }
1143 }
1144 return $list;
1145 }
1146
1147 /**
1148 * Soft References flattend to 1-dim array.
1149 *
1150 * @param array $dbrels 2-dim Array of database relations organized by table key
1151 * @return array 1-dim array where entries are arrays with properties of the soft link found and keys are a unique combination of field, spKey, structure path if applicable and token ID
1152 * @todo Define visibility
1153 */
1154 public function flatSoftRefs($dbrels) {
1155 $list = array();
1156 foreach ($dbrels as $field => $dat) {
1157 if (is_array($dat['softrefs']['keys'])) {
1158 foreach ($dat['softrefs']['keys'] as $spKey => $elements) {
1159 if (is_array($elements)) {
1160 foreach ($elements as $subKey => $el) {
1161 $lKey = $field . ':' . $spKey . ':' . $subKey;
1162 $list[$lKey] = array_merge(array('field' => $field, 'spKey' => $spKey), $el);
1163 // Add file_ID key to header - slightly "risky" way of doing this because if the calculation changes for the same value in $this->records[...] this will not work anymore!
1164 if ($el['subst'] && $el['subst']['relFileName']) {
1165 $list[$lKey]['file_ID'] = md5(PATH_site . $el['subst']['relFileName']);
1166 }
1167 }
1168 }
1169 }
1170 }
1171 if ($dat['type'] == 'flex' && is_array($dat['flexFormRels']['softrefs'])) {
1172 foreach ($dat['flexFormRels']['softrefs'] as $structurePath => $subSoftrefs) {
1173 if (is_array($subSoftrefs['keys'])) {
1174 foreach ($subSoftrefs['keys'] as $spKey => $elements) {
1175 foreach ($elements as $subKey => $el) {
1176 $lKey = $field . ':' . $structurePath . ':' . $spKey . ':' . $subKey;
1177 $list[$lKey] = array_merge(array('field' => $field, 'spKey' => $spKey, 'structurePath' => $structurePath), $el);
1178 // Add file_ID key to header - slightly "risky" way of doing this because if the calculation changes for the same value in $this->records[...] this will not work anymore!
1179 if ($el['subst'] && $el['subst']['relFileName']) {
1180 $list[$lKey]['file_ID'] = md5(PATH_site . $el['subst']['relFileName']);
1181 }
1182 }
1183 }
1184 }
1185 }
1186 }
1187 }
1188 return $list;
1189 }
1190
1191 /**
1192 * If include fields for a specific record type are set, the data
1193 * are filtered out with fields are not included in the fields.
1194 *
1195 * @param string $table The record type to be filtered
1196 * @param array $row The data to be filtered
1197 * @return array The filtered record row
1198 */
1199 protected function filterRecordFields($table, array $row) {
1200 if (isset($this->recordTypesIncludeFields[$table])) {
1201 $includeFields = array_unique(array_merge(
1202 $this->recordTypesIncludeFields[$table],
1203 $this->defaultRecordIncludeFields
1204 ));
1205 $newRow = array();
1206 foreach ($row as $key => $value) {
1207 if (in_array($key, $includeFields)) {
1208 $newRow[$key] = $value;
1209 }
1210 }
1211 } else {
1212 $newRow = $row;
1213 }
1214 return $newRow;
1215 }
1216
1217
1218 /**************************
1219 * File Output
1220 *************************/
1221
1222 /**
1223 * This compiles and returns the data content for an exported file
1224 *
1225 * @param string $type Type of output; "xml" gives xml, otherwise serialized array, possibly compressed.
1226 * @return string The output file stream
1227 * @todo Define visibility
1228 */
1229 public function compileMemoryToFileContent($type = '') {
1230 if ($type == 'xml') {
1231 $out = $this->createXML();
1232 } else {
1233 $compress = $this->doOutputCompress();
1234 $out = '';
1235 // adding header:
1236 $out .= $this->addFilePart(serialize($this->dat['header']), $compress);
1237 // adding records:
1238 $out .= $this->addFilePart(serialize($this->dat['records']), $compress);
1239 // adding files:
1240 $out .= $this->addFilePart(serialize($this->dat['files']), $compress);
1241 // adding files_fal:
1242 $out .= $this->addFilePart(serialize($this->dat['files_fal']), $compress);
1243 }
1244 return $out;
1245 }
1246
1247 /**
1248 * Creates XML string from input array
1249 *
1250 * @return string XML content
1251 * @todo Define visibility
1252 */
1253 public function createXML() {
1254 // Options:
1255 $options = array(
1256 'alt_options' => array(
1257 '/header' => array(
1258 'disableTypeAttrib' => TRUE,
1259 'clearStackPath' => TRUE,
1260 'parentTagMap' => array(
1261 'files' => 'file',
1262 'files_fal' => 'file',
1263 'records' => 'table',
1264 'table' => 'rec',
1265 'rec:rels' => 'relations',
1266 'relations' => 'element',
1267 'filerefs' => 'file',
1268 'pid_lookup' => 'page_contents',
1269 'header:relStaticTables' => 'static_tables',
1270 'static_tables' => 'tablename',
1271 'excludeMap' => 'item',
1272 'softrefCfg' => 'softrefExportMode',
1273 'extensionDependencies' => 'extkey',
1274 'softrefs' => 'softref_element'
1275 ),
1276 'alt_options' => array(
1277 '/pagetree' => array(
1278 'disableTypeAttrib' => TRUE,
1279 'useIndexTagForNum' => 'node',
1280 'parentTagMap' => array(
1281 'node:subrow' => 'node'
1282 )
1283 ),
1284 '/pid_lookup/page_contents' => array(
1285 'disableTypeAttrib' => TRUE,
1286 'parentTagMap' => array(
1287 'page_contents' => 'table'
1288 ),
1289 'grandParentTagMap' => array(
1290 'page_contents/table' => 'item'
1291 )
1292 )
1293 )
1294 ),
1295 '/records' => array(
1296 'disableTypeAttrib' => TRUE,
1297 'parentTagMap' => array(
1298 'records' => 'tablerow',
1299 'tablerow:data' => 'fieldlist',
1300 'tablerow:rels' => 'related',
1301 'related' => 'field',
1302 'field:itemArray' => 'relations',
1303 'field:newValueFiles' => 'filerefs',
1304 'field:flexFormRels' => 'flexform',
1305 'relations' => 'element',
1306 'filerefs' => 'file',
1307 'flexform:db' => 'db_relations',
1308 'flexform:file' => 'file_relations',
1309 'flexform:softrefs' => 'softref_relations',
1310 'softref_relations' => 'structurePath',
1311 'db_relations' => 'path',
1312 'file_relations' => 'path',
1313 'path' => 'element',
1314 'keys' => 'softref_key',
1315 'softref_key' => 'softref_element'
1316 ),
1317 'alt_options' => array(
1318 '/records/tablerow/fieldlist' => array(
1319 'useIndexTagForAssoc' => 'field'
1320 )
1321 )
1322 ),
1323 '/files' => array(
1324 'disableTypeAttrib' => TRUE,
1325 'parentTagMap' => array(
1326 'files' => 'file'
1327 )
1328 ),
1329 '/files_fal' => array(
1330 'disableTypeAttrib' => TRUE,
1331 'parentTagMap' => array(
1332 'files_fal' => 'file'
1333 )
1334 )
1335 )
1336 );
1337 // Creating XML file from $outputArray:
1338 $charset = $this->dat['header']['charset'] ?: 'utf-8';
1339 $XML = '<?xml version="1.0" encoding="' . $charset . '" standalone="yes" ?>' . LF;
1340 $XML .= GeneralUtility::array2xml($this->dat, '', 0, 'T3RecordDocument', 0, $options);
1341 return $XML;
1342 }
1343
1344 /**
1345 * Returns TRUE if the output should be compressed.
1346 *
1347 * @return boolean TRUE if compression is possible AND requested.
1348 * @todo Define visibility
1349 */
1350 public function doOutputCompress() {
1351 return $this->compress && !$this->dontCompress;
1352 }
1353
1354 /**
1355 * Returns a content part for a filename being build.
1356 *
1357 * @param array $data Data to store in part
1358 * @param boolean $compress Compress file?
1359 * @return string Content stream.
1360 * @todo Define visibility
1361 */
1362 public function addFilePart($data, $compress = FALSE) {
1363 if ($compress) {
1364 $data = gzcompress($data);
1365 }
1366 return md5($data) . ':' . ($compress ? '1' : '0') . ':' . str_pad(strlen($data), 10, '0', STR_PAD_LEFT) . ':' . $data . ':';
1367 }
1368
1369 /***********************
1370 * Import
1371 ***********************/
1372
1373 /**
1374 * Initialize all settings for the import
1375 *
1376 * @return void
1377 */
1378 protected function initializeImport() {
1379 // Set this flag to indicate that an import is being/has been done.
1380 $this->doesImport = 1;
1381 // Initialize:
1382 // These vars MUST last for the whole section not being cleared. They are used by the method setRelations() which are called at the end of the import session.
1383 $this->import_mapId = array();
1384 $this->import_newId = array();
1385 $this->import_newId_pids = array();
1386 // Temporary files stack initialized:
1387 $this->unlinkFiles = array();
1388 $this->alternativeFileName = array();
1389 $this->alternativeFilePath = array();
1390
1391 $this->initializeStorageObjects();
1392 }
1393
1394 /**
1395 * Initialize the all present storage objects
1396 *
1397 * @return void
1398 */
1399 protected function initializeStorageObjects() {
1400 /** @var $storageRepository \TYPO3\CMS\Core\Resource\StorageRepository */
1401 $storageRepository = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\Resource\\StorageRepository');
1402 $this->storageObjects = $storageRepository->findAll();
1403 }
1404
1405
1406 /**
1407 * Imports the internal data array to $pid.
1408 *
1409 * @param integer $pid Page ID in which to import the content
1410 * @return void
1411 */
1412 public function importData($pid) {
1413
1414 $this->initializeImport();
1415
1416 // Write sys_file_storages first
1417 $this->writeSysFileStorageRecords();
1418 // Write sys_file records and write the binary file data
1419 $this->writeSysFileRecords();
1420 // Write records, first pages, then the rest
1421 // Fields with "hard" relations to database, files and flexform fields are kept empty during this run
1422 $this->writeRecords_pages($pid);
1423 $this->writeRecords_records($pid);
1424 // Finally all the file and DB record references must be fixed. This is done after all records have supposedly been written to database:
1425 // $this->import_mapId will indicate two things: 1) that a record WAS written to db and 2) that it has got a new id-number.
1426 $this->setRelations();
1427 // And when all DB relations are in place, we can fix file and DB relations in flexform fields (since data structures often depends on relations to a DS record):
1428 $this->setFlexFormRelations();
1429 // Unlink temporary files:
1430 $this->unlinkTempFiles();
1431 // Finally, traverse all records and process softreferences with substitution attributes.
1432 $this->processSoftReferences();
1433 // After all migrate records using sys_file_reference now
1434 if ($this->legacyImport) {
1435 $this->migrateLegacyImportRecords();
1436 }
1437 }
1438
1439 /**
1440 * Imports the sys_file_storage records from internal data array.
1441 *
1442 * @return void
1443 */
1444 protected function writeSysFileStorageRecords() {
1445 if (!isset($this->dat['header']['records']['sys_file_storage'])) {
1446 return;
1447 }
1448 $sysFileStorageUidsToBeResetToDefaultStorage = array();
1449 foreach (array_keys($this->dat['header']['records']['sys_file_storage']) as $sysFileStorageUid) {
1450 $storageRecord = $this->dat['records']['sys_file_storage:' . $sysFileStorageUid]['data'];
1451 // continue with Local, writable and online storage only
1452 if ($storageRecord['driver'] === 'Local' && $storageRecord['is_writable'] && $storageRecord['is_online']) {
1453 $useThisStorageUidInsteadOfTheOneInImport = 0;
1454 /** @var $localStorage \TYPO3\CMS\Core\Resource\ResourceStorage */
1455 foreach ($this->storageObjects as $localStorage) {
1456 // check the available storage for Local, writable and online ones
1457 if ($localStorage->getDriverType() === 'Local' && $localStorage->isWritable() && $localStorage->isOnline()) {
1458 // check if there is already a identical storage present (same pathType and basePath)
1459 $storageRecordConfiguration = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->convertFlexFormDataToConfigurationArray($storageRecord['configuration']);
1460 $localStorageRecordConfiguration = $localStorage->getConfiguration();
1461 if ($storageRecordConfiguration['pathType'] == $localStorageRecordConfiguration['pathType'] && $storageRecordConfiguration['basePath'] == $localStorageRecordConfiguration['basePath']) {
1462 // same storage is already present
1463 $useThisStorageUidInsteadOfTheOneInImport = $localStorage->getUid();
1464 break;
1465 }
1466 }
1467 }
1468 if ($useThisStorageUidInsteadOfTheOneInImport > 0) {
1469 // same storage is already present; map the to be imported one to the present one
1470 $this->import_mapId['sys_file_storage'][$sysFileStorageUid] = $useThisStorageUidInsteadOfTheOneInImport;
1471 } else {
1472 // Local, writable and online storage. Is allowed to be used to later write files in.
1473 $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);
1474 }
1475 } else {
1476 // Storage with non Local drivers could be imported but must not be used to saves files in, because you
1477 // could not be sure, that this is supported. The default storage will be used in this case.
1478 // It could happen that non writable and non online storage will be created as dupes because you could not
1479 // check the detailed configuration options at this point
1480 $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);
1481 $sysFileStorageUidsToBeResetToDefaultStorage[] = $sysFileStorageUid;
1482 }
1483
1484 }
1485
1486 // Importing the added ones
1487 $tce = $this->getNewTCE();
1488 // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.
1489 $tce->reverseOrder = 1;
1490 $tce->isImporting = TRUE;
1491 $tce->start($this->import_data, array());
1492 $tce->process_datamap();
1493 $this->addToMapId($tce->substNEWwithIDs);
1494
1495 $defaultStorageUid = NULL;
1496 // get default storage
1497 $defaultStorage = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->getDefaultStorage();
1498 if ($defaultStorage !== NULL) {
1499 $defaultStorageUid = $defaultStorage->getUid();
1500 }
1501 foreach ($sysFileStorageUidsToBeResetToDefaultStorage as $sysFileStorageUidToBeResetToDefaultStorage) {
1502 $this->import_mapId['sys_file_storage'][$sysFileStorageUidToBeResetToDefaultStorage] = $defaultStorageUid;
1503 }
1504
1505 // unset the sys_file_storage records to prevent a import in writeRecords_records
1506 unset($this->dat['header']['records']['sys_file_storage']);
1507 }
1508
1509 /**
1510 * Imports the sys_file records and the binary files data from internal data array.
1511 *
1512 * @return void
1513 */
1514 protected function writeSysFileRecords() {
1515 if (!isset($this->dat['header']['records']['sys_file'])) {
1516 return;
1517 }
1518
1519 // fetch fresh storage records from database
1520 $storageRecords = $this->fetchStorageRecords();
1521
1522 $defaultStorage = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->getDefaultStorage();
1523
1524 $sanitizedFolderMappings = array();
1525
1526 foreach (array_keys($this->dat['header']['records']['sys_file']) as $sysFileUid) {
1527 $fileRecord = $this->dat['records']['sys_file:' . $sysFileUid]['data'];
1528
1529 // save file to disk
1530 $fileId = md5($fileRecord['storage'] . ':' . $fileRecord['identifier_hash']);
1531 $temporaryFile = $this->writeTemporaryFileFromData($fileId);
1532 if ($temporaryFile === NULL) {
1533 // error on writing the file. Error message was already added
1534 continue;
1535 }
1536
1537 $originalStorageUid = $fileRecord['storage'];
1538 $useStorageFromStorageRecords = FALSE;
1539
1540 // replace storage id, if a alternative one was registered
1541 if (isset($this->import_mapId['sys_file_storage'][$fileRecord['storage']])) {
1542 $fileRecord['storage'] = $this->import_mapId['sys_file_storage'][$fileRecord['storage']];
1543 $useStorageFromStorageRecords = TRUE;
1544 }
1545
1546 if (empty($fileRecord['storage'])) {
1547 // no storage for the file is defined, mostly because of a missing default storage.
1548 $this->error('Error: No storage for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $originalStorageUid . '"');
1549 continue;
1550 }
1551
1552 // using a storage from the local storage is only allowed, if the uid is present in the
1553 // mapping. Only in this case we could be sure, that it's a local, online and writable storage.
1554 if ($useStorageFromStorageRecords && isset($storageRecords[$fileRecord['storage']])) {
1555 /** @var $storage \TYPO3\CMS\Core\Resource\ResourceStorage */
1556 $storage = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->getStorageObject($fileRecord['storage'], $storageRecords[$fileRecord['storage']]);
1557 } elseif ($defaultStorage !== NULL) {
1558 $storage = $defaultStorage;
1559 } else {
1560 $this->error('Error: No storage available for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1561 continue;
1562 }
1563
1564 $newFile = NULL;
1565
1566 // check, if there is a identical file
1567 try {
1568 if ($storage->hasFile($fileRecord['identifier'])) {
1569 $file = $storage->getFile($fileRecord['identifier']);
1570 if ($file->getSha1() === $fileRecord['sha1']) {
1571 $newFile = $file;
1572 }
1573 }
1574 } catch (Exception $e) {}
1575
1576 if ($newFile === NULL) {
1577
1578 $folderName = PathUtility::dirname(ltrim($fileRecord['identifier'], '/'));
1579 if (in_array($folderName, $sanitizedFolderMappings)) {
1580 $folderName = $sanitizedFolderMappings[$folderName];
1581 }
1582 if (!$storage->hasFolder($folderName)) {
1583 try {
1584 $importFolder = $storage->createFolder($folderName);
1585 if ($importFolder->getIdentifier() !== $folderName && !in_array($folderName, $sanitizedFolderMappings)) {
1586 $sanitizedFolderMappings[$folderName] = $importFolder->getIdentifier();
1587 }
1588 } catch (Exception $e) {
1589 $this->error('Error: Folder could not be created for file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1590 continue;
1591 }
1592 } else {
1593 $importFolder = $storage->getFolder($folderName);
1594 }
1595
1596 try {
1597 /** @var $file \TYPO3\CMS\Core\Resource\File */
1598 $newFile = $storage->addFile($temporaryFile, $importFolder, $fileRecord['name']);
1599 } catch (Exception $e) {
1600 $this->error('Error: File could not be added to the storage: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1601 continue;
1602 }
1603
1604 if ($newFile->getSha1() !== $fileRecord['sha1']) {
1605 $this->error('Error: The hash of the written file is not identical to the import data! File could be corrupted! File: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1606 }
1607 }
1608
1609 // save the new uid in the import id map
1610 $this->import_mapId['sys_file'][$fileRecord['uid']] = $newFile->getUid();
1611 $this->fixUidLocalInSysFileReferenceRecords($fileRecord['uid'], $newFile->getUid());
1612
1613 }
1614
1615 // unset the sys_file records to prevent a import in writeRecords_records
1616 unset($this->dat['header']['records']['sys_file']);
1617 }
1618
1619 /**
1620 * Normally the importer works like the following:
1621 * Step 1: import the records with cleared field values of relation fields (see addSingle())
1622 * Step 2: update the records with the right relation ids (see setRelations())
1623 *
1624 * In step 2 the saving fields of type "relation to sys_file_reference" checks the related sys_file_reference
1625 * record (created in step 1) with the FileExtensionFilter for matching file extensions of the related file.
1626 * To make this work correct, the uid_local of sys_file_reference records has to be not empty AND has to
1627 * relate to the correct (imported) sys_file record uid!!!
1628 *
1629 * This is fixed here.
1630 *
1631 * @param int $oldFileUid
1632 * @param int $newFileUid
1633 * @return void
1634 */
1635 protected function fixUidLocalInSysFileReferenceRecords($oldFileUid, $newFileUid) {
1636 if (!isset($this->dat['header']['records']['sys_file_reference'])) {
1637 return;
1638 }
1639
1640 foreach (array_keys($this->dat['header']['records']['sys_file_reference']) as $sysFileReferenceUid) {
1641 $fileReferenceRecord = $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'];
1642 if ($fileReferenceRecord['uid_local'] == $oldFileUid) {
1643 $fileReferenceRecord['uid_local'] = $newFileUid;
1644 $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'] = $fileReferenceRecord;
1645 }
1646 }
1647 }
1648
1649 /**
1650 * Initializes the folder for legacy imports as subfolder of backend users default upload folder
1651 *
1652 * @return void
1653 */
1654 protected function initializeLegacyImportFolder() {
1655 /** @var \TYPO3\CMS\Core\Resource\Folder $folder */
1656 $folder = $GLOBALS['BE_USER']->getDefaultUploadFolder();
1657 if ($folder === FALSE) {
1658 $this->error('Error: the backend users default upload folder is missing! No files will be imported!');
1659 }
1660 if (!$folder->hasFolder($this->legacyImportTargetPath)) {
1661 try {
1662 $this->legacyImportFolder = $folder->createFolder($this->legacyImportTargetPath);
1663 } catch (\TYPO3\CMS\Core\Exception $e) {
1664 $this->error('Error: the import folder in the default upload folder could not be created! No files will be imported!');
1665 }
1666 } else {
1667 $this->legacyImportFolder = $folder->getSubFolder($this->legacyImportTargetPath);
1668 }
1669
1670 }
1671
1672 /**
1673 * Fetched fresh storage records from database because the new imported
1674 * ones are not in cached data of the StorageRepository
1675 *
1676 * @return bool|array
1677 */
1678 protected function fetchStorageRecords() {
1679 $whereClause = \TYPO3\CMS\Backend\Utility\BackendUtility::BEenableFields('sys_file_storage');
1680 $whereClause .= \TYPO3\CMS\Backend\Utility\BackendUtility::deleteClause('sys_file_storage');
1681
1682 $rows = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows(
1683 '*',
1684 'sys_file_storage',
1685 '1=1' . $whereClause,
1686 '',
1687 '',
1688 '',
1689 'uid'
1690 );
1691
1692 return $rows;
1693 }
1694
1695 /**
1696 * Writes the file from import array to temp dir and returns the filename of it.
1697 *
1698 * @param string $fileId
1699 * @param string $dataKey
1700 * @return string Absolute filename of the temporary filename of the file
1701 */
1702 protected function writeTemporaryFileFromData($fileId, $dataKey = 'files_fal') {
1703 $temporaryFilePath = NULL;
1704 if (is_array($this->dat[$dataKey][$fileId])) {
1705 $temporaryFilePathInternal = GeneralUtility::tempnam('import_temp_');
1706 GeneralUtility::writeFile($temporaryFilePathInternal, $this->dat[$dataKey][$fileId]['content']);
1707 clearstatcache();
1708 if (@is_file($temporaryFilePathInternal)) {
1709 $this->unlinkFiles[] = $temporaryFilePathInternal;
1710 if (filesize($temporaryFilePathInternal) == $this->dat[$dataKey][$fileId]['filesize']) {
1711 $temporaryFilePath = $temporaryFilePathInternal;
1712 } else {
1713 $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' had a size (' . filesize($temporaryFilePathInternal) . ') different from the original (' . $this->dat[$dataKey][$fileId]['filesize'] . ')', 1);
1714 }
1715 } else {
1716 $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' was not written as it should have been!', 1);
1717 }
1718 } else {
1719 $this->error('Error: No file found for ID ' . $fileId, 1);
1720 }
1721 return $temporaryFilePath;
1722 }
1723
1724 /**
1725 * Writing pagetree/pages to database:
1726 *
1727 * @param integer $pid PID in which to import. If the operation is an update operation, the root of the page tree inside will be moved to this PID unless it is the same as the root page from the import
1728 * @return void
1729 * @see writeRecords_records()
1730 * @todo Define visibility
1731 */
1732 public function writeRecords_pages($pid) {
1733 // First, write page structure if any:
1734 if (is_array($this->dat['header']['records']['pages'])) {
1735 // $pageRecords is a copy of the pages array in the imported file. Records here are unset one by one when the addSingle function is called.
1736 $pageRecords = $this->dat['header']['records']['pages'];
1737 $this->import_data = array();
1738 // First add page tree if any
1739 if (is_array($this->dat['header']['pagetree'])) {
1740 $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);
1741 foreach ($pagesFromTree as $uid) {
1742 $thisRec = $this->dat['header']['records']['pages'][$uid];
1743 // PID: Set the main $pid, unless a NEW-id is found
1744 $setPid = isset($this->import_newId_pids[$thisRec['pid']]) ? $this->import_newId_pids[$thisRec['pid']] : $pid;
1745 $this->addSingle('pages', $uid, $setPid);
1746 unset($pageRecords[$uid]);
1747 }
1748 }
1749 // Then add all remaining pages not in tree on root level:
1750 if (count($pageRecords)) {
1751 $remainingPageUids = array_keys($pageRecords);
1752 foreach ($remainingPageUids as $pUid) {
1753 $this->addSingle('pages', $pUid, $pid);
1754 }
1755 }
1756 // Now write to database:
1757 $tce = $this->getNewTCE();
1758 $tce->isImporting = TRUE;
1759 $this->callHook('before_writeRecordsPages', array(
1760 'tce' => &$tce,
1761 'data' => &$this->import_data
1762 ));
1763 $tce->suggestedInsertUids = $this->suggestedInsertUids;
1764 $tce->start($this->import_data, array());
1765 $tce->process_datamap();
1766 $this->callHook('after_writeRecordsPages', array(
1767 'tce' => &$tce
1768 ));
1769 // post-processing: Registering new ids (end all tcemain sessions with this)
1770 $this->addToMapId($tce->substNEWwithIDs);
1771 // In case of an update, order pages from the page tree correctly:
1772 if ($this->update && is_array($this->dat['header']['pagetree'])) {
1773 $this->writeRecords_pages_order($pid);
1774 }
1775 }
1776 }
1777
1778 /**
1779 * Organize all updated pages in page tree so they are related like in the import file
1780 * Only used for updates and when $this->dat['header']['pagetree'] is an array.
1781 *
1782 * @param integer $pid Page id in which to import
1783 * @return void
1784 * @access private
1785 * @see writeRecords_pages(), writeRecords_records_order()
1786 * @todo Define visibility
1787 */
1788 public function writeRecords_pages_order($pid) {
1789 $cmd_data = array();
1790 // Get uid-pid relations and traverse them in order to map to possible new IDs
1791 $pidsFromTree = $this->flatInversePageTree_pid($this->dat['header']['pagetree']);
1792 foreach ($pidsFromTree as $origPid => $newPid) {
1793 if ($newPid >= 0 && $this->dontIgnorePid('pages', $origPid)) {
1794 // If the page had a new id (because it was created) use that instead!
1795 if (substr($this->import_newId_pids[$origPid], 0, 3) === 'NEW') {
1796 if ($this->import_mapId['pages'][$origPid]) {
1797 $mappedPid = $this->import_mapId['pages'][$origPid];
1798 $cmd_data['pages'][$mappedPid]['move'] = $newPid;
1799 }
1800 } else {
1801 $cmd_data['pages'][$origPid]['move'] = $newPid;
1802 }
1803 }
1804 }
1805 // Execute the move commands if any:
1806 if (count($cmd_data)) {
1807 $tce = $this->getNewTCE();
1808 $this->callHook('before_writeRecordsPagesOrder', array(
1809 'tce' => &$tce,
1810 'data' => &$cmd_data
1811 ));
1812 $tce->start(array(), $cmd_data);
1813 $tce->process_cmdmap();
1814 $this->callHook('after_writeRecordsPagesOrder', array(
1815 'tce' => &$tce
1816 ));
1817 }
1818 }
1819
1820 /**
1821 * Write all database records except pages (writtein in writeRecords_pages())
1822 *
1823 * @param integer $pid Page id in which to import
1824 * @return void
1825 * @see writeRecords_pages()
1826 * @todo Define visibility
1827 */
1828 public function writeRecords_records($pid) {
1829 // Write the rest of the records
1830 $this->import_data = array();
1831 if (is_array($this->dat['header']['records'])) {
1832 foreach ($this->dat['header']['records'] as $table => $recs) {
1833 if ($table != 'pages') {
1834 foreach ($recs as $uid => $thisRec) {
1835 // PID: Set the main $pid, unless a NEW-id is found
1836 $setPid = isset($this->import_mapId['pages'][$thisRec['pid']]) ? $this->import_mapId['pages'][$thisRec['pid']] : $pid;
1837 if (is_array($GLOBALS['TCA'][$table]) && $GLOBALS['TCA'][$table]['ctrl']['rootLevel']) {
1838 $setPid = 0;
1839 }
1840 // Add record:
1841 $this->addSingle($table, $uid, $setPid);
1842 }
1843 }
1844 }
1845 } else {
1846 $this->error('Error: No records defined in internal data array.');
1847 }
1848 // Now write to database:
1849 $tce = $this->getNewTCE();
1850 $this->callHook('before_writeRecordsRecords', array(
1851 'tce' => &$tce,
1852 'data' => &$this->import_data
1853 ));
1854 $tce->suggestedInsertUids = $this->suggestedInsertUids;
1855 // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.
1856 $tce->reverseOrder = 1;
1857 $tce->isImporting = TRUE;
1858 $tce->start($this->import_data, array());
1859 $tce->process_datamap();
1860 $this->callHook('after_writeRecordsRecords', array(
1861 'tce' => &$tce
1862 ));
1863 // post-processing: Removing files and registering new ids (end all tcemain sessions with this)
1864 $this->addToMapId($tce->substNEWwithIDs);
1865 // In case of an update, order pages from the page tree correctly:
1866 if ($this->update) {
1867 $this->writeRecords_records_order($pid);
1868 }
1869 }
1870
1871 /**
1872 * Organize all updated record to their new positions.
1873 * Only used for updates
1874 *
1875 * @param integer $mainPid Main PID into which we import.
1876 * @return void
1877 * @access private
1878 * @see writeRecords_records(), writeRecords_pages_order()
1879 * @todo Define visibility
1880 */
1881 public function writeRecords_records_order($mainPid) {
1882 $cmd_data = array();
1883 if (is_array($this->dat['header']['pagetree'])) {
1884 $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);
1885 } else {
1886 $pagesFromTree = array();
1887 }
1888 if (is_array($this->dat['header']['pid_lookup'])) {
1889 foreach ($this->dat['header']['pid_lookup'] as $pid => $recList) {
1890 $newPid = isset($this->import_mapId['pages'][$pid]) ? $this->import_mapId['pages'][$pid] : $mainPid;
1891 if (\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($newPid)) {
1892 foreach ($recList as $tableName => $uidList) {
1893 // If $mainPid===$newPid then we are on root level and we can consider to move pages as well!
1894 // (they will not be in the page tree!)
1895 if (($tableName != 'pages' || !$pagesFromTree[$pid]) && is_array($uidList)) {
1896 $uidList = array_reverse(array_keys($uidList));
1897 foreach ($uidList as $uid) {
1898 if ($this->dontIgnorePid($tableName, $uid)) {
1899 $cmd_data[$tableName][$uid]['move'] = $newPid;
1900 } else {
1901
1902 }
1903 }
1904 }
1905 }
1906 }
1907 }
1908 }
1909 // Execute the move commands if any:
1910 if (count($cmd_data)) {
1911 $tce = $this->getNewTCE();
1912 $this->callHook('before_writeRecordsRecordsOrder', array(
1913 'tce' => &$tce,
1914 'data' => &$cmd_data
1915 ));
1916 $tce->start(array(), $cmd_data);
1917 $tce->process_cmdmap();
1918 $this->callHook('after_writeRecordsRecordsOrder', array(
1919 'tce' => &$tce
1920 ));
1921 }
1922 }
1923
1924 /**
1925 * Adds a single record to the $importData array. Also copies files to tempfolder.
1926 * However all File/DB-references and flexform field contents are set to blank for now! That is done with setRelations() later
1927 *
1928 * @param string $table Table name (from import memory)
1929 * @param integer $uid Record UID (from import memory)
1930 * @param integer $pid Page id
1931 * @return void
1932 * @see writeRecords()
1933 * @todo Define visibility
1934 */
1935 public function addSingle($table, $uid, $pid) {
1936 if ($this->import_mode[$table . ':' . $uid] !== 'exclude') {
1937 $record = $this->dat['records'][$table . ':' . $uid]['data'];
1938 if (is_array($record)) {
1939 if ($this->update && $this->doesRecordExist($table, $uid) && $this->import_mode[$table . ':' . $uid] !== 'as_new') {
1940 $ID = $uid;
1941 } elseif ($table === 'sys_file_metadata' && $record['sys_language_uid'] == '0' && $this->import_mapId['sys_file'][$record['file']]) {
1942 // on adding sys_file records the belonging sys_file_metadata record was also created
1943 // if there is one the record need to be overwritten instead of creating a new one.
1944 $recordInDatabase = $GLOBALS['TYPO3_DB']->exec_SELECTgetSingleRow(
1945 'uid',
1946 'sys_file_metadata',
1947 'file = ' . $this->import_mapId['sys_file'][$record['file']] . ' AND sys_language_uid = 0 AND pid = 0'
1948 );
1949 // if no record could be found, $this->import_mapId['sys_file'][$record['file']] is pointing
1950 // to a file, that was already there, thus a new metadata record should be created
1951 if (is_array($recordInDatabase)) {
1952 $this->import_mapId['sys_file_metadata'][$record['uid']] = $recordInDatabase['uid'];
1953 $ID = $recordInDatabase['uid'];
1954 } else {
1955 $ID = uniqid('NEW');
1956 }
1957
1958 } else {
1959 $ID = uniqid('NEW');
1960 }
1961 $this->import_newId[$table . ':' . $ID] = array('table' => $table, 'uid' => $uid);
1962 if ($table == 'pages') {
1963 $this->import_newId_pids[$uid] = $ID;
1964 }
1965 // Set main record data:
1966 $this->import_data[$table][$ID] = $record;
1967 $this->import_data[$table][$ID]['tx_impexp_origuid'] = $this->import_data[$table][$ID]['uid'];
1968 // Reset permission data:
1969 if ($table === 'pages') {
1970 // Have to reset the user/group IDs so pages are owned by importing user. Otherwise strange things may happen for non-admins!
1971 unset($this->import_data[$table][$ID]['perms_userid']);
1972 unset($this->import_data[$table][$ID]['perms_groupid']);
1973 }
1974 // PID and UID:
1975 unset($this->import_data[$table][$ID]['uid']);
1976 // Updates:
1977 if (\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($ID)) {
1978 unset($this->import_data[$table][$ID]['pid']);
1979 } else {
1980 // Inserts:
1981 $this->import_data[$table][$ID]['pid'] = $pid;
1982 if (($this->import_mode[$table . ':' . $uid] === 'force_uid' && $this->update || $this->force_all_UIDS) && $GLOBALS['BE_USER']->isAdmin()) {
1983 $this->import_data[$table][$ID]['uid'] = $uid;
1984 $this->suggestedInsertUids[$table . ':' . $uid] = 'DELETE';
1985 }
1986 }
1987 // Setting db/file blank:
1988 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
1989 switch ((string) $config['type']) {
1990 case 'db':
1991
1992 case 'file':
1993 // Fixed later in ->setRelations() [because we need to know ALL newly created IDs before we can map relations!]
1994 // In the meantime we set NO values for relations.
1995 //
1996 // BUT for field uid_local of table sys_file_reference the relation MUST not be cleared here,
1997 // because the value is already the uid of the right imported sys_file record.
1998 // @see fixUidLocalInSysFileReferenceRecords()
1999 // If it's empty or a uid to another record the FileExtensionFilter will throw an exception or
2000 // delete the reference record if the file extension of the related record doesn't match.
2001 if ($table !== 'sys_file_reference' && $field !== 'uid_local') {
2002 $this->import_data[$table][$ID][$field] = '';
2003 }
2004 break;
2005 case 'flex':
2006 // Fixed later in setFlexFormRelations()
2007 // In the meantime we set NO value for flexforms - this is mainly because file references inside will not be processed properly; In fact references will point to no file or existing files (in which case there will be double-references which is a big problem of course!)
2008 $this->import_data[$table][$ID][$field] = '';
2009 break;
2010 }
2011 }
2012 } elseif ($table . ':' . $uid != 'pages:0') {
2013 // On root level we don't want this error message.
2014 $this->error('Error: no record was found in data array!', 1);
2015 }
2016 }
2017 }
2018
2019 /**
2020 * Registers the substNEWids in memory.
2021 *
2022 * @param array $substNEWwithIDs From tcemain to be merged into internal mapping variable in this object
2023 * @return void
2024 * @see writeRecords()
2025 * @todo Define visibility
2026 */
2027 public function addToMapId($substNEWwithIDs) {
2028 foreach ($this->import_data as $table => $recs) {
2029 foreach ($recs as $id => $value) {
2030 $old_uid = $this->import_newId[$table . ':' . $id]['uid'];
2031 if (isset($substNEWwithIDs[$id])) {
2032 $this->import_mapId[$table][$old_uid] = $substNEWwithIDs[$id];
2033 } elseif ($this->update) {
2034 // Map same ID to same ID....
2035 $this->import_mapId[$table][$old_uid] = $id;
2036 } else {
2037 $this->error('Possible error: ' . $table . ':' . $old_uid . ' had no new id assigned to it. This indicates that the record was not added to database during import. Please check changelog!', 1);
2038 }
2039 }
2040 }
2041 }
2042
2043 /**
2044 * Returns a new $TCE object
2045 *
2046 * @return object $TCE object
2047 * @todo Define visibility
2048 */
2049 public function getNewTCE() {
2050 $tce = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\DataHandling\\DataHandler');
2051 $tce->stripslashes_values = 0;
2052 $tce->dontProcessTransformations = 1;
2053 $tce->enableLogging = $this->enableLogging;
2054 $tce->alternativeFileName = $this->alternativeFileName;
2055 $tce->alternativeFilePath = $this->alternativeFilePath;
2056 return $tce;
2057 }
2058
2059 /**
2060 * Cleaning up all the temporary files stored in typo3temp/ folder
2061 *
2062 * @return void
2063 * @todo Define visibility
2064 */
2065 public function unlinkTempFiles() {
2066 foreach ($this->unlinkFiles as $fileName) {
2067 if (GeneralUtility::isFirstPartOfStr($fileName, PATH_site . 'typo3temp/')) {
2068 GeneralUtility::unlink_tempfile($fileName);
2069 clearstatcache();
2070 if (is_file($fileName)) {
2071 $this->error('Error: ' . $fileName . ' was NOT unlinked as it should have been!', 1);
2072 }
2073 } else {
2074 $this->error('Error: ' . $fileName . ' was not in temp-path. Not removed!', 1);
2075 }
2076 }
2077 $this->unlinkFiles = array();
2078 }
2079
2080 /***************************
2081 * Import / Relations setting
2082 ***************************/
2083
2084 /**
2085 * At the end of the import process all file and DB relations should be set properly (that is relations to imported records are all re-created so imported records are correctly related again)
2086 * Relations in flexform fields are processed in setFlexFormRelations() after this function
2087 *
2088 * @return void
2089 * @see setFlexFormRelations()
2090 * @todo Define visibility
2091 */
2092 public function setRelations() {
2093 $updateData = array();
2094 // import_newId contains a register of all records that was in the import memorys "records" key
2095 foreach ($this->import_newId as $nId => $dat) {
2096 $table = $dat['table'];
2097 $uid = $dat['uid'];
2098 // original UID - NOT the new one!
2099 // If the record has been written and received a new id, then proceed:
2100 if (is_array($this->import_mapId[$table]) && isset($this->import_mapId[$table][$uid])) {
2101 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
2102 if (is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {
2103 if ($this->legacyImport) {
2104 if ($table != 'pages') {
2105 $oldPid = $this->dat['records'][$table . ':' . $uid]['data']['pid'];
2106 $thisNewPageUid = \TYPO3\CMS\Backend\Utility\BackendUtility::wsMapId($table, $this->import_mapId['pages'][$oldPid]);
2107 } else {
2108 $thisNewPageUid = $thisNewUid;
2109 }
2110 }
2111 // Traverse relation fields of each record
2112 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
2113 // uid_local of sys_file_reference needs no update because the correct reference uid was already written
2114 // @see ImportExport::fixUidLocalInSysFileReferenceRecords()
2115 if ($table === 'sys_file_reference' && $field === 'uid_local') {
2116 continue;
2117 }
2118 switch ((string) $config['type']) {
2119 case 'db':
2120 if (is_array($config['itemArray']) && count($config['itemArray'])) {
2121 $itemConfig = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
2122 $valArray = $this->setRelations_db($config['itemArray'], $itemConfig);
2123 $updateData[$table][$thisNewUid][$field] = implode(',', $valArray);
2124 }
2125 break;
2126 case 'file':
2127 if (is_array($config['newValueFiles']) && count($config['newValueFiles'])) {
2128 $valArr = array();
2129 foreach ($config['newValueFiles'] as $fI) {
2130 $valArr[] = $this->import_addFileNameToBeCopied($fI);
2131 }
2132 if ($this->legacyImport && $this->legacyImportFolder === NULL && isset($this->legacyImportMigrationTables[$table][$field])) {
2133 // Do nothing - the legacy import folder is missing
2134 } elseif ($this->legacyImport && $this->legacyImportFolder !== NULL && isset($this->legacyImportMigrationTables[$table][$field])) {
2135 $refIds = array();
2136 foreach ($valArr as $tempFile) {
2137 $fileName = $this->alternativeFileName[$tempFile];
2138 $fileObject = NULL;
2139
2140 try {
2141 // check, if there is alreay the same file in the folder
2142 if ($this->legacyImportFolder->hasFile($fileName)) {
2143 $file = $this->legacyImportFolder->getFile($fileName);
2144 if ($file->getSha1() === sha1_file($tempFile)) {
2145 $fileObject = $file;
2146 }
2147 }
2148 } catch (Exception $e) {}
2149
2150 if ($fileObject === NULL) {
2151 try {
2152 $fileObject = $this->legacyImportFolder->addFile($tempFile, $fileName, 'changeName');
2153 } catch (\TYPO3\CMS\Core\Exception $e) {
2154 $this->error('Error: no file could be added to the storage for file name' . $this->alternativeFileName[$tempFile]);
2155 }
2156 }
2157 if ($fileObject !== NULL) {
2158 $refId = uniqid('NEW');
2159 $refIds[] = $refId;
2160 $updateData['sys_file_reference'][$refId] = array(
2161 'uid_local' => $fileObject->getUid(),
2162 'uid_foreign' => $thisNewUid, // uid of your content record
2163 'tablenames' => $table,
2164 'fieldname' => $field,
2165 'pid' => $thisNewPageUid, // parent id of the parent page
2166 'table_local' => 'sys_file',
2167 );
2168 }
2169 }
2170 $updateData[$table][$thisNewUid][$field] = implode(',', $refIds);
2171 if (!empty($this->legacyImportMigrationTables[$table][$field])) {
2172 $this->legacyImportMigrationRecords[$table][$thisNewUid][$field] = $refIds;
2173 }
2174 } else {
2175 $updateData[$table][$thisNewUid][$field] = implode(',', $valArr);
2176 }
2177 }
2178 break;
2179 }
2180 }
2181 } else {
2182 $this->error('Error: no record was found in data array!', 1);
2183 }
2184 } else {
2185 $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')', 1);
2186 }
2187 }
2188 if (count($updateData)) {
2189 $tce = $this->getNewTCE();
2190 $tce->isImporting = TRUE;
2191 $this->callHook('before_setRelation', array(
2192 'tce' => &$tce,
2193 'data' => &$updateData
2194 ));
2195 $tce->start($updateData, array());
2196 $tce->process_datamap();
2197 // Replace the temporary "NEW" ids with the final ones.
2198 foreach ($this->legacyImportMigrationRecords as $table => $records) {
2199 foreach ($records as $uid => $fields) {
2200 foreach ($fields as $field => $referenceIds) {
2201 foreach ($referenceIds as $key => $referenceId) {
2202 $this->legacyImportMigrationRecords[$table][$uid][$field][$key] = $tce->substNEWwithIDs[$referenceId];
2203 }
2204 }
2205 }
2206 }
2207 $this->callHook('after_setRelations', array(
2208 'tce' => &$tce
2209 ));
2210 }
2211 }
2212
2213 /**
2214 * Maps relations for database
2215 *
2216 * @param array $itemArray Array of item sets (table/uid) from a dbAnalysis object
2217 * @param array $itemConfig Array of TCA config of the field the relation to be set on
2218 * @return array Array with values [table]_[uid] or [uid] for field of type group / internal_type file_reference. These values have the regular tcemain-input group/select type which means they will automatically be processed into a uid-list or MM relations.
2219 * @todo Define visibility
2220 */
2221 public function setRelations_db($itemArray, $itemConfig) {
2222 $valArray = array();
2223 foreach ($itemArray as $relDat) {
2224 if (is_array($this->import_mapId[$relDat['table']]) && isset($this->import_mapId[$relDat['table']][$relDat['id']])) {
2225 // Since non FAL file relation type group internal_type file_reference are handled as reference to
2226 // sys_file records Datahandler requires the value as uid of the the related sys_file record only
2227 if ($itemConfig['type'] === 'group' && $itemConfig['internal_type'] === 'file_reference') {
2228 $value = $this->import_mapId[$relDat['table']][$relDat['id']];
2229 } else {
2230 $value = $relDat['table'] . '_' . $this->import_mapId[$relDat['table']][$relDat['id']];
2231 }
2232 $valArray[] = $value;
2233 } elseif ($this->isTableStatic($relDat['table']) || $this->isExcluded($relDat['table'], $relDat['id']) || $relDat['id'] < 0) {
2234 // Checking for less than zero because some select types could contain negative values, eg. fe_groups (-1, -2) and sys_language (-1 = ALL languages). This must be handled on both export and import.
2235 $valArray[] = $relDat['table'] . '_' . $relDat['id'];
2236 } else {
2237 $this->error('Lost relation: ' . $relDat['table'] . ':' . $relDat['id'], 1);
2238 }
2239 }
2240 return $valArray;
2241 }
2242
2243 /**
2244 * Writes the file from import array to temp dir and returns the filename of it.
2245 *
2246 * @param array $fI File information with three keys: "filename" = filename without path, "ID_absFile" = absolute filepath to the file (including the filename), "ID" = md5 hash of "ID_absFile
2247 * @return string Absolute filename of the temporary filename of the file. In ->alternativeFileName the original name is set.
2248 * @todo Define visibility
2249 */
2250 public function import_addFileNameToBeCopied($fI) {
2251 if (is_array($this->dat['files'][$fI['ID']])) {
2252 $tmpFile = GeneralUtility::tempnam('import_temp_');
2253 GeneralUtility::writeFile($tmpFile, $this->dat['files'][$fI['ID']]['content']);
2254 clearstatcache();
2255 if (@is_file($tmpFile)) {
2256 $this->unlinkFiles[] = $tmpFile;
2257 if (filesize($tmpFile) == $this->dat['files'][$fI['ID']]['filesize']) {
2258 $this->alternativeFileName[$tmpFile] = $fI['filename'];
2259 $this->alternativeFilePath[$tmpFile] = $this->dat['files'][$fI['ID']]['relFileRef'];
2260 return $tmpFile;
2261 } else {
2262 $this->error('Error: temporary file ' . $tmpFile . ' had a size (' . filesize($tmpFile) . ') different from the original (' . $this->dat['files'][$fI['ID']]['filesize'] . ')', 1);
2263 }
2264 } else {
2265 $this->error('Error: temporary file ' . $tmpFile . ' was not written as it should have been!', 1);
2266 }
2267 } else {
2268 $this->error('Error: No file found for ID ' . $fI['ID'], 1);
2269 }
2270 }
2271
2272 /**
2273 * After all DB relations has been set in the end of the import (see setRelations()) then it is time to correct all relations inside of FlexForm fields.
2274 * The reason for doing this after is that the setting of relations may affect (quite often!) which data structure is used for the flexforms field!
2275 *
2276 * @return void
2277 * @see setRelations()
2278 * @todo Define visibility
2279 */
2280 public function setFlexFormRelations() {
2281 $updateData = array();
2282 // import_newId contains a register of all records that was in the import memorys "records" key
2283 foreach ($this->import_newId as $nId => $dat) {
2284 $table = $dat['table'];
2285 $uid = $dat['uid'];
2286 // original UID - NOT the new one!
2287 // If the record has been written and received a new id, then proceed:
2288 if (is_array($this->import_mapId[$table]) && isset($this->import_mapId[$table][$uid])) {
2289 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
2290 if (is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {
2291 // Traverse relation fields of each record
2292 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
2293 switch ((string) $config['type']) {
2294 case 'flex':
2295 // Get XML content and set as default value (string, non-processed):
2296 $updateData[$table][$thisNewUid][$field] = $this->dat['records'][$table . ':' . $uid]['data'][$field];
2297 // If there has been registered relations inside the flex form field, run processing on the content:
2298 if (count($config['flexFormRels']['db']) || count($config['flexFormRels']['file'])) {
2299 $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');
2300 // This will fetch the new row for the element (which should be updated with any references to data structures etc.)
2301 $conf = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
2302 if (is_array($origRecordRow) && is_array($conf) && $conf['type'] === 'flex') {
2303 // Get current data structure and value array:
2304 $dataStructArray = BackendUtility::getFlexFormDS($conf, $origRecordRow, $table);
2305 $currentValueArray = GeneralUtility::xml2array($updateData[$table][$thisNewUid][$field]);
2306 // Do recursive processing of the XML data:
2307 $iteratorObj = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\DataHandling\\DataHandler');
2308 $iteratorObj->callBackObj = $this;
2309 $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData($currentValueArray['data'], array(), array(), $dataStructArray, array($table, $thisNewUid, $field, $config), 'remapListedDBRecords_flexFormCallBack');
2310 // The return value is set as an array which means it will be processed by tcemain for file and DB references!
2311 if (is_array($currentValueArray['data'])) {
2312 $updateData[$table][$thisNewUid][$field] = $currentValueArray;
2313 }
2314 }
2315 }
2316 break;
2317 }
2318 }
2319 } else {
2320 $this->error('Error: no record was found in data array!', 1);
2321 }
2322 } else {
2323 $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')', 1);
2324 }
2325 }
2326 if (count($updateData)) {
2327 $tce = $this->getNewTCE();
2328 $tce->isImporting = TRUE;
2329 $this->callHook('before_setFlexFormRelations', array(
2330 'tce' => &$tce,
2331 'data' => &$updateData
2332 ));
2333 $tce->start($updateData, array());
2334 $tce->process_datamap();
2335 $this->callHook('after_setFlexFormRelations', array(
2336 'tce' => &$tce
2337 ));
2338 }
2339 }
2340
2341 /**
2342 * Callback function for traversing the FlexForm structure in relation to remapping database relations
2343 *
2344 * @param array $pParams Set of parameters in numeric array: table, uid, field
2345 * @param array $dsConf TCA config for field (from Data Structure of course)
2346 * @param string $dataValue Field value (from FlexForm XML)
2347 * @param string $dataValue_ext1 Not used
2348 * @param string $dataValue_ext2 Not used
2349 * @param string $path Path of where the data structure of the element is found
2350 * @return array Array where the "value" key carries the value.
2351 * @see setFlexFormRelations()
2352 * @todo Define visibility
2353 */
2354 public function remapListedDBRecords_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path) {
2355 // Extract parameters:
2356 list($table, $uid, $field, $config) = $pParams;
2357 // In case the $path is used as index without a trailing slash we will remove that
2358 if (!is_array($config['flexFormRels']['db'][$path]) && is_array($config['flexFormRels']['db'][rtrim($path, '/')])) {
2359 $path = rtrim($path, '/');
2360 }
2361 if (is_array($config['flexFormRels']['db'][$path])) {
2362 $valArray = $this->setRelations_db($config['flexFormRels']['db'][$path], $dsConf);
2363 $dataValue = implode(',', $valArray);
2364 }
2365 if (is_array($config['flexFormRels']['file'][$path])) {
2366 foreach ($config['flexFormRels']['file'][$path] as $fI) {
2367 $valArr[] = $this->import_addFileNameToBeCopied($fI);
2368 }
2369 $dataValue = implode(',', $valArr);
2370 }
2371 return array('value' => $dataValue);
2372 }
2373
2374 /**************************
2375 * Import / Soft References
2376 *************************/
2377
2378 /**
2379 * Processing of soft references
2380 *
2381 * @return void
2382 * @todo Define visibility
2383 */
2384 public function processSoftReferences() {
2385 // Initialize:
2386 $inData = array();
2387 // Traverse records:
2388 if (is_array($this->dat['header']['records'])) {
2389 foreach ($this->dat['header']['records'] as $table => $recs) {
2390 foreach ($recs as $uid => $thisRec) {
2391 // If there are soft references defined, traverse those:
2392 if (isset($GLOBALS['TCA'][$table]) && is_array($thisRec['softrefs'])) {
2393 // First traversal is to collect softref configuration and split them up based on fields. This could probably also have been done with the "records" key instead of the header.
2394 $fieldsIndex = array();
2395 foreach ($thisRec['softrefs'] as $softrefDef) {
2396 // If a substitution token is set:
2397 if ($softrefDef['field'] && is_array($softrefDef['subst']) && $softrefDef['subst']['tokenID']) {
2398 $fieldsIndex[$softrefDef['field']][$softrefDef['subst']['tokenID']] = $softrefDef;
2399 }
2400 }
2401 // The new id:
2402 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
2403 // Now, if there are any fields that require substitution to be done, lets go for that:
2404 foreach ($fieldsIndex as $field => $softRefCfgs) {
2405 if (is_array($GLOBALS['TCA'][$table]['columns'][$field])) {
2406 $conf = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
2407 if ($conf['type'] === 'flex') {
2408 // This will fetch the new row for the element (which should be updated with any references to data structures etc.)
2409 $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');
2410 if (is_array($origRecordRow)) {
2411 // Get current data structure and value array:
2412 $dataStructArray = BackendUtility::getFlexFormDS($conf, $origRecordRow, $table);
2413 $currentValueArray = GeneralUtility::xml2array($origRecordRow[$field]);
2414 // Do recursive processing of the XML data:
2415 /** @var $iteratorObj \TYPO3\CMS\Core\DataHandling\DataHandler */
2416 $iteratorObj = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\DataHandling\\DataHandler');
2417 $iteratorObj->callBackObj = $this;
2418 $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData($currentValueArray['data'], array(), array(), $dataStructArray, array($table, $uid, $field, $softRefCfgs), 'processSoftReferences_flexFormCallBack');
2419 // The return value is set as an array which means it will be processed by tcemain for file and DB references!
2420 if (is_array($currentValueArray['data'])) {
2421 $inData[$table][$thisNewUid][$field] = $currentValueArray;
2422 }
2423 }
2424 } else {
2425 // Get tokenizedContent string and proceed only if that is not blank:
2426 $tokenizedContent = $this->dat['records'][$table . ':' . $uid]['rels'][$field]['softrefs']['tokenizedContent'];
2427 if (strlen($tokenizedContent) && is_array($softRefCfgs)) {
2428 $inData[$table][$thisNewUid][$field] = $this->processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid);
2429 }
2430 }
2431 }
2432 }
2433 }
2434 }
2435 }
2436 }
2437 // Now write to database:
2438 $tce = $this->getNewTCE();
2439 $tce->isImporting = TRUE;
2440 $this->callHook('before_processSoftReferences', array(
2441 'tce' => &$tce,
2442 'data' => &$inData
2443 ));
2444 $tce->enableLogging = TRUE;
2445 $tce->start($inData, array());
2446 $tce->process_datamap();
2447 $this->callHook('after_processSoftReferences', array(
2448 'tce' => &$tce
2449 ));
2450 }
2451
2452 /**
2453 * Callback function for traversing the FlexForm structure in relation to remapping softreference relations
2454 *
2455 * @param array $pParams Set of parameters in numeric array: table, uid, field
2456 * @param array $dsConf TCA config for field (from Data Structure of course)
2457 * @param string $dataValue Field value (from FlexForm XML)
2458 * @param string $dataValue_ext1 Not used
2459 * @param string $dataValue_ext2 Not used
2460 * @param string $path Path of where the data structure where the element is found
2461 * @return array Array where the "value" key carries the value.
2462 * @see setFlexFormRelations()
2463 * @todo Define visibility
2464 */
2465 public function processSoftReferences_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path) {
2466 // Extract parameters:
2467 list($table, $origUid, $field, $softRefCfgs) = $pParams;
2468 if (is_array($softRefCfgs)) {
2469 // First, find all soft reference configurations for this structure path (they are listed flat in the header):
2470 $thisSoftRefCfgList = array();
2471 foreach ($softRefCfgs as $sK => $sV) {
2472 if ($sV['structurePath'] === $path) {
2473 $thisSoftRefCfgList[$sK] = $sV;
2474 }
2475 }
2476 // If any was found, do processing:
2477 if (count($thisSoftRefCfgList)) {
2478 // Get tokenizedContent string and proceed only if that is not blank:
2479 $tokenizedContent = $this->dat['records'][$table . ':' . $origUid]['rels'][$field]['flexFormRels']['softrefs'][$path]['tokenizedContent'];
2480 if (strlen($tokenizedContent)) {
2481 $dataValue = $this->processSoftReferences_substTokens($tokenizedContent, $thisSoftRefCfgList, $table, $origUid);
2482 }
2483 }
2484 }
2485 // Return
2486 return array('value' => $dataValue);
2487 }
2488
2489 /**
2490 * Substition of softreference tokens
2491 *
2492 * @param string $tokenizedContent Content of field with soft reference tokens in.
2493 * @param array $softRefCfgs Soft reference configurations
2494 * @param string $table Table for which the processing occurs
2495 * @param string $uid UID of record from table
2496 * @return string The input content with tokens substituted according to entries in softRefCfgs
2497 * @todo Define visibility
2498 */
2499 public function processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid) {
2500 // traverse each softref type for this field:
2501 foreach ($softRefCfgs as $cfg) {
2502 // Get token ID:
2503 $tokenID = $cfg['subst']['tokenID'];
2504 // Default is current token value:
2505 $insertValue = $cfg['subst']['tokenValue'];
2506 // Based on mode:
2507 switch ((string) $this->softrefCfg[$tokenID]['mode']) {
2508 case 'exclude':
2509 // Exclude is a simple passthrough of the value
2510 break;
2511 case 'editable':
2512 // Editable always picks up the value from this input array:
2513 $insertValue = $this->softrefInputValues[$tokenID];
2514 break;
2515 default:
2516 // Mapping IDs/creating files: Based on type, look up new value:
2517 switch ((string) $cfg['subst']['type']) {
2518 case 'file':
2519 // Create / Overwrite file:
2520 $insertValue = $this->processSoftReferences_saveFile($cfg['subst']['relFileName'], $cfg, $table, $uid);
2521 break;
2522 case 'db':
2523 default:
2524 // Trying to map database element if found in the mapID array:
2525 list($tempTable, $tempUid) = explode(':', $cfg['subst']['recordRef']);
2526 if (isset($this->import_mapId[$tempTable][$tempUid])) {
2527 $insertValue = BackendUtility::wsMapId($tempTable, $this->import_mapId[$tempTable][$tempUid]);
2528 // Look if reference is to a page and the original token value was NOT an integer - then we assume is was an alias and try to look up the new one!
2529 if ($tempTable === 'pages' && !\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($cfg['subst']['tokenValue'])) {
2530 $recWithUniqueValue = BackendUtility::getRecord($tempTable, $insertValue, 'alias');
2531 if ($recWithUniqueValue['alias']) {
2532 $insertValue = $recWithUniqueValue['alias'];
2533 }
2534 } elseif (strpos($cfg['subst']['tokenValue'], ':') !== FALSE) {
2535 list($tokenKey, $tokenId) = explode(':', $cfg['subst']['tokenValue']);
2536 $insertValue = $tokenKey . ':' . $insertValue;
2537 }
2538 }
2539 }
2540 }
2541 // Finally, swap the soft reference token in tokenized content with the insert value:
2542 $tokenizedContent = str_replace('{softref:' . $tokenID . '}', $insertValue, $tokenizedContent);
2543 }
2544 return $tokenizedContent;
2545 }
2546
2547 /**
2548 * Process a soft reference file
2549 *
2550 * @param string $relFileName Old Relative filename
2551 * @param array $cfg soft reference configuration array
2552 * @param string $table Table for which the processing occurs
2553 * @param string $uid UID of record from table
2554 * @return string New relative filename (value to insert instead of the softref token)
2555 * @todo Define visibility
2556 */
2557 public function processSoftReferences_saveFile($relFileName, $cfg, $table, $uid) {
2558 if ($fileHeaderInfo = $this->dat['header']['files'][$cfg['file_ID']]) {
2559 // Initialize; Get directory prefix for file and find possible RTE filename
2560 $dirPrefix = PathUtility::dirname($relFileName) . '/';
2561 $rteOrigName = $this->getRTEoriginalFilename(PathUtility::basename($relFileName));
2562 // If filename looks like an RTE file, and the directory is in "uploads/", then process as a RTE file!
2563 if ($rteOrigName && GeneralUtility::isFirstPartOfStr($dirPrefix, 'uploads/')) {
2564 // RTE:
2565 // First, find unique RTE file name:
2566 if (@is_dir((PATH_site . $dirPrefix))) {
2567 // From the "original" RTE filename, produce a new "original" destination filename which is unused. Even if updated, the image should be unique. Currently the problem with this is that it leaves a lot of unused RTE images...
2568 $fileProcObj = $this->getFileProcObj();
2569 $origDestName = $fileProcObj->getUniqueName($rteOrigName, PATH_site . $dirPrefix);
2570 // Create copy file name:
2571 $pI = pathinfo($relFileName);
2572 $copyDestName = PathUtility::dirname($origDestName) . '/RTEmagicC_' . substr(PathUtility::basename($origDestName), 10) . '.' . $pI['extension'];
2573 if (!@is_file($copyDestName) && !@is_file($origDestName) && $origDestName === GeneralUtility::getFileAbsFileName($origDestName) && $copyDestName === GeneralUtility::getFileAbsFileName($copyDestName)) {
2574 if ($this->dat['header']['files'][$fileHeaderInfo['RTE_ORIG_ID']]) {
2575 if ($this->legacyImport) {
2576 $fileName = PathUtility::basename($copyDestName);
2577 $this->writeSysFileResourceForLegacyImport($fileName, $cfg['file_ID']);
2578 $relFileName = $this->filePathMap[$cfg['file_ID']] . '" data-htmlarea-file-uid="' . $fileName . '" data-htmlarea-file-table="sys_file';
2579 // Also save the original file
2580 $originalFileName = PathUtility::basename($origDestName);
2581 $this->writeSysFileResourceForLegacyImport($originalFileName, $fileHeaderInfo['RTE_ORIG_ID']);
2582 } else {
2583 // Write the copy and original RTE file to the respective filenames:
2584 $this->writeFileVerify($copyDestName, $cfg['file_ID'], TRUE);
2585 $this->writeFileVerify($origDestName, $fileHeaderInfo['RTE_ORIG_ID'], TRUE);
2586 // Return the relative path of the copy file name:
2587 return \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($copyDestName);
2588 }
2589 } else {
2590 $this->error('ERROR: Could not find original file ID');
2591 }
2592 } else {
2593 $this->error('ERROR: The destination filenames "' . $copyDestName . '" and "' . $origDestName . '" either existed or have non-valid names');
2594 }
2595 } else {
2596 $this->error('ERROR: "' . PATH_site . $dirPrefix . '" was not a directory, so could not process file "' . $relFileName . '"');
2597 }
2598 } elseif (GeneralUtility::isFirstPartOfStr($dirPrefix, $this->fileadminFolderName . '/')) {
2599 // File in fileadmin/ folder:
2600 // Create file (and possible resources)
2601 $newFileName = $this->processSoftReferences_saveFile_createRelFile($dirPrefix, PathUtility::basename($relFileName), $cfg['file_ID'], $table, $uid);
2602 if (strlen($newFileName)) {
2603 $relFileName = $newFileName;
2604 } else {
2605 $this->error('ERROR: No new file created for "' . $relFileName . '"');
2606 }
2607 } else {
2608 $this->error('ERROR: Sorry, cannot operate on non-RTE files which are outside the fileadmin folder.');
2609 }
2610 } else {
2611 $this->error('ERROR: Could not find file ID in header.');
2612 }
2613 // Return (new) filename relative to PATH_site:
2614 return $relFileName;
2615 }
2616
2617 /**
2618 * Create file in directory and return the new (unique) filename
2619 *
2620 * @param string $origDirPrefix Directory prefix, relative, with trailing slash
2621 * @param string $fileName Filename (without path)
2622 * @param string $fileID File ID from import memory
2623 * @param string $table Table for which the processing occurs
2624 * @param string $uid UID of record from table
2625 * @return string New relative filename, if any
2626 * @todo Define visibility
2627 */
2628 public function processSoftReferences_saveFile_createRelFile($origDirPrefix, $fileName, $fileID, $table, $uid) {
2629 // If the fileID map contains an entry for this fileID then just return the relative filename of that entry; we don't want to write another unique filename for this one!
2630 if (isset($this->fileIDMap[$fileID])) {
2631 return \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($this->fileIDMap[$fileID]);
2632 }
2633 if ($this->legacyImport) {
2634 // set dirPrefix to fileadmin because the right target folder is set and checked for permissions later
2635 $dirPrefix = $this->fileadminFolderName . '/';
2636 } else {
2637 // Verify FileMount access to dir-prefix. Returns the best alternative relative path if any
2638 $dirPrefix = $this->verifyFolderAccess($origDirPrefix);
2639 }
2640 if ($dirPrefix && (!$this->update || $origDirPrefix === $dirPrefix) && $this->checkOrCreateDir($dirPrefix)) {
2641 $fileHeaderInfo = $this->dat['header']['files'][$fileID];
2642 $updMode = $this->update && $this->import_mapId[$table][$uid] === $uid && $this->import_mode[$table . ':' . $uid] !== 'as_new';
2643 // Create new name for file:
2644 // Must have same ID in map array (just for security, is not really needed) and NOT be set "as_new".
2645
2646 // Write main file:
2647 if ($this->legacyImport) {
2648 $fileWritten = $this->writeSysFileResourceForLegacyImport($fileName, $fileID);
2649 if ($fileWritten) {
2650 $newName = 'file:' . $fileName;
2651 return $newName;
2652 // no support for HTML/CSS file resources attached ATM - see below
2653 }
2654 } else {
2655 if ($updMode) {
2656 $newName = PATH_site . $dirPrefix . $fileName;
2657 } else {
2658 // Create unique filename:
2659 $fileProcObj = $this->getFileProcObj();
2660 $newName = $fileProcObj->getUniqueName($fileName, PATH_site . $dirPrefix);
2661 }
2662 if ($this->writeFileVerify($newName, $fileID)) {
2663 // If the resource was an HTML/CSS file with resources attached, we will write those as well!
2664 if (is_array($fileHeaderInfo['EXT_RES_ID'])) {
2665 $tokenizedContent = $this->dat['files'][$fileID]['tokenizedContent'];
2666 $tokenSubstituted = FALSE;
2667 $fileProcObj = $this->getFileProcObj();
2668 if ($updMode) {
2669 foreach ($fileHeaderInfo['EXT_RES_ID'] as $res_fileID) {
2670 if ($this->dat['files'][$res_fileID]['filename']) {
2671 // Resolve original filename:
2672 $relResourceFileName = $this->dat['files'][$res_fileID]['parentRelFileName'];
2673 $absResourceFileName = GeneralUtility::resolveBackPath(PATH_site . $origDirPrefix . $relResourceFileName);
2674 $absResourceFileName = GeneralUtility::getFileAbsFileName($absResourceFileName);
2675 if ($absResourceFileName && GeneralUtility::isFirstPartOfStr($absResourceFileName, PATH_site . $this->fileadminFolderName . '/')) {
2676 $destDir = \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix(PathUtility::dirname($absResourceFileName) . '/');
2677 if ($this->verifyFolderAccess($destDir, TRUE) && $this->checkOrCreateDir($destDir)) {
2678 $this->writeFileVerify($absResourceFileName, $res_fileID);
2679 } else {
2680 $this->error('ERROR: Could not create file in directory "' . $destDir . '"');
2681 }
2682 } else {
2683 $this->error('ERROR: Could not resolve path for "' . $relResourceFileName . '"');
2684 }
2685 $tokenizedContent = str_replace('{EXT_RES_ID:' . $res_fileID . '}', $relResourceFileName, $tokenizedContent);
2686 $tokenSubstituted = TRUE;
2687 }
2688 }
2689 } else {
2690 // Create the resouces directory name (filename without extension, suffixed "_FILES")
2691 $resourceDir = PathUtility::dirname($newName) . '/' . preg_replace('/\\.[^.]*$/', '', PathUtility::basename($newName)) . '_FILES';
2692 if (GeneralUtility::mkdir($resourceDir)) {
2693 foreach ($fileHeaderInfo['EXT_RES_ID'] as $res_fileID) {
2694 if ($this->dat['files'][$res_fileID]['filename']) {
2695 $absResourceFileName = $fileProcObj->getUniqueName($this->dat['files'][$res_fileID]['filename'], $resourceDir);
2696 $relResourceFileName = substr($absResourceFileName, strlen(PathUtility::dirname($resourceDir)) + 1);
2697 $this->writeFileVerify($absResourceFileName, $res_fileID);
2698 $tokenizedContent = str_replace('{EXT_RES_ID:' . $res_fileID . '}', $relResourceFileName, $tokenizedContent);
2699 $tokenSubstituted = TRUE;
2700 }
2701 }
2702 }
2703 }
2704 // If substitutions has been made, write the content to the file again:
2705 if ($tokenSubstituted) {
2706 GeneralUtility::writeFile($newName, $tokenizedContent);
2707 }
2708 }
2709 return \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($newName);
2710 }
2711 }
2712 }
2713 }
2714
2715 /**
2716 * Writes a file from the import memory having $fileID to file name $fileName which must be an absolute path inside PATH_site
2717 *
2718 * @param string $fileName Absolute filename inside PATH_site to write to
2719 * @param string $fileID File ID from import memory
2720 * @param boolean $bypassMountCheck Bypasses the checking against filemounts - only for RTE files!
2721 * @return boolean Returns TRUE if it went well. Notice that the content of the file is read again, and md5 from import memory is validated.
2722 * @todo Define visibility
2723 */
2724 public function writeFileVerify($fileName, $fileID, $bypassMountCheck = FALSE) {
2725 $fileProcObj = $this->getFileProcObj();
2726 if ($fileProcObj->actionPerms['addFile']) {
2727 // Just for security, check again. Should actually not be necessary.
2728 if ($fileProcObj->checkPathAgainstMounts($fileName) || $bypassMountCheck) {
2729 $fI = GeneralUtility::split_fileref($fileName);
2730 if ($fileProcObj->checkIfAllowed($fI['fileext'], $fI['path'], $fI['file']) || $this->allowPHPScripts && $GLOBALS['BE_USER']->isAdmin()) {
2731 if (GeneralUtility::getFileAbsFileName($fileName)) {
2732 if ($this->dat['files'][$fileID]) {
2733 GeneralUtility::writeFile($fileName, $this->dat['files'][$fileID]['content']);
2734 $this->fileIDMap[$fileID] = $fileName;
2735 if (md5(GeneralUtility::getUrl($fileName)) == $this->dat['files'][$fileID]['content_md5']) {
2736 return TRUE;
2737 } else {
2738 $this->error('ERROR: File content "' . $fileName . '" was corrupted');
2739 }
2740 } else {
2741 $this->error('ERROR: File ID "' . $fileID . '" could not be found');
2742 }
2743 } else {
2744 $this->error('ERROR: Filename "' . $fileName . '" was not a valid relative file path!');
2745 }
2746 } else {
2747 $this->error('ERROR: Filename "' . $fileName . '" failed against extension check or deny-pattern!');
2748 }
2749 } else {
2750 $this->error('ERROR: Filename "' . $fileName . '" was not allowed in destination path!');
2751 }
2752 } else {
2753 $this->error('ERROR: You did not have sufficient permissions to write the file "' . $fileName . '"');
2754 }
2755 }
2756
2757
2758 /**
2759 * Writes the file with the is $fileId to the legacy import folder. The file name will used from
2760 * argument $fileName and the file was successfully created or an identical file was already found,
2761 * $fileName will held the uid of the new created file record.
2762 *
2763 * @param $fileName The file name for the new file. Value would be changed to the uid of the new created file record.
2764 * @param $fileID The id of the file in data array
2765 * @return bool
2766 */
2767 protected function writeSysFileResourceForLegacyImport(&$fileName, $fileId) {
2768 if ($this->legacyImportFolder === NULL) {