[BUGFIX] Prevent fatal when importing a t3d file without fal information
[Packages/TYPO3.CMS.git] / typo3 / sysext / impexp / Classes / ImportExport.php
1 <?php
2 namespace TYPO3\CMS\Impexp;
3
4 /**
5 * This file is part of the TYPO3 CMS project.
6 *
7 * It is free software; you can redistribute it and/or modify it under
8 * the terms of the GNU General Public License, either version 2
9 * of the License, or any later version.
10 *
11 * For the full copyright and license information, please read the
12 * LICENSE.txt file that was distributed with this source code.
13 *
14 * The TYPO3 project - inspiring people to share!
15 */
16
17 use TYPO3\CMS\Backend\Utility\BackendUtility;
18 use TYPO3\CMS\Core\Exception;
19 use TYPO3\CMS\Core\Utility\GeneralUtility;
20 use TYPO3\CMS\Core\Utility\PathUtility;
21
22 /**
23 * EXAMPLE for using the impexp-class for exporting stuff:
24 *
25 * Create and initialize:
26 * $this->export = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance('TYPO3\\CMS\\Impexp\\ImportExport');
27 * $this->export->init();
28 * Set which tables relations we will allow:
29 * $this->export->relOnlyTables[]="tt_news"; // exclusively includes. See comment in the class
30 *
31 * Adding records:
32 * $this->export->export_addRecord("pages", $this->pageinfo);
33 * $this->export->export_addRecord("pages", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("pages", 38));
34 * $this->export->export_addRecord("pages", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("pages", 39));
35 * $this->export->export_addRecord("tt_content", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("tt_content", 12));
36 * $this->export->export_addRecord("tt_content", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("tt_content", 74));
37 * $this->export->export_addRecord("sys_template", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("sys_template", 20));
38 *
39 * Adding all the relations (recursively in 5 levels so relations has THEIR relations registered as well)
40 * for($a=0;$a<5;$a++) {
41 * $addR = $this->export->export_addDBRelations($a);
42 * if (!count($addR)) break;
43 * }
44 *
45 * Finally load all the files.
46 * $this->export->export_addFilesFromRelations(); // MUST be after the DBrelations are set so that file from ALL added records are included!
47 *
48 * Write export
49 * $out = $this->export->compileMemoryToFileContent();
50 */
51 /**
52 * T3D file Import/Export library (TYPO3 Record Document)
53 *
54 * @author Kasper Skårhøj <kasperYYYY@typo3.com>
55 */
56 class ImportExport {
57
58 /**
59 * If set, static relations (not exported) will be shown in overview as well
60 *
61 * @todo Define visibility
62 */
63 public $showStaticRelations = FALSE;
64
65 /**
66 * Name of the "fileadmin" folder where files for export/import should be located
67 *
68 * @todo Define visibility
69 */
70 public $fileadminFolderName = '';
71
72 /**
73 * Whether "import" or "export" mode of object. Set through init() function
74 *
75 * @todo Define visibility
76 */
77 public $mode = '';
78
79 /**
80 * Updates all records that has same UID instead of creating new!
81 *
82 * @todo Define visibility
83 */
84 public $update = FALSE;
85
86 /**
87 * Is set by importData() when an import has been done.
88 *
89 * @todo Define visibility
90 */
91 public $doesImport = FALSE;
92
93 /**
94 * If set to a page-record, then the preview display of the content will expect this page-record to be the target
95 * for the import and accordingly display validation information. This triggers the visual view of the
96 * import/export memory to validate if import is possible
97 *
98 * @todo Define visibility
99 */
100 public $display_import_pid_record = '';
101
102 /**
103 * Used to register the forged UID values for imported records that we want to create with the same UIDs as in the import file. Admin-only feature.
104 *
105 * @todo Define visibility
106 */
107 public $suggestedInsertUids = array();
108
109 /**
110 * Setting import modes during update state: as_new, exclude, force_uid
111 *
112 * @todo Define visibility
113 */
114 public $import_mode = array();
115
116 /**
117 * If set, PID correct is ignored globally
118 *
119 * @todo Define visibility
120 */
121 public $global_ignore_pid = FALSE;
122
123 /**
124 * If set, all UID values are forced! (update or import)
125 *
126 * @todo Define visibility
127 */
128 public $force_all_UIDS = FALSE;
129
130 /**
131 * If set, a diff-view column is added to the overview.
132 *
133 * @todo Define visibility
134 */
135 public $showDiff = FALSE;
136
137 /**
138 * If set, and if the user is admin, allow the writing of PHP scripts to fileadmin/ area.
139 *
140 * @todo Define visibility
141 */
142 public $allowPHPScripts = FALSE;
143
144 /**
145 * Disable logging when importing
146 *
147 * @todo Define visibility
148 */
149 public $enableLogging = FALSE;
150
151 /**
152 * Array of values to substitute in editable softreferences.
153 *
154 * @todo Define visibility
155 */
156 public $softrefInputValues = array();
157
158 /**
159 * Mapping between the fileID from import memory and the final filenames they are written to.
160 *
161 * @todo Define visibility
162 */
163 public $fileIDMap = array();
164
165 /**
166 * 1MB max file size
167 *
168 * @todo Define visibility
169 */
170 public $maxFileSize = 1000000;
171
172 /**
173 * 1MB max record size
174 *
175 * @todo Define visibility
176 */
177 public $maxRecordSize = 1000000;
178
179 /**
180 * 10MB max export size
181 *
182 * @todo Define visibility
183 */
184 public $maxExportSize = 10000000;
185
186 /**
187 * Add table names here which are THE ONLY ones which will be included into export if found as relations. '_ALL' will allow all tables.
188 *
189 * @todo Define visibility
190 */
191 public $relOnlyTables = array();
192
193 /**
194 * Add tables names here which should not be exported with the file. (Where relations should be mapped to same UIDs in target system).
195 *
196 * @todo Define visibility
197 */
198 public $relStaticTables = array();
199
200 /**
201 * Exclude map. Keys are table:uid pairs and if set, records are not added to the export.
202 *
203 * @todo Define visibility
204 */
205 public $excludeMap = array();
206
207 /**
208 * Soft Reference Token ID modes.
209 *
210 * @todo Define visibility
211 */
212 public $softrefCfg = array();
213
214 /**
215 * Listing extension dependencies.
216 *
217 * @todo Define visibility
218 */
219 public $extensionDependencies = array();
220
221 /**
222 * Set by user: If set, compression in t3d files is disabled
223 *
224 * @todo Define visibility
225 */
226 public $dontCompress = 0;
227
228 /**
229 * Boolean, if set, HTML file resources are included.
230 *
231 * @todo Define visibility
232 */
233 public $includeExtFileResources = 0;
234
235 /**
236 * Files with external media (HTML/css style references inside)
237 *
238 * @todo Define visibility
239 */
240 public $extFileResourceExtensions = 'html,htm,css';
241
242 /**
243 * After records are written this array is filled with [table][original_uid] = [new_uid]
244 *
245 * @todo Define visibility
246 */
247 public $import_mapId = array();
248
249 /**
250 * Keys are [tablename]:[new NEWxxx ids (or when updating it is uids)] while values are arrays with table/uid of the original record it is based on.
251 * By the array keys the new ids can be looked up inside tcemain
252 *
253 * @todo Define visibility
254 */
255 public $import_newId = array();
256
257 /**
258 * Page id map for page tree (import)
259 *
260 * @todo Define visibility
261 */
262 public $import_newId_pids = array();
263
264 /**
265 * Internal data accumulation for writing records during import
266 *
267 * @todo Define visibility
268 */
269 public $import_data = array();
270
271 /**
272 * Error log.
273 *
274 * @todo Define visibility
275 */
276 public $errorLog = array();
277
278 /**
279 * Cache for record paths
280 *
281 * @todo Define visibility
282 */
283 public $cache_getRecordPath = array();
284
285 /**
286 * Cache of checkPID values.
287 *
288 * @todo Define visibility
289 */
290 public $checkPID_cache = array();
291
292 /**
293 * Set internally if the gzcompress function exists
294 *
295 * @todo Define visibility
296 */
297 public $compress = 0;
298
299 /**
300 * Internal import/export memory
301 *
302 * @todo Define visibility
303 */
304 public $dat = array();
305
306 /**
307 * File processing object
308 *
309 * @var \TYPO3\CMS\Core\Utility\File\ExtendedFileUtility
310 * @todo Define visibility
311 */
312 public $fileProcObj = '';
313
314 /**
315 * Keys are [recordname], values are an array of fields to be included
316 * in the export
317 *
318 * @var array
319 */
320 protected $recordTypesIncludeFields = array();
321
322 /**
323 * Default array of fields to be included in the export
324 *
325 * @var array
326 */
327 protected $defaultRecordIncludeFields = array('uid', 'pid');
328
329 /**
330 * Array of current registered storage objects
331 *
332 * @var array
333 */
334 protected $storageObjects = array();
335
336 /**
337 * Is set, if the import file has a TYPO3 version below 6.0
338 *
339 * @var bool
340 */
341 protected $legacyImport = FALSE;
342
343 /**
344 * @var \TYPO3\CMS\Core\Resource\Folder
345 */
346 protected $legacyImportFolder = NULL;
347
348 /**
349 * Related to the default storage root
350 *
351 * @var string
352 */
353 protected $legacyImportTargetPath = '_imported/';
354
355 /**
356 * Table fields to migrate
357 *
358 * @var array
359 */
360 protected $legacyImportMigrationTables = array(
361 'tt_content' => array(
362 'image' => array(
363 'titleTexts' => 'titleText',
364 'description' => 'imagecaption',
365 'links' => 'image_link',
366 'alternativeTexts' => 'altText'
367 ),
368 'media' => array(
369 'description' => 'imagecaption',
370 )
371 ),
372 'pages' => array(
373 'media' => array()
374 ),
375 'pages_language_overlay' => array(
376 'media' => array()
377 )
378 );
379
380
381 /**
382 * Records to be migrated after all
383 * Multidimensional array [table][uid][field] = array([related sys_file_reference uids])
384 *
385 * @var array
386 */
387 protected $legacyImportMigrationRecords = array();
388
389
390 /**************************
391 * Initialize
392 *************************/
393
394 /**
395 * Init the object, both import and export
396 *
397 * @param boolean $dontCompress If set, compression in t3d files is disabled
398 * @param string $mode Mode of usage, either "import" or "export
399 * @return void
400 * @todo Define visibility
401 */
402 public function init($dontCompress = 0, $mode = '') {
403 $this->compress = function_exists('gzcompress');
404 $this->dontCompress = $dontCompress;
405 $this->mode = $mode;
406 $this->fileadminFolderName = !empty($GLOBALS['TYPO3_CONF_VARS']['BE']['fileadminDir']) ? rtrim($GLOBALS['TYPO3_CONF_VARS']['BE']['fileadminDir'], '/') : 'fileadmin';
407 }
408
409 /**************************
410 * Export / Init + Meta Data
411 *************************/
412
413 /**
414 * Set header basics
415 *
416 * @return void
417 * @todo Define visibility
418 */
419 public function setHeaderBasics() {
420 // Initializing:
421 if (is_array($this->softrefCfg)) {
422 foreach ($this->softrefCfg as $key => $value) {
423 if (!strlen($value['mode'])) {
424 unset($this->softrefCfg[$key]);
425 }
426 }
427 }
428 // Setting in header memory:
429 // Version of file format
430 $this->dat['header']['XMLversion'] = '1.0';
431 // Initialize meta data array (to put it in top of file)
432 $this->dat['header']['meta'] = array();
433 // Add list of tables to consider static
434 $this->dat['header']['relStaticTables'] = $this->relStaticTables;
435 // The list of excluded records
436 $this->dat['header']['excludeMap'] = $this->excludeMap;
437 // Soft Reference mode for elements
438 $this->dat['header']['softrefCfg'] = $this->softrefCfg;
439 // List of extensions the import depends on.
440 $this->dat['header']['extensionDependencies'] = $this->extensionDependencies;
441 }
442
443 /**
444 * Set charset
445 *
446 * @param string $charset Charset for the content in the export. During import the character set will be converted if the target system uses another charset.
447 * @return void
448 * @todo Define visibility
449 */
450 public function setCharset($charset) {
451 $this->dat['header']['charset'] = $charset;
452 }
453
454 /**
455 * Sets meta data
456 *
457 * @param string $title Title of the export
458 * @param string $description Description of the export
459 * @param string $notes Notes about the contents
460 * @param string $packager_username Backend Username of the packager (the guy making the export)
461 * @param string $packager_name Real name of the packager
462 * @param string $packager_email Email of the packager
463 * @return void
464 * @todo Define visibility
465 */
466 public function setMetaData($title, $description, $notes, $packager_username, $packager_name, $packager_email) {
467 $this->dat['header']['meta'] = array(
468 'title' => $title,
469 'description' => $description,
470 'notes' => $notes,
471 'packager_username' => $packager_username,
472 'packager_name' => $packager_name,
473 'packager_email' => $packager_email,
474 'TYPO3_version' => TYPO3_version,
475 'created' => strftime('%A %e. %B %Y', $GLOBALS['EXEC_TIME'])
476 );
477 }
478
479 /**
480 * Sets a thumbnail image to the exported file
481 *
482 * @param string $imgFilepath Filename reference, gif, jpg, png. Absolute path.
483 * @return void
484 * @todo Define visibility
485 */
486 public function addThumbnail($imgFilepath) {
487 if (@is_file($imgFilepath)) {
488 $imgInfo = @getimagesize($imgFilepath);
489 if (is_array($imgInfo)) {
490 $fileContent = GeneralUtility::getUrl($imgFilepath);
491 $this->dat['header']['thumbnail'] = array(
492 'imgInfo' => $imgInfo,
493 'content' => $fileContent,
494 'filesize' => strlen($fileContent),
495 'filemtime' => filemtime($imgFilepath),
496 'filename' => PathUtility::basename($imgFilepath)
497 );
498 }
499 }
500 }
501
502 /**************************
503 * Export / Init Page tree
504 *************************/
505
506 /**
507 * Sets the page-tree array in the export header and returns the array in a flattened version
508 *
509 * @param array $idH Hierarchy of ids, the page tree: array([uid] => array("uid" => [uid], "subrow" => array(.....)), [uid] => ....)
510 * @return array The hierarchical page tree converted to a one-dimensional list of pages
511 * @todo Define visibility
512 */
513 public function setPageTree($idH) {
514 $this->dat['header']['pagetree'] = $this->unsetExcludedSections($idH);
515 return $this->flatInversePageTree($this->dat['header']['pagetree']);
516 }
517
518 /**
519 * Removes entries in the page tree which are found in ->excludeMap[]
520 *
521 * @param array $idH Page uid hierarchy
522 * @return array Modified input array
523 * @access private
524 * @see setPageTree()
525 * @todo Define visibility
526 */
527 public function unsetExcludedSections($idH) {
528 if (is_array($idH)) {
529 foreach ($idH as $k => $v) {
530 if ($this->excludeMap['pages:' . $idH[$k]['uid']]) {
531 unset($idH[$k]);
532 } elseif (is_array($idH[$k]['subrow'])) {
533 $idH[$k]['subrow'] = $this->unsetExcludedSections($idH[$k]['subrow']);
534 }
535 }
536 }
537 return $idH;
538 }
539
540 /**
541 * Recursively flattening the idH array (for setPageTree() function)
542 *
543 * @param array $idH Page uid hierarchy
544 * @param array $a Accumulation array of pages (internal, don't set from outside)
545 * @return array Array with uid-uid pairs for all pages in the page tree.
546 * @see flatInversePageTree_pid()
547 * @todo Define visibility
548 */
549 public function flatInversePageTree($idH, $a = array()) {
550 if (is_array($idH)) {
551 $idH = array_reverse($idH);
552 foreach ($idH as $k => $v) {
553 $a[$v['uid']] = $v['uid'];
554 if (is_array($v['subrow'])) {
555 $a = $this->flatInversePageTree($v['subrow'], $a);
556 }
557 }
558 }
559 return $a;
560 }
561
562 /**
563 * Recursively flattening the idH array (for setPageTree() function), setting PIDs as values
564 *
565 * @param array $idH Page uid hierarchy
566 * @param array $a Accumulation array of pages (internal, don't set from outside)
567 * @param integer $pid PID value (internal)
568 * @return array Array with uid-pid pairs for all pages in the page tree.
569 * @see flatInversePageTree()
570 * @todo Define visibility
571 */
572 public function flatInversePageTree_pid($idH, $a = array(), $pid = -1) {
573 if (is_array($idH)) {
574 $idH = array_reverse($idH);
575 foreach ($idH as $k => $v) {
576 $a[$v['uid']] = $pid;
577 if (is_array($v['subrow'])) {
578 $a = $this->flatInversePageTree_pid($v['subrow'], $a, $v['uid']);
579 }
580 }
581 }
582 return $a;
583 }
584
585 /**************************
586 * Export
587 *************************/
588
589 /**
590 * Sets the fields of record types to be included in the export
591 *
592 * @param array $recordTypesIncludeFields Keys are [recordname], values are an array of fields to be included in the export
593 * @throws \TYPO3\CMS\Core\Exception if an array value is not type of array
594 * @return void
595 */
596 public function setRecordTypesIncludeFields(array $recordTypesIncludeFields) {
597 foreach ($recordTypesIncludeFields as $table => $fields) {
598 if (!is_array($fields)) {
599 throw new \TYPO3\CMS\Core\Exception('The include fields for record type ' . htmlspecialchars($table) . ' are not defined by an array.', 1391440658);
600 }
601 $this->setRecordTypeIncludeFields($table, $fields);
602 }
603 }
604
605 /**
606 * Sets the fields of a record type to be included in the export
607 *
608 * @param string $table The record type
609 * @param array $fields The fields to be included
610 * @return void
611 */
612 public function setRecordTypeIncludeFields($table, array $fields) {
613 $this->recordTypesIncludeFields[$table] = $fields;
614 }
615
616 /**
617 * Adds the record $row from $table.
618 * No checking for relations done here. Pure data.
619 *
620 * @param string $table Table name
621 * @param array $row Record row.
622 * @param integer $relationLevel (Internal) if the record is added as a relation, this is set to the "level" it was on.
623 * @return void
624 * @todo Define visibility
625 */
626 public function export_addRecord($table, $row, $relationLevel = 0) {
627 BackendUtility::workspaceOL($table, $row);
628 if ((string)$table !== '' && is_array($row) && $row['uid'] > 0 && !$this->excludeMap[($table . ':' . $row['uid'])]) {
629 if ($this->checkPID($table === 'pages' ? $row['uid'] : $row['pid'])) {
630 if (!isset($this->dat['records'][($table . ':' . $row['uid'])])) {
631 // Prepare header info:
632 $row = $this->filterRecordFields($table, $row);
633 $headerInfo = array();
634 $headerInfo['uid'] = $row['uid'];
635 $headerInfo['pid'] = $row['pid'];
636 $headerInfo['title'] = GeneralUtility::fixed_lgd_cs(BackendUtility::getRecordTitle($table, $row), 40);
637 $headerInfo['size'] = strlen(serialize($row));
638 if ($relationLevel) {
639 $headerInfo['relationLevel'] = $relationLevel;
640 }
641 // If record content is not too large in size, set the header content and add the rest:
642 if ($headerInfo['size'] < $this->maxRecordSize) {
643 // Set the header summary:
644 $this->dat['header']['records'][$table][$row['uid']] = $headerInfo;
645 // Create entry in the PID lookup:
646 $this->dat['header']['pid_lookup'][$row['pid']][$table][$row['uid']] = 1;
647 // Initialize reference index object:
648 $refIndexObj = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\Database\\ReferenceIndex');
649 // Yes to workspace overlays for exporting....
650 $refIndexObj->WSOL = TRUE;
651 $relations = $refIndexObj->getRelations($table, $row);
652 $relations = $this->fixFileIDsInRelations($relations);
653 // Data:
654 $this->dat['records'][$table . ':' . $row['uid']] = array();
655 $this->dat['records'][$table . ':' . $row['uid']]['data'] = $row;
656 $this->dat['records'][$table . ':' . $row['uid']]['rels'] = $relations;
657 $this->errorLog = array_merge($this->errorLog, $refIndexObj->errorLog);
658 // Merge error logs.
659 // Add information about the relations in the record in the header:
660 $this->dat['header']['records'][$table][$row['uid']]['rels'] = $this->flatDBrels($this->dat['records'][$table . ':' . $row['uid']]['rels']);
661 // Add information about the softrefs to header:
662 $this->dat['header']['records'][$table][$row['uid']]['softrefs'] = $this->flatSoftRefs($this->dat['records'][$table . ':' . $row['uid']]['rels']);
663 } else {
664 $this->error('Record ' . $table . ':' . $row['uid'] . ' was larger than maxRecordSize (' . GeneralUtility::formatSize($this->maxRecordSize) . ')');
665 }
666 } else {
667 $this->error('Record ' . $table . ':' . $row['uid'] . ' already added.');
668 }
669 } else {
670 $this->error('Record ' . $table . ':' . $row['uid'] . ' was outside your DB mounts!');
671 }
672 }
673 }
674
675 /**
676 * This changes the file reference ID from a hash based on the absolute file path
677 * (coming from ReferenceIndex) to a hash based on the relative file path.
678 *
679 * @param array $relations
680 * @return array
681 */
682 protected function fixFileIDsInRelations(array $relations) {
683 foreach ($relations as $field => $relation) {
684 if (isset($relation['type']) && $relation['type'] === 'file') {
685 foreach ($relation['newValueFiles'] as $key => $fileRelationData) {
686 $absoluteFilePath = $fileRelationData['ID_absFile'];
687 if (GeneralUtility::isFirstPartOfStr($absoluteFilePath, PATH_site)) {
688 $relatedFilePath = \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($absoluteFilePath);
689 $relations[$field]['newValueFiles'][$key]['ID'] = md5($relatedFilePath);
690 }
691 }
692 }
693 if ($relation['type'] == 'flex') {
694 if (is_array($relation['flexFormRels']['file'])) {
695 foreach ($relation['flexFormRels']['file'] as $key => $subList) {
696 foreach ($subList as $subKey => $fileRelationData) {
697 $absoluteFilePath = $fileRelationData['ID_absFile'];
698 if (GeneralUtility::isFirstPartOfStr($absoluteFilePath, PATH_site)) {
699 $relatedFilePath = \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($absoluteFilePath);
700 $relations[$field]['flexFormRels']['file'][$key][$subKey]['ID'] = md5($relatedFilePath);
701 }
702 }
703 }
704 }
705 }
706 }
707 return $relations;
708 }
709
710
711
712 /**
713 * This analyses the existing added records, finds all database relations to records and adds these records to the export file.
714 * This function can be called repeatedly until it returns an empty array. In principle it should not allow to infinite recursivity, but you better set a limit...
715 * Call this BEFORE the ext_addFilesFromRelations (so files from added relations are also included of course)
716 *
717 * @param integer $relationLevel Recursion level
718 * @return array overview of relations found and added: Keys [table]:[uid], values array with table and id
719 * @see export_addFilesFromRelations()
720 * @todo Define visibility
721 */
722 public function export_addDBRelations($relationLevel = 0) {
723 // Initialize:
724 $addR = array();
725 // Traverse all "rels" registered for "records"
726 if (is_array($this->dat['records'])) {
727 foreach ($this->dat['records'] as $k => $value) {
728 if (is_array($this->dat['records'][$k])) {
729 foreach ($this->dat['records'][$k]['rels'] as $fieldname => $vR) {
730 // For all DB types of relations:
731 if ($vR['type'] == 'db') {
732 foreach ($vR['itemArray'] as $fI) {
733 $this->export_addDBRelations_registerRelation($fI, $addR);
734 }
735 }
736 // For all flex/db types of relations:
737 if ($vR['type'] == 'flex') {
738 // DB relations in flex form fields:
739 if (is_array($vR['flexFormRels']['db'])) {
740 foreach ($vR['flexFormRels']['db'] as $subList) {
741 foreach ($subList as $fI) {
742 $this->export_addDBRelations_registerRelation($fI, $addR);
743 }
744 }
745 }
746 // DB oriented soft references in flex form fields:
747 if (is_array($vR['flexFormRels']['softrefs'])) {
748 foreach ($vR['flexFormRels']['softrefs'] as $subList) {
749 foreach ($subList['keys'] as $spKey => $elements) {
750 foreach ($elements as $el) {
751 if ($el['subst']['type'] === 'db' && $this->includeSoftref($el['subst']['tokenID'])) {
752 list($tempTable, $tempUid) = explode(':', $el['subst']['recordRef']);
753 $fI = array(
754 'table' => $tempTable,
755 'id' => $tempUid
756 );
757 $this->export_addDBRelations_registerRelation($fI, $addR, $el['subst']['tokenID']);
758 }
759 }
760 }
761 }
762 }
763 }
764 // In any case, if there are soft refs:
765 if (is_array($vR['softrefs']['keys'])) {
766 foreach ($vR['softrefs']['keys'] as $spKey => $elements) {
767 foreach ($elements as $el) {
768 if ($el['subst']['type'] === 'db' && $this->includeSoftref($el['subst']['tokenID'])) {
769 list($tempTable, $tempUid) = explode(':', $el['subst']['recordRef']);
770 $fI = array(
771 'table' => $tempTable,
772 'id' => $tempUid
773 );
774 $this->export_addDBRelations_registerRelation($fI, $addR, $el['subst']['tokenID']);
775 }
776 }
777 }
778 }
779 }
780 }
781 }
782 } else {
783 $this->error('There were no records available.');
784 }
785 // Now, if there were new records to add, do so:
786 if (count($addR)) {
787 foreach ($addR as $fI) {
788 // Get and set record:
789 $row = BackendUtility::getRecord($fI['table'], $fI['id']);
790 if (is_array($row)) {
791 $this->export_addRecord($fI['table'], $row, $relationLevel + 1);
792 }
793 // Set status message
794 // Relation pointers always larger than zero except certain "select" types with negative values pointing to uids - but that is not supported here.
795 if ($fI['id'] > 0) {
796 $rId = $fI['table'] . ':' . $fI['id'];
797 if (!isset($this->dat['records'][$rId])) {
798 $this->dat['records'][$rId] = 'NOT_FOUND';
799 $this->error('Relation record ' . $rId . ' was not found!');
800 }
801 }
802 }
803 }
804 // Return overview of relations found and added
805 return $addR;
806 }
807
808 /**
809 * Helper function for export_addDBRelations()
810 *
811 * @param array $fI Array with table/id keys to add
812 * @param array $addR Add array, passed by reference to be modified
813 * @param string $tokenID Softref Token ID, if applicable.
814 * @return void
815 * @see export_addDBRelations()
816 * @todo Define visibility
817 */
818 public function export_addDBRelations_registerRelation($fI, &$addR, $tokenID = '') {
819 $rId = $fI['table'] . ':' . $fI['id'];
820 if (isset($GLOBALS['TCA'][$fI['table']]) && !$this->isTableStatic($fI['table']) && !$this->isExcluded($fI['table'], $fI['id']) && (!$tokenID || $this->includeSoftref($tokenID)) && $this->inclRelation($fI['table'])) {
821 if (!isset($this->dat['records'][$rId])) {
822 // Set this record to be included since it is not already.
823 $addR[$rId] = $fI;
824 }
825 }
826 }
827
828 /**
829 * This adds all files in relations.
830 * Call this method AFTER adding all records including relations.
831 *
832 * @return void
833 * @see export_addDBRelations()
834 * @todo Define visibility
835 */
836 public function export_addFilesFromRelations() {
837 // Traverse all "rels" registered for "records"
838 if (is_array($this->dat['records'])) {
839 foreach ($this->dat['records'] as $k => $value) {
840 if (isset($this->dat['records'][$k]['rels']) && is_array($this->dat['records'][$k]['rels'])) {
841 foreach ($this->dat['records'][$k]['rels'] as $fieldname => $vR) {
842 // For all file type relations:
843 if ($vR['type'] == 'file') {
844 foreach ($vR['newValueFiles'] as $key => $fI) {
845 $this->export_addFile($fI, $k, $fieldname);
846 // Remove the absolute reference to the file so it doesn't expose absolute paths from source server:
847 unset($this->dat['records'][$k]['rels'][$fieldname]['newValueFiles'][$key]['ID_absFile']);
848 }
849 }
850 // For all flex type relations:
851 if ($vR['type'] == 'flex') {
852 if (is_array($vR['flexFormRels']['file'])) {
853 foreach ($vR['flexFormRels']['file'] as $key => $subList) {
854 foreach ($subList as $subKey => $fI) {
855 $this->export_addFile($fI, $k, $fieldname);
856 // Remove the absolute reference to the file so it doesn't expose absolute paths from source server:
857 unset($this->dat['records'][$k]['rels'][$fieldname]['flexFormRels']['file'][$key][$subKey]['ID_absFile']);
858 }
859 }
860 }
861 // DB oriented soft references in flex form fields:
862 if (is_array($vR['flexFormRels']['softrefs'])) {
863 foreach ($vR['flexFormRels']['softrefs'] as $key => $subList) {
864 foreach ($subList['keys'] as $spKey => $elements) {
865 foreach ($elements as $subKey => $el) {
866 if ($el['subst']['type'] === 'file' && $this->includeSoftref($el['subst']['tokenID'])) {
867 // Create abs path and ID for file:
868 $ID_absFile = GeneralUtility::getFileAbsFileName(PATH_site . $el['subst']['relFileName']);
869 $ID = md5($el['subst']['relFileName']);
870 if ($ID_absFile) {
871 if (!$this->dat['files'][$ID]) {
872 $fI = array(
873 'filename' => PathUtility::basename($ID_absFile),
874 'ID_absFile' => $ID_absFile,
875 'ID' => $ID,
876 'relFileName' => $el['subst']['relFileName']
877 );
878 $this->export_addFile($fI, '_SOFTREF_');
879 }
880 $this->dat['records'][$k]['rels'][$fieldname]['flexFormRels']['softrefs'][$key]['keys'][$spKey][$subKey]['file_ID'] = $ID;
881 }
882 }
883 }
884 }
885 }
886 }
887 }
888 // In any case, if there are soft refs:
889 if (is_array($vR['softrefs']['keys'])) {
890 foreach ($vR['softrefs']['keys'] as $spKey => $elements) {
891 foreach ($elements as $subKey => $el) {
892 if ($el['subst']['type'] === 'file' && $this->includeSoftref($el['subst']['tokenID'])) {
893 // Create abs path and ID for file:
894 $ID_absFile = GeneralUtility::getFileAbsFileName(PATH_site . $el['subst']['relFileName']);
895 $ID = md5($el['subst']['relFileName']);
896 if ($ID_absFile) {
897 if (!$this->dat['files'][$ID]) {
898 $fI = array(
899 'filename' => PathUtility::basename($ID_absFile),
900 'ID_absFile' => $ID_absFile,
901 'ID' => $ID,
902 'relFileName' => $el['subst']['relFileName']
903 );
904 $this->export_addFile($fI, '_SOFTREF_');
905 }
906 $this->dat['records'][$k]['rels'][$fieldname]['softrefs']['keys'][$spKey][$subKey]['file_ID'] = $ID;
907 }
908 }
909 }
910 }
911 }
912 }
913 }
914 }
915 } else {
916 $this->error('There were no records available.');
917 }
918 }
919
920 /**
921 * This adds all files from sys_file records
922 *
923 * @return void
924 */
925 public function export_addFilesFromSysFilesRecords() {
926 if (!isset($this->dat['header']['records']['sys_file']) || !is_array($this->dat['header']['records']['sys_file'])) {
927 return;
928 }
929 foreach (array_keys($this->dat['header']['records']['sys_file']) as $sysFileUid) {
930 $recordData = $this->dat['records']['sys_file:' . $sysFileUid]['data'];
931 $file = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->createFileObject($recordData);
932 $this->export_addSysFile($file);
933 }
934 }
935
936 /**
937 * Adds a files content from a sys file record to the export memory
938 *
939 * @param \TYPO3\CMS\Core\Resource\File $file
940 * @return void
941 */
942 public function export_addSysFile(\TYPO3\CMS\Core\Resource\File $file) {
943 if ($file->getProperty('size') >= $this->maxFileSize) {
944 $this->error('File ' . $file->getPublicUrl() . ' was larger (' . GeneralUtility::formatSize($file->getProperty('size')) . ') than the maxFileSize (' . GeneralUtility::formatSize($this->maxFileSize) . ')! Skipping.');
945 return;
946 }
947 try {
948 $fileContent = $file->getContents();
949 } catch (\TYPO3\CMS\Core\Resource\Exception\InsufficientFileAccessPermissionsException $e) {
950 $this->error('File ' . $file->getPublicUrl() . ': ' . $e->getMessage());
951 return;
952 } catch (\TYPO3\CMS\Core\Resource\Exception\IllegalFileExtensionException $e) {
953 $this->error('File ' . $file->getPublicUrl() . ': ' . $e->getMessage());
954 return;
955 }
956 $fileUid = $file->getUid();
957 $fileInfo = $file->getStorage()->getFileInfo($file);
958 // we sadly have to cast it to string here, because the size property is also returning a string
959 $fileSize = (string)$fileInfo['size'];
960 if ($fileSize !== $file->getProperty('size')) {
961 $this->error('File size of ' . $file->getCombinedIdentifier() . ' is not up-to-date in index! File added with current size.');
962 $this->dat['records']['sys_file:' . $fileUid]['data']['size'] = $fileSize;
963 }
964 $fileSha1 = $file->getStorage()->hashFile($file, 'sha1');
965 if ($fileSha1 !== $file->getProperty('sha1')) {
966 $this->error('File sha1 hash of ' . $file->getCombinedIdentifier() . ' is not up-to-date in index! File added on current sha1.');
967 $this->dat['records']['sys_file:' . $fileUid]['data']['sha1'] = $fileSha1;
968 }
969
970 $fileRec = array();
971 $fileRec['filesize'] = $fileSize;
972 $fileRec['filename'] = $file->getProperty('name');
973 $fileRec['filemtime'] = $file->getProperty('modification_date');
974
975 // build unique id based on the storage and the file identifier
976 $fileId = md5($file->getStorage()->getUid() . ':' . $file->getProperty('identifier_hash'));
977
978 // Setting this data in the header
979 $this->dat['header']['files_fal'][$fileId] = $fileRec;
980
981 // ... and finally add the heavy stuff:
982 $fileRec['content'] = $fileContent;
983 $fileRec['content_sha1'] = $fileSha1;
984
985 $this->dat['files_fal'][$fileId] = $fileRec;
986 }
987
988
989 /**
990 * Adds a files content to the export memory
991 *
992 * @param array $fI File information with three keys: "filename" = filename without path, "ID_absFile" = absolute filepath to the file (including the filename), "ID" = md5 hash of "ID_absFile". "relFileName" is optional for files attached to records, but mandatory for soft referenced files (since the relFileName determines where such a file should be stored!)
993 * @param string $recordRef If the file is related to a record, this is the id on the form [table]:[id]. Information purposes only.
994 * @param string $fieldname If the file is related to a record, this is the field name it was related to. Information purposes only.
995 * @return void
996 * @todo Define visibility
997 */
998 public function export_addFile($fI, $recordRef = '', $fieldname = '') {
999 if (@is_file($fI['ID_absFile'])) {
1000 if (filesize($fI['ID_absFile']) < $this->maxFileSize) {
1001 $fileInfo = stat($fI['ID_absFile']);
1002 $fileRec = array();
1003 $fileRec['filesize'] = $fileInfo['size'];
1004 $fileRec['filename'] = PathUtility::basename($fI['ID_absFile']);
1005 $fileRec['filemtime'] = $fileInfo['mtime'];
1006 //for internal type file_reference
1007 $fileRec['relFileRef'] = \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($fI['ID_absFile']);
1008 if ($recordRef) {
1009 $fileRec['record_ref'] = $recordRef . '/' . $fieldname;
1010 }
1011 if ($fI['relFileName']) {
1012 $fileRec['relFileName'] = $fI['relFileName'];
1013 }
1014 // Setting this data in the header
1015 $this->dat['header']['files'][$fI['ID']] = $fileRec;
1016 // ... and for the recordlisting, why not let us know WHICH relations there was...
1017 if ($recordRef && $recordRef !== '_SOFTREF_') {
1018 $refParts = explode(':', $recordRef, 2);
1019 if (!is_array($this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'])) {
1020 $this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'] = array();
1021 }
1022 $this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'][] = $fI['ID'];
1023 }
1024 // ... and finally add the heavy stuff:
1025 $fileRec['content'] = GeneralUtility::getUrl($fI['ID_absFile']);
1026 $fileRec['content_md5'] = md5($fileRec['content']);
1027 $this->dat['files'][$fI['ID']] = $fileRec;
1028 // For soft references, do further processing:
1029 if ($recordRef === '_SOFTREF_') {
1030 // RTE files?
1031 if ($RTEoriginal = $this->getRTEoriginalFilename(PathUtility::basename($fI['ID_absFile']))) {
1032 $RTEoriginal_absPath = PathUtility::dirname($fI['ID_absFile']) . '/' . $RTEoriginal;
1033 if (@is_file($RTEoriginal_absPath)) {
1034 $RTEoriginal_ID = md5($RTEoriginal_absPath);
1035 $fileInfo = stat($RTEoriginal_absPath);
1036 $fileRec = array();
1037 $fileRec['filesize'] = $fileInfo['size'];
1038 $fileRec['filename'] = PathUtility::basename($RTEoriginal_absPath);
1039 $fileRec['filemtime'] = $fileInfo['mtime'];
1040 $fileRec['record_ref'] = '_RTE_COPY_ID:' . $fI['ID'];
1041 $this->dat['header']['files'][$fI['ID']]['RTE_ORIG_ID'] = $RTEoriginal_ID;
1042 // Setting this data in the header
1043 $this->dat['header']['files'][$RTEoriginal_ID] = $fileRec;
1044 // ... and finally add the heavy stuff:
1045 $fileRec['content'] = GeneralUtility::getUrl($RTEoriginal_absPath);
1046 $fileRec['content_md5'] = md5($fileRec['content']);
1047 $this->dat['files'][$RTEoriginal_ID] = $fileRec;
1048 } else {
1049 $this->error('RTE original file "' . \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($RTEoriginal_absPath) . '" was not found!');
1050 }
1051 }
1052 // Files with external media?
1053 // This is only done with files grabbed by a softreference parser since it is deemed improbable that hard-referenced files should undergo this treatment.
1054 $html_fI = pathinfo(PathUtility::basename($fI['ID_absFile']));
1055 if ($this->includeExtFileResources && GeneralUtility::inList($this->extFileResourceExtensions, strtolower($html_fI['extension']))) {
1056 $uniquePrefix = '###' . md5($GLOBALS['EXEC_TIME']) . '###';
1057 if (strtolower($html_fI['extension']) === 'css') {
1058 $prefixedMedias = explode($uniquePrefix, preg_replace('/(url[[:space:]]*\\([[:space:]]*["\']?)([^"\')]*)(["\']?[[:space:]]*\\))/i', '\\1' . $uniquePrefix . '\\2' . $uniquePrefix . '\\3', $fileRec['content']));
1059 } else {
1060 // html, htm:
1061 $htmlParser = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\Html\\HtmlParser');
1062 $prefixedMedias = explode($uniquePrefix, $htmlParser->prefixResourcePath($uniquePrefix, $fileRec['content'], array(), $uniquePrefix));
1063 }
1064 $htmlResourceCaptured = FALSE;
1065 foreach ($prefixedMedias as $k => $v) {
1066 if ($k % 2) {
1067 $EXTres_absPath = GeneralUtility::resolveBackPath(PathUtility::dirname($fI['ID_absFile']) . '/' . $v);
1068 $EXTres_absPath = GeneralUtility::getFileAbsFileName($EXTres_absPath);
1069 if ($EXTres_absPath && GeneralUtility::isFirstPartOfStr($EXTres_absPath, PATH_site . $this->fileadminFolderName . '/') && @is_file($EXTres_absPath)) {
1070 $htmlResourceCaptured = TRUE;
1071 $EXTres_ID = md5($EXTres_absPath);
1072 $this->dat['header']['files'][$fI['ID']]['EXT_RES_ID'][] = $EXTres_ID;
1073 $prefixedMedias[$k] = '{EXT_RES_ID:' . $EXTres_ID . '}';
1074 // Add file to memory if it is not set already:
1075 if (!isset($this->dat['header']['files'][$EXTres_ID])) {
1076 $fileInfo = stat($EXTres_absPath);
1077 $fileRec = array();
1078 $fileRec['filesize'] = $fileInfo['size'];
1079 $fileRec['filename'] = PathUtility::basename($EXTres_absPath);
1080 $fileRec['filemtime'] = $fileInfo['mtime'];
1081 $fileRec['record_ref'] = '_EXT_PARENT_:' . $fI['ID'];
1082 // Media relative to the HTML file.
1083 $fileRec['parentRelFileName'] = $v;
1084 // Setting this data in the header
1085 $this->dat['header']['files'][$EXTres_ID] = $fileRec;
1086 // ... and finally add the heavy stuff:
1087 $fileRec['content'] = GeneralUtility::getUrl($EXTres_absPath);
1088 $fileRec['content_md5'] = md5($fileRec['content']);
1089 $this->dat['files'][$EXTres_ID] = $fileRec;
1090 }
1091 }
1092 }
1093 }
1094 if ($htmlResourceCaptured) {
1095 $this->dat['files'][$fI['ID']]['tokenizedContent'] = implode('', $prefixedMedias);
1096 }
1097 }
1098 }
1099 } else {
1100 $this->error($fI['ID_absFile'] . ' was larger (' . GeneralUtility::formatSize(filesize($fI['ID_absFile'])) . ') than the maxFileSize (' . GeneralUtility::formatSize($this->maxFileSize) . ')! Skipping.');
1101 }
1102 } else {
1103 $this->error($fI['ID_absFile'] . ' was not a file! Skipping.');
1104 }
1105 }
1106
1107 /**
1108 * DB relations flattend to 1-dim array.
1109 * The list will be unique, no table/uid combination will appear twice.
1110 *
1111 * @param array $dbrels 2-dim Array of database relations organized by table key
1112 * @return array 1-dim array where entries are table:uid and keys are array with table/id
1113 * @todo Define visibility
1114 */
1115 public function flatDBrels($dbrels) {
1116 $list = array();
1117 foreach ($dbrels as $dat) {
1118 if ($dat['type'] == 'db') {
1119 foreach ($dat['itemArray'] as $i) {
1120 $list[$i['table'] . ':' . $i['id']] = $i;
1121 }
1122 }
1123 if ($dat['type'] == 'flex' && is_array($dat['flexFormRels']['db'])) {
1124 foreach ($dat['flexFormRels']['db'] as $subList) {
1125 foreach ($subList as $i) {
1126 $list[$i['table'] . ':' . $i['id']] = $i;
1127 }
1128 }
1129 }
1130 }
1131 return $list;
1132 }
1133
1134 /**
1135 * Soft References flattend to 1-dim array.
1136 *
1137 * @param array $dbrels 2-dim Array of database relations organized by table key
1138 * @return array 1-dim array where entries are arrays with properties of the soft link found and keys are a unique combination of field, spKey, structure path if applicable and token ID
1139 * @todo Define visibility
1140 */
1141 public function flatSoftRefs($dbrels) {
1142 $list = array();
1143 foreach ($dbrels as $field => $dat) {
1144 if (is_array($dat['softrefs']['keys'])) {
1145 foreach ($dat['softrefs']['keys'] as $spKey => $elements) {
1146 if (is_array($elements)) {
1147 foreach ($elements as $subKey => $el) {
1148 $lKey = $field . ':' . $spKey . ':' . $subKey;
1149 $list[$lKey] = array_merge(array('field' => $field, 'spKey' => $spKey), $el);
1150 // Add file_ID key to header - slightly "risky" way of doing this because if the calculation changes for the same value in $this->records[...] this will not work anymore!
1151 if ($el['subst'] && $el['subst']['relFileName']) {
1152 $list[$lKey]['file_ID'] = md5(PATH_site . $el['subst']['relFileName']);
1153 }
1154 }
1155 }
1156 }
1157 }
1158 if ($dat['type'] == 'flex' && is_array($dat['flexFormRels']['softrefs'])) {
1159 foreach ($dat['flexFormRels']['softrefs'] as $structurePath => $subSoftrefs) {
1160 if (is_array($subSoftrefs['keys'])) {
1161 foreach ($subSoftrefs['keys'] as $spKey => $elements) {
1162 foreach ($elements as $subKey => $el) {
1163 $lKey = $field . ':' . $structurePath . ':' . $spKey . ':' . $subKey;
1164 $list[$lKey] = array_merge(array('field' => $field, 'spKey' => $spKey, 'structurePath' => $structurePath), $el);
1165 // Add file_ID key to header - slightly "risky" way of doing this because if the calculation changes for the same value in $this->records[...] this will not work anymore!
1166 if ($el['subst'] && $el['subst']['relFileName']) {
1167 $list[$lKey]['file_ID'] = md5(PATH_site . $el['subst']['relFileName']);
1168 }
1169 }
1170 }
1171 }
1172 }
1173 }
1174 }
1175 return $list;
1176 }
1177
1178 /**
1179 * If include fields for a specific record type are set, the data
1180 * are filtered out with fields are not included in the fields.
1181 *
1182 * @param string $table The record type to be filtered
1183 * @param array $row The data to be filtered
1184 * @return array The filtered record row
1185 */
1186 protected function filterRecordFields($table, array $row) {
1187 if (isset($this->recordTypesIncludeFields[$table])) {
1188 $includeFields = array_unique(array_merge(
1189 $this->recordTypesIncludeFields[$table],
1190 $this->defaultRecordIncludeFields
1191 ));
1192 $newRow = array();
1193 foreach ($row as $key => $value) {
1194 if (in_array($key, $includeFields)) {
1195 $newRow[$key] = $value;
1196 }
1197 }
1198 } else {
1199 $newRow = $row;
1200 }
1201 return $newRow;
1202 }
1203
1204
1205 /**************************
1206 * File Output
1207 *************************/
1208
1209 /**
1210 * This compiles and returns the data content for an exported file
1211 *
1212 * @param string $type Type of output; "xml" gives xml, otherwise serialized array, possibly compressed.
1213 * @return string The output file stream
1214 * @todo Define visibility
1215 */
1216 public function compileMemoryToFileContent($type = '') {
1217 if ($type == 'xml') {
1218 $out = $this->createXML();
1219 } else {
1220 $compress = $this->doOutputCompress();
1221 $out = '';
1222 // adding header:
1223 $out .= $this->addFilePart(serialize($this->dat['header']), $compress);
1224 // adding records:
1225 $out .= $this->addFilePart(serialize($this->dat['records']), $compress);
1226 // adding files:
1227 $out .= $this->addFilePart(serialize($this->dat['files']), $compress);
1228 // adding files_fal:
1229 $out .= $this->addFilePart(serialize($this->dat['files_fal']), $compress);
1230 }
1231 return $out;
1232 }
1233
1234 /**
1235 * Creates XML string from input array
1236 *
1237 * @return string XML content
1238 * @todo Define visibility
1239 */
1240 public function createXML() {
1241 // Options:
1242 $options = array(
1243 'alt_options' => array(
1244 '/header' => array(
1245 'disableTypeAttrib' => TRUE,
1246 'clearStackPath' => TRUE,
1247 'parentTagMap' => array(
1248 'files' => 'file',
1249 'files_fal' => 'file',
1250 'records' => 'table',
1251 'table' => 'rec',
1252 'rec:rels' => 'relations',
1253 'relations' => 'element',
1254 'filerefs' => 'file',
1255 'pid_lookup' => 'page_contents',
1256 'header:relStaticTables' => 'static_tables',
1257 'static_tables' => 'tablename',
1258 'excludeMap' => 'item',
1259 'softrefCfg' => 'softrefExportMode',
1260 'extensionDependencies' => 'extkey',
1261 'softrefs' => 'softref_element'
1262 ),
1263 'alt_options' => array(
1264 '/pagetree' => array(
1265 'disableTypeAttrib' => TRUE,
1266 'useIndexTagForNum' => 'node',
1267 'parentTagMap' => array(
1268 'node:subrow' => 'node'
1269 )
1270 ),
1271 '/pid_lookup/page_contents' => array(
1272 'disableTypeAttrib' => TRUE,
1273 'parentTagMap' => array(
1274 'page_contents' => 'table'
1275 ),
1276 'grandParentTagMap' => array(
1277 'page_contents/table' => 'item'
1278 )
1279 )
1280 )
1281 ),
1282 '/records' => array(
1283 'disableTypeAttrib' => TRUE,
1284 'parentTagMap' => array(
1285 'records' => 'tablerow',
1286 'tablerow:data' => 'fieldlist',
1287 'tablerow:rels' => 'related',
1288 'related' => 'field',
1289 'field:itemArray' => 'relations',
1290 'field:newValueFiles' => 'filerefs',
1291 'field:flexFormRels' => 'flexform',
1292 'relations' => 'element',
1293 'filerefs' => 'file',
1294 'flexform:db' => 'db_relations',
1295 'flexform:file' => 'file_relations',
1296 'flexform:softrefs' => 'softref_relations',
1297 'softref_relations' => 'structurePath',
1298 'db_relations' => 'path',
1299 'file_relations' => 'path',
1300 'path' => 'element',
1301 'keys' => 'softref_key',
1302 'softref_key' => 'softref_element'
1303 ),
1304 'alt_options' => array(
1305 '/records/tablerow/fieldlist' => array(
1306 'useIndexTagForAssoc' => 'field'
1307 )
1308 )
1309 ),
1310 '/files' => array(
1311 'disableTypeAttrib' => TRUE,
1312 'parentTagMap' => array(
1313 'files' => 'file'
1314 )
1315 ),
1316 '/files_fal' => array(
1317 'disableTypeAttrib' => TRUE,
1318 'parentTagMap' => array(
1319 'files_fal' => 'file'
1320 )
1321 )
1322 )
1323 );
1324 // Creating XML file from $outputArray:
1325 $charset = $this->dat['header']['charset'] ?: 'utf-8';
1326 $XML = '<?xml version="1.0" encoding="' . $charset . '" standalone="yes" ?>' . LF;
1327 $XML .= GeneralUtility::array2xml($this->dat, '', 0, 'T3RecordDocument', 0, $options);
1328 return $XML;
1329 }
1330
1331 /**
1332 * Returns TRUE if the output should be compressed.
1333 *
1334 * @return boolean TRUE if compression is possible AND requested.
1335 * @todo Define visibility
1336 */
1337 public function doOutputCompress() {
1338 return $this->compress && !$this->dontCompress;
1339 }
1340
1341 /**
1342 * Returns a content part for a filename being build.
1343 *
1344 * @param array $data Data to store in part
1345 * @param boolean $compress Compress file?
1346 * @return string Content stream.
1347 * @todo Define visibility
1348 */
1349 public function addFilePart($data, $compress = FALSE) {
1350 if ($compress) {
1351 $data = gzcompress($data);
1352 }
1353 return md5($data) . ':' . ($compress ? '1' : '0') . ':' . str_pad(strlen($data), 10, '0', STR_PAD_LEFT) . ':' . $data . ':';
1354 }
1355
1356 /***********************
1357 * Import
1358 ***********************/
1359
1360 /**
1361 * Initialize all settings for the import
1362 *
1363 * @return void
1364 */
1365 protected function initializeImport() {
1366 // Set this flag to indicate that an import is being/has been done.
1367 $this->doesImport = 1;
1368 // Initialize:
1369 // These vars MUST last for the whole section not being cleared. They are used by the method setRelations() which are called at the end of the import session.
1370 $this->import_mapId = array();
1371 $this->import_newId = array();
1372 $this->import_newId_pids = array();
1373 // Temporary files stack initialized:
1374 $this->unlinkFiles = array();
1375 $this->alternativeFileName = array();
1376 $this->alternativeFilePath = array();
1377
1378 $this->initializeStorageObjects();
1379 }
1380
1381 /**
1382 * Initialize the all present storage objects
1383 *
1384 * @return void
1385 */
1386 protected function initializeStorageObjects() {
1387 /** @var $storageRepository \TYPO3\CMS\Core\Resource\StorageRepository */
1388 $storageRepository = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\Resource\\StorageRepository');
1389 $this->storageObjects = $storageRepository->findAll();
1390 }
1391
1392
1393 /**
1394 * Imports the internal data array to $pid.
1395 *
1396 * @param integer $pid Page ID in which to import the content
1397 * @return void
1398 */
1399 public function importData($pid) {
1400
1401 $this->initializeImport();
1402
1403 // Write sys_file_storages first
1404 $this->writeSysFileStorageRecords();
1405 // Write sys_file records and write the binary file data
1406 $this->writeSysFileRecords();
1407 // Write records, first pages, then the rest
1408 // Fields with "hard" relations to database, files and flexform fields are kept empty during this run
1409 $this->writeRecords_pages($pid);
1410 $this->writeRecords_records($pid);
1411 // Finally all the file and DB record references must be fixed. This is done after all records have supposedly been written to database:
1412 // $this->import_mapId will indicate two things: 1) that a record WAS written to db and 2) that it has got a new id-number.
1413 $this->setRelations();
1414 // And when all DB relations are in place, we can fix file and DB relations in flexform fields (since data structures often depends on relations to a DS record):
1415 $this->setFlexFormRelations();
1416 // Unlink temporary files:
1417 $this->unlinkTempFiles();
1418 // Finally, traverse all records and process softreferences with substitution attributes.
1419 $this->processSoftReferences();
1420 // After all migrate records using sys_file_reference now
1421 if ($this->legacyImport) {
1422 $this->migrateLegacyImportRecords();
1423 }
1424 }
1425
1426 /**
1427 * Imports the sys_file_storage records from internal data array.
1428 *
1429 * @return void
1430 */
1431 protected function writeSysFileStorageRecords() {
1432 if (!isset($this->dat['header']['records']['sys_file_storage'])) {
1433 return;
1434 }
1435 $sysFileStorageUidsToBeResetToDefaultStorage = array();
1436 foreach (array_keys($this->dat['header']['records']['sys_file_storage']) as $sysFileStorageUid) {
1437 $storageRecord = $this->dat['records']['sys_file_storage:' . $sysFileStorageUid]['data'];
1438 // continue with Local, writable and online storage only
1439 if ($storageRecord['driver'] === 'Local' && $storageRecord['is_writable'] && $storageRecord['is_online']) {
1440 $useThisStorageUidInsteadOfTheOneInImport = 0;
1441 /** @var $localStorage \TYPO3\CMS\Core\Resource\ResourceStorage */
1442 foreach ($this->storageObjects as $localStorage) {
1443 // check the available storage for Local, writable and online ones
1444 if ($localStorage->getDriverType() === 'Local' && $localStorage->isWritable() && $localStorage->isOnline()) {
1445 // check if there is already a identical storage present (same pathType and basePath)
1446 $storageRecordConfiguration = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->convertFlexFormDataToConfigurationArray($storageRecord['configuration']);
1447 $localStorageRecordConfiguration = $localStorage->getConfiguration();
1448 if ($storageRecordConfiguration['pathType'] == $localStorageRecordConfiguration['pathType'] && $storageRecordConfiguration['basePath'] == $localStorageRecordConfiguration['basePath']) {
1449 // same storage is already present
1450 $useThisStorageUidInsteadOfTheOneInImport = $localStorage->getUid();
1451 break;
1452 }
1453 }
1454 }
1455 if ($useThisStorageUidInsteadOfTheOneInImport > 0) {
1456 // same storage is already present; map the to be imported one to the present one
1457 $this->import_mapId['sys_file_storage'][$sysFileStorageUid] = $useThisStorageUidInsteadOfTheOneInImport;
1458 } else {
1459 // Local, writable and online storage. Is allowed to be used to later write files in.
1460 $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);
1461 }
1462 } else {
1463 // Storage with non Local drivers could be imported but must not be used to saves files in, because you
1464 // could not be sure, that this is supported. The default storage will be used in this case.
1465 // It could happen that non writable and non online storage will be created as dupes because you could not
1466 // check the detailed configuration options at this point
1467 $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);
1468 $sysFileStorageUidsToBeResetToDefaultStorage[] = $sysFileStorageUid;
1469 }
1470
1471 }
1472
1473 // Importing the added ones
1474 $tce = $this->getNewTCE();
1475 // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.
1476 $tce->reverseOrder = 1;
1477 $tce->isImporting = TRUE;
1478 $tce->start($this->import_data, array());
1479 $tce->process_datamap();
1480 $this->addToMapId($tce->substNEWwithIDs);
1481
1482 $defaultStorageUid = NULL;
1483 // get default storage
1484 $defaultStorage = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->getDefaultStorage();
1485 if ($defaultStorage !== NULL) {
1486 $defaultStorageUid = $defaultStorage->getUid();
1487 }
1488 foreach ($sysFileStorageUidsToBeResetToDefaultStorage as $sysFileStorageUidToBeResetToDefaultStorage) {
1489 $this->import_mapId['sys_file_storage'][$sysFileStorageUidToBeResetToDefaultStorage] = $defaultStorageUid;
1490 }
1491
1492 // unset the sys_file_storage records to prevent a import in writeRecords_records
1493 unset($this->dat['header']['records']['sys_file_storage']);
1494 }
1495
1496 /**
1497 * Imports the sys_file records and the binary files data from internal data array.
1498 *
1499 * @return void
1500 */
1501 protected function writeSysFileRecords() {
1502 if (!isset($this->dat['header']['records']['sys_file'])) {
1503 return;
1504 }
1505
1506 // fetch fresh storage records from database
1507 $storageRecords = $this->fetchStorageRecords();
1508
1509 $defaultStorage = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->getDefaultStorage();
1510
1511 $sanitizedFolderMappings = array();
1512
1513 foreach (array_keys($this->dat['header']['records']['sys_file']) as $sysFileUid) {
1514 $fileRecord = $this->dat['records']['sys_file:' . $sysFileUid]['data'];
1515
1516 // save file to disk
1517 $fileId = md5($fileRecord['storage'] . ':' . $fileRecord['identifier_hash']);
1518 $temporaryFile = $this->writeTemporaryFileFromData($fileId);
1519 if ($temporaryFile === NULL) {
1520 // error on writing the file. Error message was already added
1521 continue;
1522 }
1523
1524 $originalStorageUid = $fileRecord['storage'];
1525 $useStorageFromStorageRecords = FALSE;
1526
1527 // replace storage id, if a alternative one was registered
1528 if (isset($this->import_mapId['sys_file_storage'][$fileRecord['storage']])) {
1529 $fileRecord['storage'] = $this->import_mapId['sys_file_storage'][$fileRecord['storage']];
1530 $useStorageFromStorageRecords = TRUE;
1531 }
1532
1533 if (empty($fileRecord['storage']) && !$this->isFallbackStorage($fileRecord['storage'])) {
1534 // no storage for the file is defined, mostly because of a missing default storage.
1535 $this->error('Error: No storage for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $originalStorageUid . '"');
1536 continue;
1537 }
1538
1539 // using a storage from the local storage is only allowed, if the uid is present in the
1540 // mapping. Only in this case we could be sure, that it's a local, online and writable storage.
1541 if ($useStorageFromStorageRecords && isset($storageRecords[$fileRecord['storage']])) {
1542 /** @var $storage \TYPO3\CMS\Core\Resource\ResourceStorage */
1543 $storage = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->getStorageObject($fileRecord['storage'], $storageRecords[$fileRecord['storage']]);
1544 } elseif ($this->isFallbackStorage($fileRecord['storage'])) {
1545 $storage = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->getStorageObject(0);
1546 } elseif ($defaultStorage !== NULL) {
1547 $storage = $defaultStorage;
1548 } else {
1549 $this->error('Error: No storage available for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1550 continue;
1551 }
1552
1553 $newFile = NULL;
1554
1555 // check, if there is a identical file
1556 try {
1557 if ($storage->hasFile($fileRecord['identifier'])) {
1558 $file = $storage->getFile($fileRecord['identifier']);
1559 if ($file->getSha1() === $fileRecord['sha1']) {
1560 $newFile = $file;
1561 }
1562 }
1563 } catch (Exception $e) {}
1564
1565 if ($newFile === NULL) {
1566
1567 $folderName = PathUtility::dirname(ltrim($fileRecord['identifier'], '/'));
1568 if (in_array($folderName, $sanitizedFolderMappings)) {
1569 $folderName = $sanitizedFolderMappings[$folderName];
1570 }
1571 if (!$storage->hasFolder($folderName)) {
1572 try {
1573 $importFolder = $storage->createFolder($folderName);
1574 if ($importFolder->getIdentifier() !== $folderName && !in_array($folderName, $sanitizedFolderMappings)) {
1575 $sanitizedFolderMappings[$folderName] = $importFolder->getIdentifier();
1576 }
1577 } catch (Exception $e) {
1578 $this->error('Error: Folder could not be created for file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1579 continue;
1580 }
1581 } else {
1582 $importFolder = $storage->getFolder($folderName);
1583 }
1584
1585 try {
1586 /** @var $file \TYPO3\CMS\Core\Resource\File */
1587 $newFile = $storage->addFile($temporaryFile, $importFolder, $fileRecord['name']);
1588 } catch (Exception $e) {
1589 $this->error('Error: File could not be added to the storage: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1590 continue;
1591 }
1592
1593 if ($newFile->getSha1() !== $fileRecord['sha1']) {
1594 $this->error('Error: The hash of the written file is not identical to the import data! File could be corrupted! File: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1595 }
1596 }
1597
1598 // save the new uid in the import id map
1599 $this->import_mapId['sys_file'][$fileRecord['uid']] = $newFile->getUid();
1600 $this->fixUidLocalInSysFileReferenceRecords($fileRecord['uid'], $newFile->getUid());
1601
1602 }
1603
1604 // unset the sys_file records to prevent a import in writeRecords_records
1605 unset($this->dat['header']['records']['sys_file']);
1606 }
1607
1608 /**
1609 * Checks if the $storageId is the id of the fallback storage
1610 *
1611 * @param int|string $storageId
1612 * @return bool
1613 */
1614 protected function isFallbackStorage($storageId) {
1615 return $storageId === 0 || $storageId === '0';
1616 }
1617
1618 /**
1619 * Normally the importer works like the following:
1620 * Step 1: import the records with cleared field values of relation fields (see addSingle())
1621 * Step 2: update the records with the right relation ids (see setRelations())
1622 *
1623 * In step 2 the saving fields of type "relation to sys_file_reference" checks the related sys_file_reference
1624 * record (created in step 1) with the FileExtensionFilter for matching file extensions of the related file.
1625 * To make this work correct, the uid_local of sys_file_reference records has to be not empty AND has to
1626 * relate to the correct (imported) sys_file record uid!!!
1627 *
1628 * This is fixed here.
1629 *
1630 * @param int $oldFileUid
1631 * @param int $newFileUid
1632 * @return void
1633 */
1634 protected function fixUidLocalInSysFileReferenceRecords($oldFileUid, $newFileUid) {
1635 if (!isset($this->dat['header']['records']['sys_file_reference'])) {
1636 return;
1637 }
1638
1639 foreach (array_keys($this->dat['header']['records']['sys_file_reference']) as $sysFileReferenceUid) {
1640 $fileReferenceRecord = $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'];
1641 if ($fileReferenceRecord['uid_local'] == $oldFileUid) {
1642 $fileReferenceRecord['uid_local'] = $newFileUid;
1643 $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'] = $fileReferenceRecord;
1644 }
1645 }
1646 }
1647
1648 /**
1649 * Initializes the folder for legacy imports as subfolder of backend users default upload folder
1650 *
1651 * @return void
1652 */
1653 protected function initializeLegacyImportFolder() {
1654 /** @var \TYPO3\CMS\Core\Resource\Folder $folder */
1655 $folder = $GLOBALS['BE_USER']->getDefaultUploadFolder();
1656 if ($folder === FALSE) {
1657 $this->error('Error: the backend users default upload folder is missing! No files will be imported!');
1658 }
1659 if (!$folder->hasFolder($this->legacyImportTargetPath)) {
1660 try {
1661 $this->legacyImportFolder = $folder->createFolder($this->legacyImportTargetPath);
1662 } catch (\TYPO3\CMS\Core\Exception $e) {
1663 $this->error('Error: the import folder in the default upload folder could not be created! No files will be imported!');
1664 }
1665 } else {
1666 $this->legacyImportFolder = $folder->getSubFolder($this->legacyImportTargetPath);
1667 }
1668
1669 }
1670
1671 /**
1672 * Fetched fresh storage records from database because the new imported
1673 * ones are not in cached data of the StorageRepository
1674 *
1675 * @return bool|array
1676 */
1677 protected function fetchStorageRecords() {
1678 $whereClause = \TYPO3\CMS\Backend\Utility\BackendUtility::BEenableFields('sys_file_storage');
1679 $whereClause .= \TYPO3\CMS\Backend\Utility\BackendUtility::deleteClause('sys_file_storage');
1680
1681 $rows = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows(
1682 '*',
1683 'sys_file_storage',
1684 '1=1' . $whereClause,
1685 '',
1686 '',
1687 '',
1688 'uid'
1689 );
1690
1691 return $rows;
1692 }
1693
1694 /**
1695 * Writes the file from import array to temp dir and returns the filename of it.
1696 *
1697 * @param string $fileId
1698 * @param string $dataKey
1699 * @return string Absolute filename of the temporary filename of the file
1700 */
1701 protected function writeTemporaryFileFromData($fileId, $dataKey = 'files_fal') {
1702 $temporaryFilePath = NULL;
1703 if (is_array($this->dat[$dataKey][$fileId])) {
1704 $temporaryFilePathInternal = GeneralUtility::tempnam('import_temp_');
1705 GeneralUtility::writeFile($temporaryFilePathInternal, $this->dat[$dataKey][$fileId]['content']);
1706 clearstatcache();
1707 if (@is_file($temporaryFilePathInternal)) {
1708 $this->unlinkFiles[] = $temporaryFilePathInternal;
1709 if (filesize($temporaryFilePathInternal) == $this->dat[$dataKey][$fileId]['filesize']) {
1710 $temporaryFilePath = $temporaryFilePathInternal;
1711 } else {
1712 $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' had a size (' . filesize($temporaryFilePathInternal) . ') different from the original (' . $this->dat[$dataKey][$fileId]['filesize'] . ')', 1);
1713 }
1714 } else {
1715 $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' was not written as it should have been!', 1);
1716 }
1717 } else {
1718 $this->error('Error: No file found for ID ' . $fileId, 1);
1719 }
1720 return $temporaryFilePath;
1721 }
1722
1723 /**
1724 * Writing pagetree/pages to database:
1725 *
1726 * @param integer $pid PID in which to import. If the operation is an update operation, the root of the page tree inside will be moved to this PID unless it is the same as the root page from the import
1727 * @return void
1728 * @see writeRecords_records()
1729 * @todo Define visibility
1730 */
1731 public function writeRecords_pages($pid) {
1732 // First, write page structure if any:
1733 if (is_array($this->dat['header']['records']['pages'])) {
1734 // $pageRecords is a copy of the pages array in the imported file. Records here are unset one by one when the addSingle function is called.
1735 $pageRecords = $this->dat['header']['records']['pages'];
1736 $this->import_data = array();
1737 // First add page tree if any
1738 if (is_array($this->dat['header']['pagetree'])) {
1739 $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);
1740 foreach ($pagesFromTree as $uid) {
1741 $thisRec = $this->dat['header']['records']['pages'][$uid];
1742 // PID: Set the main $pid, unless a NEW-id is found
1743 $setPid = isset($this->import_newId_pids[$thisRec['pid']]) ? $this->import_newId_pids[$thisRec['pid']] : $pid;
1744 $this->addSingle('pages', $uid, $setPid);
1745 unset($pageRecords[$uid]);
1746 }
1747 }
1748 // Then add all remaining pages not in tree on root level:
1749 if (count($pageRecords)) {
1750 $remainingPageUids = array_keys($pageRecords);
1751 foreach ($remainingPageUids as $pUid) {
1752 $this->addSingle('pages', $pUid, $pid);
1753 }
1754 }
1755 // Now write to database:
1756 $tce = $this->getNewTCE();
1757 $tce->isImporting = TRUE;
1758 $this->callHook('before_writeRecordsPages', array(
1759 'tce' => &$tce,
1760 'data' => &$this->import_data
1761 ));
1762 $tce->suggestedInsertUids = $this->suggestedInsertUids;
1763 $tce->start($this->import_data, array());
1764 $tce->process_datamap();
1765 $this->callHook('after_writeRecordsPages', array(
1766 'tce' => &$tce
1767 ));
1768 // post-processing: Registering new ids (end all tcemain sessions with this)
1769 $this->addToMapId($tce->substNEWwithIDs);
1770 // In case of an update, order pages from the page tree correctly:
1771 if ($this->update && is_array($this->dat['header']['pagetree'])) {
1772 $this->writeRecords_pages_order($pid);
1773 }
1774 }
1775 }
1776
1777 /**
1778 * Organize all updated pages in page tree so they are related like in the import file
1779 * Only used for updates and when $this->dat['header']['pagetree'] is an array.
1780 *
1781 * @param integer $pid Page id in which to import
1782 * @return void
1783 * @access private
1784 * @see writeRecords_pages(), writeRecords_records_order()
1785 * @todo Define visibility
1786 */
1787 public function writeRecords_pages_order($pid) {
1788 $cmd_data = array();
1789 // Get uid-pid relations and traverse them in order to map to possible new IDs
1790 $pidsFromTree = $this->flatInversePageTree_pid($this->dat['header']['pagetree']);
1791 foreach ($pidsFromTree as $origPid => $newPid) {
1792 if ($newPid >= 0 && $this->dontIgnorePid('pages', $origPid)) {
1793 // If the page had a new id (because it was created) use that instead!
1794 if (substr($this->import_newId_pids[$origPid], 0, 3) === 'NEW') {
1795 if ($this->import_mapId['pages'][$origPid]) {
1796 $mappedPid = $this->import_mapId['pages'][$origPid];
1797 $cmd_data['pages'][$mappedPid]['move'] = $newPid;
1798 }
1799 } else {
1800 $cmd_data['pages'][$origPid]['move'] = $newPid;
1801 }
1802 }
1803 }
1804 // Execute the move commands if any:
1805 if (count($cmd_data)) {
1806 $tce = $this->getNewTCE();
1807 $this->callHook('before_writeRecordsPagesOrder', array(
1808 'tce' => &$tce,
1809 'data' => &$cmd_data
1810 ));
1811 $tce->start(array(), $cmd_data);
1812 $tce->process_cmdmap();
1813 $this->callHook('after_writeRecordsPagesOrder', array(
1814 'tce' => &$tce
1815 ));
1816 }
1817 }
1818
1819 /**
1820 * Write all database records except pages (writtein in writeRecords_pages())
1821 *
1822 * @param integer $pid Page id in which to import
1823 * @return void
1824 * @see writeRecords_pages()
1825 * @todo Define visibility
1826 */
1827 public function writeRecords_records($pid) {
1828 // Write the rest of the records
1829 $this->import_data = array();
1830 if (is_array($this->dat['header']['records'])) {
1831 foreach ($this->dat['header']['records'] as $table => $recs) {
1832 if ($table != 'pages') {
1833 foreach ($recs as $uid => $thisRec) {
1834 // PID: Set the main $pid, unless a NEW-id is found
1835 $setPid = isset($this->import_mapId['pages'][$thisRec['pid']]) ? $this->import_mapId['pages'][$thisRec['pid']] : $pid;
1836 if (is_array($GLOBALS['TCA'][$table]) && $GLOBALS['TCA'][$table]['ctrl']['rootLevel']) {
1837 $setPid = 0;
1838 }
1839 // Add record:
1840 $this->addSingle($table, $uid, $setPid);
1841 }
1842 }
1843 }
1844 } else {
1845 $this->error('Error: No records defined in internal data array.');
1846 }
1847 // Now write to database:
1848 $tce = $this->getNewTCE();
1849 $this->callHook('before_writeRecordsRecords', array(
1850 'tce' => &$tce,
1851 'data' => &$this->import_data
1852 ));
1853 $tce->suggestedInsertUids = $this->suggestedInsertUids;
1854 // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.
1855 $tce->reverseOrder = 1;
1856 $tce->isImporting = TRUE;
1857 $tce->start($this->import_data, array());
1858 $tce->process_datamap();
1859 $this->callHook('after_writeRecordsRecords', array(
1860 'tce' => &$tce
1861 ));
1862 // post-processing: Removing files and registering new ids (end all tcemain sessions with this)
1863 $this->addToMapId($tce->substNEWwithIDs);
1864 // In case of an update, order pages from the page tree correctly:
1865 if ($this->update) {
1866 $this->writeRecords_records_order($pid);
1867 }
1868 }
1869
1870 /**
1871 * Organize all updated record to their new positions.
1872 * Only used for updates
1873 *
1874 * @param integer $mainPid Main PID into which we import.
1875 * @return void
1876 * @access private
1877 * @see writeRecords_records(), writeRecords_pages_order()
1878 * @todo Define visibility
1879 */
1880 public function writeRecords_records_order($mainPid) {
1881 $cmd_data = array();
1882 if (is_array($this->dat['header']['pagetree'])) {
1883 $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);
1884 } else {
1885 $pagesFromTree = array();
1886 }
1887 if (is_array($this->dat['header']['pid_lookup'])) {
1888 foreach ($this->dat['header']['pid_lookup'] as $pid => $recList) {
1889 $newPid = isset($this->import_mapId['pages'][$pid]) ? $this->import_mapId['pages'][$pid] : $mainPid;
1890 if (\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($newPid)) {
1891 foreach ($recList as $tableName => $uidList) {
1892 // If $mainPid===$newPid then we are on root level and we can consider to move pages as well!
1893 // (they will not be in the page tree!)
1894 if (($tableName != 'pages' || !$pagesFromTree[$pid]) && is_array($uidList)) {
1895 $uidList = array_reverse(array_keys($uidList));
1896 foreach ($uidList as $uid) {
1897 if ($this->dontIgnorePid($tableName, $uid)) {
1898 $cmd_data[$tableName][$uid]['move'] = $newPid;
1899 } else {
1900
1901 }
1902 }
1903 }
1904 }
1905 }
1906 }
1907 }
1908 // Execute the move commands if any:
1909 if (count($cmd_data)) {
1910 $tce = $this->getNewTCE();
1911 $this->callHook('before_writeRecordsRecordsOrder', array(
1912 'tce' => &$tce,
1913 'data' => &$cmd_data
1914 ));
1915 $tce->start(array(), $cmd_data);
1916 $tce->process_cmdmap();
1917 $this->callHook('after_writeRecordsRecordsOrder', array(
1918 'tce' => &$tce
1919 ));
1920 }
1921 }
1922
1923 /**
1924 * Adds a single record to the $importData array. Also copies files to tempfolder.
1925 * However all File/DB-references and flexform field contents are set to blank for now! That is done with setRelations() later
1926 *
1927 * @param string $table Table name (from import memory)
1928 * @param integer $uid Record UID (from import memory)
1929 * @param integer $pid Page id
1930 * @return void
1931 * @see writeRecords()
1932 * @todo Define visibility
1933 */
1934 public function addSingle($table, $uid, $pid) {
1935 if ($this->import_mode[$table . ':' . $uid] !== 'exclude') {
1936 $record = $this->dat['records'][$table . ':' . $uid]['data'];
1937 if (is_array($record)) {
1938 if ($this->update && $this->doesRecordExist($table, $uid) && $this->import_mode[$table . ':' . $uid] !== 'as_new') {
1939 $ID = $uid;
1940 } elseif ($table === 'sys_file_metadata' && $record['sys_language_uid'] == '0' && $this->import_mapId['sys_file'][$record['file']]) {
1941 // on adding sys_file records the belonging sys_file_metadata record was also created
1942 // if there is one the record need to be overwritten instead of creating a new one.
1943 $recordInDatabase = $GLOBALS['TYPO3_DB']->exec_SELECTgetSingleRow(
1944 'uid',
1945 'sys_file_metadata',
1946 'file = ' . $this->import_mapId['sys_file'][$record['file']] . ' AND sys_language_uid = 0 AND pid = 0'
1947 );
1948 // if no record could be found, $this->import_mapId['sys_file'][$record['file']] is pointing
1949 // to a file, that was already there, thus a new metadata record should be created
1950 if (is_array($recordInDatabase)) {
1951 $this->import_mapId['sys_file_metadata'][$record['uid']] = $recordInDatabase['uid'];
1952 $ID = $recordInDatabase['uid'];
1953 } else {
1954 $ID = uniqid('NEW');
1955 }
1956
1957 } else {
1958 $ID = uniqid('NEW');
1959 }
1960 $this->import_newId[$table . ':' . $ID] = array('table' => $table, 'uid' => $uid);
1961 if ($table == 'pages') {
1962 $this->import_newId_pids[$uid] = $ID;
1963 }
1964 // Set main record data:
1965 $this->import_data[$table][$ID] = $record;
1966 $this->import_data[$table][$ID]['tx_impexp_origuid'] = $this->import_data[$table][$ID]['uid'];
1967 // Reset permission data:
1968 if ($table === 'pages') {
1969 // Have to reset the user/group IDs so pages are owned by importing user. Otherwise strange things may happen for non-admins!
1970 unset($this->import_data[$table][$ID]['perms_userid']);
1971 unset($this->import_data[$table][$ID]['perms_groupid']);
1972 }
1973 // PID and UID:
1974 unset($this->import_data[$table][$ID]['uid']);
1975 // Updates:
1976 if (\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($ID)) {
1977 unset($this->import_data[$table][$ID]['pid']);
1978 } else {
1979 // Inserts:
1980 $this->import_data[$table][$ID]['pid'] = $pid;
1981 if (($this->import_mode[$table . ':' . $uid] === 'force_uid' && $this->update || $this->force_all_UIDS) && $GLOBALS['BE_USER']->isAdmin()) {
1982 $this->import_data[$table][$ID]['uid'] = $uid;
1983 $this->suggestedInsertUids[$table . ':' . $uid] = 'DELETE';
1984 }
1985 }
1986 // Setting db/file blank:
1987 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
1988 switch ((string) $config['type']) {
1989 case 'db':
1990
1991 case 'file':
1992 // Fixed later in ->setRelations() [because we need to know ALL newly created IDs before we can map relations!]
1993 // In the meantime we set NO values for relations.
1994 //
1995 // BUT for field uid_local of table sys_file_reference the relation MUST not be cleared here,
1996 // because the value is already the uid of the right imported sys_file record.
1997 // @see fixUidLocalInSysFileReferenceRecords()
1998 // If it's empty or a uid to another record the FileExtensionFilter will throw an exception or
1999 // delete the reference record if the file extension of the related record doesn't match.
2000 if ($table !== 'sys_file_reference' && $field !== 'uid_local') {
2001 $this->import_data[$table][$ID][$field] = '';
2002 }
2003 break;
2004 case 'flex':
2005 // Fixed later in setFlexFormRelations()
2006 // In the meantime we set NO value for flexforms - this is mainly because file references inside will not be processed properly; In fact references will point to no file or existing files (in which case there will be double-references which is a big problem of course!)
2007 $this->import_data[$table][$ID][$field] = '';
2008 break;
2009 }
2010 }
2011 } elseif ($table . ':' . $uid != 'pages:0') {
2012 // On root level we don't want this error message.
2013 $this->error('Error: no record was found in data array!', 1);
2014 }
2015 }
2016 }
2017
2018 /**
2019 * Registers the substNEWids in memory.
2020 *
2021 * @param array $substNEWwithIDs From tcemain to be merged into internal mapping variable in this object
2022 * @return void
2023 * @see writeRecords()
2024 * @todo Define visibility
2025 */
2026 public function addToMapId($substNEWwithIDs) {
2027 foreach ($this->import_data as $table => $recs) {
2028 foreach ($recs as $id => $value) {
2029 $old_uid = $this->import_newId[$table . ':' . $id]['uid'];
2030 if (isset($substNEWwithIDs[$id])) {
2031 $this->import_mapId[$table][$old_uid] = $substNEWwithIDs[$id];
2032 } elseif ($this->update) {
2033 // Map same ID to same ID....
2034 $this->import_mapId[$table][$old_uid] = $id;
2035 } else {
2036 // if $this->import_mapId contains already the right mapping, skip the error msg. See special handling of sys_file_metadata in addSingle() => nothing to do
2037 if (!($table === 'sys_file_metadata' && isset($this->import_mapId[$table][$old_uid]) && $this->import_mapId[$table][$old_uid] == $id)) {
2038 $this->error('Possible error: ' . $table . ':' . $old_uid . ' had no new id assigned to it. This indicates that the record was not added to database during import. Please check changelog!', 1);
2039 }
2040
2041 }
2042 }
2043 }
2044 }
2045
2046 /**
2047 * Returns a new $TCE object
2048 *
2049 * @return object $TCE object
2050 * @todo Define visibility
2051 */
2052 public function getNewTCE() {
2053 $tce = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\DataHandling\\DataHandler');
2054 $tce->stripslashes_values = 0;
2055 $tce->dontProcessTransformations = 1;
2056 $tce->enableLogging = $this->enableLogging;
2057 $tce->alternativeFileName = $this->alternativeFileName;
2058 $tce->alternativeFilePath = $this->alternativeFilePath;
2059 return $tce;
2060 }
2061
2062 /**
2063 * Cleaning up all the temporary files stored in typo3temp/ folder
2064 *
2065 * @return void
2066 * @todo Define visibility
2067 */
2068 public function unlinkTempFiles() {
2069 foreach ($this->unlinkFiles as $fileName) {
2070 if (GeneralUtility::isFirstPartOfStr($fileName, PATH_site . 'typo3temp/')) {
2071 GeneralUtility::unlink_tempfile($fileName);
2072 clearstatcache();
2073 if (is_file($fileName)) {
2074 $this->error('Error: ' . $fileName . ' was NOT unlinked as it should have been!', 1);
2075 }
2076 } else {
2077 $this->error('Error: ' . $fileName . ' was not in temp-path. Not removed!', 1);
2078 }
2079 }
2080 $this->unlinkFiles = array();
2081 }
2082
2083 /***************************
2084 * Import / Relations setting
2085 ***************************/
2086
2087 /**
2088 * At the end of the import process all file and DB relations should be set properly (that is relations
2089 * to imported records are all re-created so imported records are correctly related again)
2090 * Relations in flexform fields are processed in setFlexFormRelations() after this function
2091 *
2092 * @return void
2093 * @see setFlexFormRelations()
2094 * @todo Define visibility
2095 */
2096 public function setRelations() {
2097 $updateData = array();
2098 // import_newId contains a register of all records that was in the import memorys "records" key
2099 foreach ($this->import_newId as $nId => $dat) {
2100 $table = $dat['table'];
2101 $uid = $dat['uid'];
2102 // original UID - NOT the new one!
2103 // If the record has been written and received a new id, then proceed:
2104 if (is_array($this->import_mapId[$table]) && isset($this->import_mapId[$table][$uid])) {
2105 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
2106 if (is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {
2107 if ($this->legacyImport) {
2108 if ($table != 'pages') {
2109 $oldPid = $this->dat['records'][$table . ':' . $uid]['data']['pid'];
2110 $thisNewPageUid = \TYPO3\CMS\Backend\Utility\BackendUtility::wsMapId($table, $this->import_mapId['pages'][$oldPid]);
2111 } else {
2112 $thisNewPageUid = $thisNewUid;
2113 }
2114 }
2115 // Traverse relation fields of each record
2116 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
2117 // uid_local of sys_file_reference needs no update because the correct reference uid was already written
2118 // @see ImportExport::fixUidLocalInSysFileReferenceRecords()
2119 if ($table === 'sys_file_reference' && $field === 'uid_local') {
2120 continue;
2121 }
2122 switch ((string) $config['type']) {
2123 case 'db':
2124 if (is_array($config['itemArray']) && count($config['itemArray'])) {
2125 $itemConfig = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
2126 $valArray = $this->setRelations_db($config['itemArray'], $itemConfig);
2127 $updateData[$table][$thisNewUid][$field] = implode(',', $valArray);
2128 }
2129 break;
2130 case 'file':
2131 if (is_array($config['newValueFiles']) && count($config['newValueFiles'])) {
2132 $valArr = array();
2133 foreach ($config['newValueFiles'] as $fI) {
2134 $valArr[] = $this->import_addFileNameToBeCopied($fI);
2135 }
2136 if ($this->legacyImport && $this->legacyImportFolder === NULL && isset($this->legacyImportMigrationTables[$table][$field])) {
2137 // Do nothing - the legacy import folder is missing
2138 } elseif ($this->legacyImport && $this->legacyImportFolder !== NULL && isset($this->legacyImportMigrationTables[$table][$field])) {
2139 $refIds = array();
2140 foreach ($valArr as $tempFile) {
2141 $fileName = $this->alternativeFileName[$tempFile];
2142 $fileObject = NULL;
2143
2144 try {
2145 // check, if there is alreay the same file in the folder
2146 if ($this->legacyImportFolder->hasFile($fileName)) {
2147 $fileStorage = $this->legacyImportFolder->getStorage();
2148 $file = $fileStorage->getFile($this->legacyImportFolder->getIdentifier() . $fileName);
2149 if ($file->getSha1() === sha1_file($tempFile)) {
2150 $fileObject = $file;
2151 }
2152 }
2153 } catch (Exception $e) {}
2154
2155 if ($fileObject === NULL) {
2156 try {
2157 $fileObject = $this->legacyImportFolder->addFile($tempFile, $fileName, 'changeName');
2158 } catch (\TYPO3\CMS\Core\Exception $e) {
2159 $this->error('Error: no file could be added to the storage for file name' . $this->alternativeFileName[$tempFile]);
2160 }
2161 }
2162 if ($fileObject !== NULL) {
2163 $refId = uniqid('NEW');
2164 $refIds[] = $refId;
2165 $updateData['sys_file_reference'][$refId] = array(
2166 'uid_local' => $fileObject->getUid(),
2167 'uid_foreign' => $thisNewUid, // uid of your content record
2168 'tablenames' => $table,
2169 'fieldname' => $field,
2170 'pid' => $thisNewPageUid, // parent id of the parent page
2171 'table_local' => 'sys_file',
2172 );
2173 }
2174 }
2175 $updateData[$table][$thisNewUid][$field] = implode(',', $refIds);
2176 if (!empty($this->legacyImportMigrationTables[$table][$field])) {
2177 $this->legacyImportMigrationRecords[$table][$thisNewUid][$field] = $refIds;
2178 }
2179 } else {
2180 $updateData[$table][$thisNewUid][$field] = implode(',', $valArr);
2181 }
2182 }
2183 break;
2184 }
2185 }
2186 } else {
2187 $this->error('Error: no record was found in data array!', 1);
2188 }
2189 } else {
2190 $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')', 1);
2191 }
2192 }
2193 if (count($updateData)) {
2194 $tce = $this->getNewTCE();
2195 $tce->isImporting = TRUE;
2196 $this->callHook('before_setRelation', array(
2197 'tce' => &$tce,
2198 'data' => &$updateData
2199 ));
2200 $tce->start($updateData, array());
2201 $tce->process_datamap();
2202 // Replace the temporary "NEW" ids with the final ones.
2203 foreach ($this->legacyImportMigrationRecords as $table => $records) {
2204 foreach ($records as $uid => $fields) {
2205 foreach ($fields as $field => $referenceIds) {
2206 foreach ($referenceIds as $key => $referenceId) {
2207 $this->legacyImportMigrationRecords[$table][$uid][$field][$key] = $tce->substNEWwithIDs[$referenceId];
2208 }
2209 }
2210 }
2211 }
2212 $this->callHook('after_setRelations', array(
2213 'tce' => &$tce
2214 ));
2215 }
2216 }
2217
2218 /**
2219 * Maps relations for database
2220 *
2221 * @param array $itemArray Array of item sets (table/uid) from a dbAnalysis object
2222 * @param array $itemConfig Array of TCA config of the field the relation to be set on
2223 * @return array Array with values [table]_[uid] or [uid] for field of type group / internal_type file_reference. These values have the regular tcemain-input group/select type which means they will automatically be processed into a uid-list or MM relations.
2224 * @todo Define visibility
2225 */
2226 public function setRelations_db($itemArray, $itemConfig) {
2227 $valArray = array();
2228 foreach ($itemArray as $relDat) {
2229 if (is_array($this->import_mapId[$relDat['table']]) && isset($this->import_mapId[$relDat['table']][$relDat['id']])) {
2230 // Since non FAL file relation type group internal_type file_reference are handled as reference to
2231 // sys_file records Datahandler requires the value as uid of the the related sys_file record only
2232 if ($itemConfig['type'] === 'group' && $itemConfig['internal_type'] === 'file_reference') {
2233 $value = $this->import_mapId[$relDat['table']][$relDat['id']];
2234 } else {
2235 $value = $relDat['table'] . '_' . $this->import_mapId[$relDat['table']][$relDat['id']];
2236 }
2237 $valArray[] = $value;
2238 } elseif ($this->isTableStatic($relDat['table']) || $this->isExcluded($relDat['table'], $relDat['id']) || $relDat['id'] < 0) {
2239 // Checking for less than zero because some select types could contain negative values, eg. fe_groups (-1, -2) and sys_language (-1 = ALL languages). This must be handled on both export and import.
2240 $valArray[] = $relDat['table'] . '_' . $relDat['id'];
2241 } else {
2242 $this->error('Lost relation: ' . $relDat['table'] . ':' . $relDat['id'], 1);
2243 }
2244 }
2245 return $valArray;
2246 }
2247
2248 /**
2249 * Writes the file from import array to temp dir and returns the filename of it.
2250 *
2251 * @param array $fI File information with three keys: "filename" = filename without path, "ID_absFile" = absolute filepath to the file (including the filename), "ID" = md5 hash of "ID_absFile
2252 * @return string Absolute filename of the temporary filename of the file. In ->alternativeFileName the original name is set.
2253 * @todo Define visibility
2254 */
2255 public function import_addFileNameToBeCopied($fI) {
2256 if (is_array($this->dat['files'][$fI['ID']])) {
2257 $tmpFile = GeneralUtility::tempnam('import_temp_');
2258 GeneralUtility::writeFile($tmpFile, $this->dat['files'][$fI['ID']]['content']);
2259 clearstatcache();
2260 if (@is_file($tmpFile)) {
2261 $this->unlinkFiles[] = $tmpFile;
2262 if (filesize($tmpFile) == $this->dat['files'][$fI['ID']]['filesize']) {
2263 $this->alternativeFileName[$tmpFile] = $fI['filename'];
2264 $this->alternativeFilePath[$tmpFile] = $this->dat['files'][$fI['ID']]['relFileRef'];
2265 return $tmpFile;
2266 } else {
2267 $this->error('Error: temporary file ' . $tmpFile . ' had a size (' . filesize($tmpFile) . ') different from the original (' . $this->dat['files'][$fI['ID']]['filesize'] . ')', 1);
2268 }
2269 } else {
2270 $this->error('Error: temporary file ' . $tmpFile . ' was not written as it should have been!', 1);
2271 }
2272 } else {
2273 $this->error('Error: No file found for ID ' . $fI['ID'], 1);
2274 }
2275 }
2276
2277 /**
2278 * After all DB relations has been set in the end of the import (see setRelations()) then it is time to correct all relations inside of FlexForm fields.
2279 * The reason for doing this after is that the setting of relations may affect (quite often!) which data structure is used for the flexforms field!
2280 *
2281 * @return void
2282 * @see setRelations()
2283 * @todo Define visibility
2284 */
2285 public function setFlexFormRelations() {
2286 $updateData = array();
2287 // import_newId contains a register of all records that was in the import memorys "records" key
2288 foreach ($this->import_newId as $nId => $dat) {
2289 $table = $dat['table'];
2290 $uid = $dat['uid'];
2291 // original UID - NOT the new one!
2292 // If the record has been written and received a new id, then proceed:
2293 if (is_array($this->import_mapId[$table]) && isset($this->import_mapId[$table][$uid])) {
2294 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
2295 if (is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {
2296 // Traverse relation fields of each record
2297 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
2298 switch ((string) $config['type']) {
2299 case 'flex':
2300 // Get XML content and set as default value (string, non-processed):
2301 $updateData[$table][$thisNewUid][$field] = $this->dat['records'][$table . ':' . $uid]['data'][$field];
2302 // If there has been registered relations inside the flex form field, run processing on the content:
2303 if (count($config['flexFormRels']['db']) || count($config['flexFormRels']['file'])) {
2304 $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');
2305 // This will fetch the new row for the element (which should be updated with any references to data structures etc.)
2306 $conf = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
2307 if (is_array($origRecordRow) && is_array($conf) && $conf['type'] === 'flex') {
2308 // Get current data structure and value array:
2309 $dataStructArray = BackendUtility::getFlexFormDS($conf, $origRecordRow, $table, $field);
2310 $currentValueArray = GeneralUtility::xml2array($updateData[$table][$thisNewUid][$field]);
2311 // Do recursive processing of the XML data:
2312 $iteratorObj = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\DataHandling\\DataHandler');
2313 $iteratorObj->callBackObj = $this;
2314 $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData($currentValueArray['data'], array(), array(), $dataStructArray, array($table, $thisNewUid, $field, $config), 'remapListedDBRecords_flexFormCallBack');
2315 // The return value is set as an array which means it will be processed by tcemain for file and DB references!
2316 if (is_array($currentValueArray['data'])) {
2317 $updateData[$table][$thisNewUid][$field] = $currentValueArray;
2318 }
2319 }
2320 }
2321 break;
2322 }
2323 }
2324 } else {
2325 $this->error('Error: no record was found in data array!', 1);
2326 }
2327 } else {
2328 $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')', 1);
2329 }
2330 }
2331 if (count($updateData)) {
2332 $tce = $this->getNewTCE();
2333 $tce->isImporting = TRUE;
2334 $this->callHook('before_setFlexFormRelations', array(
2335 'tce' => &$tce,
2336 'data' => &$updateData
2337 ));
2338 $tce->start($updateData, array());
2339 $tce->process_datamap();
2340 $this->callHook('after_setFlexFormRelations', array(
2341 'tce' => &$tce
2342 ));
2343 }
2344 }
2345
2346 /**
2347 * Callback function for traversing the FlexForm structure in relation to remapping database relations
2348 *
2349 * @param array $pParams Set of parameters in numeric array: table, uid, field
2350 * @param array $dsConf TCA config for field (from Data Structure of course)
2351 * @param string $dataValue Field value (from FlexForm XML)
2352 * @param string $dataValue_ext1 Not used
2353 * @param string $dataValue_ext2 Not used
2354 * @param string $path Path of where the data structure of the element is found
2355 * @return array Array where the "value" key carries the value.
2356 * @see setFlexFormRelations()
2357 * @todo Define visibility
2358 */
2359 public function remapListedDBRecords_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path) {
2360 // Extract parameters:
2361 list($table, $uid, $field, $config) = $pParams;
2362 // In case the $path is used as index without a trailing slash we will remove that
2363 if (!is_array($config['flexFormRels']['db'][$path]) && is_array($config['flexFormRels']['db'][rtrim($path, '/')])) {
2364 $path = rtrim($path, '/');
2365 }
2366 if (is_array($config['flexFormRels']['db'][$path])) {
2367 $valArray = $this->setRelations_db($config['flexFormRels']['db'][$path], $dsConf);
2368 $dataValue = implode(',', $valArray);
2369 }
2370 if (is_array($config['flexFormRels']['file'][$path])) {
2371 foreach ($config['flexFormRels']['file'][$path] as $fI) {
2372 $valArr[] = $this->import_addFileNameToBeCopied($fI);
2373 }
2374 $dataValue = implode(',', $valArr);
2375 }
2376 return array('value' => $dataValue);
2377 }
2378
2379 /**************************
2380 * Import / Soft References
2381 *************************/
2382
2383 /**
2384 * Processing of soft references
2385 *
2386 * @return void
2387 * @todo Define visibility
2388 */
2389 public function processSoftReferences() {
2390 // Initialize:
2391 $inData = array();
2392 // Traverse records:
2393 if (is_array($this->dat['header']['records'])) {
2394 foreach ($this->dat['header']['records'] as $table => $recs) {
2395 foreach ($recs as $uid => $thisRec) {
2396 // If there are soft references defined, traverse those:
2397 if (isset($GLOBALS['TCA'][$table]) && is_array($thisRec['softrefs'])) {
2398 // First traversal is to collect softref configuration and split them up based on fields. This could probably also have been done with the "records" key instead of the header.
2399 $fieldsIndex = array();
2400 foreach ($thisRec['softrefs'] as $softrefDef) {
2401 // If a substitution token is set:
2402 if ($softrefDef['field'] && is_array($softrefDef['subst']) && $softrefDef['subst']['tokenID']) {
2403 $fieldsIndex[$softrefDef['field']][$softrefDef['subst']['tokenID']] = $softrefDef;
2404 }
2405 }
2406 // The new id:
2407 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
2408 // Now, if there are any fields that require substitution to be done, lets go for that:
2409 foreach ($fieldsIndex as $field => $softRefCfgs) {
2410 if (is_array($GLOBALS['TCA'][$table]['columns'][$field])) {
2411 $conf = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
2412 if ($conf['type'] === 'flex') {
2413 // This will fetch the new row for the element (which should be updated with any references to data structures etc.)
2414 $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');
2415 if (is_array($origRecordRow)) {
2416 // Get current data structure and value array:
2417 $dataStructArray = BackendUtility::getFlexFormDS($conf, $origRecordRow, $table, $field);
2418 $currentValueArray = GeneralUtility::xml2array($origRecordRow[$field]);
2419 // Do recursive processing of the XML data:
2420 /** @var $iteratorObj \TYPO3\CMS\Core\DataHandling\DataHandler */
2421 $iteratorObj = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\DataHandling\\DataHandler');
2422 $iteratorObj->callBackObj = $this;
2423 $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData($currentValueArray['data'], array(), array(), $dataStructArray, array($table, $uid, $field, $softRefCfgs), 'processSoftReferences_flexFormCallBack');
2424 // The return value is set as an array which means it will be processed by tcemain for file and DB references!
2425 if (is_array($currentValueArray['data'])) {
2426 $inData[$table][$thisNewUid][$field] = $currentValueArray;
2427 }
2428 }
2429 } else {
2430 // Get tokenizedContent string and proceed only if that is not blank:
2431 $tokenizedContent = $this->dat['records'][$table . ':' . $uid]['rels'][$field]['softrefs']['tokenizedContent'];
2432 if (strlen($tokenizedContent) && is_array($softRefCfgs)) {
2433 $inData[$table][$thisNewUid][$field] = $this->processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid);
2434 }
2435 }
2436 }
2437 }
2438 }
2439 }
2440 }
2441 }
2442 // Now write to database:
2443 $tce = $this->getNewTCE();
2444 $tce->isImporting = TRUE;
2445 $this->callHook('before_processSoftReferences', array(
2446 'tce' => &$tce,
2447 'data' => &$inData
2448 ));
2449 $tce->enableLogging = TRUE;
2450 $tce->start($inData, array());
2451 $tce->process_datamap();
2452 $this->callHook('after_processSoftReferences', array(
2453 'tce' => &$tce
2454 ));
2455 }
2456
2457 /**
2458 * Callback function for traversing the FlexForm structure in relation to remapping softreference relations
2459 *
2460 * @param array $pParams Set of parameters in numeric array: table, uid, field
2461 * @param array $dsConf TCA config for field (from Data Structure of course)
2462 * @param string $dataValue Field value (from FlexForm XML)
2463 * @param string $dataValue_ext1 Not used
2464 * @param string $dataValue_ext2 Not used
2465 * @param string $path Path of where the data structure where the element is found
2466 * @return array Array where the "value" key carries the value.
2467 * @see setFlexFormRelations()
2468 * @todo Define visibility
2469 */
2470 public function processSoftReferences_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path) {
2471 // Extract parameters:
2472 list($table, $origUid, $field, $softRefCfgs) = $pParams;
2473 if (is_array($softRefCfgs)) {
2474 // First, find all soft reference configurations for this structure path (they are listed flat in the header):
2475 $thisSoftRefCfgList = array();
2476 foreach ($softRefCfgs as $sK => $sV) {
2477 if ($sV['structurePath'] === $path) {
2478 $thisSoftRefCfgList[$sK] = $sV;
2479 }
2480 }
2481 // If any was found, do processing:
2482 if (count($thisSoftRefCfgList)) {
2483 // Get tokenizedContent string and proceed only if that is not blank:
2484 $tokenizedContent = $this->dat['records'][$table . ':' . $origUid]['rels'][$field]['flexFormRels']['softrefs'][$path]['tokenizedContent'];
2485 if (strlen($tokenizedContent)) {
2486 $dataValue = $this->processSoftReferences_substTokens($tokenizedContent, $thisSoftRefCfgList, $table, $origUid);
2487 }
2488 }
2489 }
2490 // Return
2491 return array('value' => $dataValue);
2492 }
2493
2494 /**
2495 * Substition of softreference tokens
2496 *
2497 * @param string $tokenizedContent Content of field with soft reference tokens in.
2498 * @param array $softRefCfgs Soft reference configurations
2499 * @param string $table Table for which the processing occurs
2500 * @param string $uid UID of record from table
2501 * @return string The input content with tokens substituted according to entries in softRefCfgs
2502 * @todo Define visibility
2503 */
2504 public function processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid) {
2505 // traverse each softref type for this field:
2506 foreach ($softRefCfgs as $cfg) {
2507 // Get token ID:
2508 $tokenID = $cfg['subst']['tokenID'];
2509 // Default is current token value:
2510 $insertValue = $cfg['subst']['tokenValue'];
2511 // Based on mode:
2512 switch ((string) $this->softrefCfg[$tokenID]['mode']) {
2513 case 'exclude':
2514 // Exclude is a simple passthrough of the value
2515 break;
2516 case 'editable':
2517 // Editable always picks up the value from this input array:
2518 $insertValue = $this->softrefInputValues[$tokenID];
2519 break;
2520 default:
2521 // Mapping IDs/creating files: Based on type, look up new value:
2522 switch ((string) $cfg['subst']['type']) {
2523 case 'file':
2524 // Create / Overwrite file:
2525 $insertValue = $this->processSoftReferences_saveFile($cfg['subst']['relFileName'], $cfg, $table, $uid);
2526 break;
2527 case 'db':
2528 default:
2529 // Trying to map database element if found in the mapID array:
2530 list($tempTable, $tempUid) = explode(':', $cfg['subst']['recordRef']);
2531 if (isset($this->import_mapId[$tempTable][$tempUid])) {
2532 $insertValue = BackendUtility::wsMapId($tempTable, $this->import_mapId[$tempTable][$tempUid]);
2533 // Look if reference is to a page and the original token value was NOT an integer - then we assume is was an alias and try to look up the new one!
2534 if ($tempTable === 'pages' && !\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($cfg['subst']['tokenValue'])) {
2535 $recWithUniqueValue = BackendUtility::getRecord($tempTable, $insertValue, 'alias');
2536 if ($recWithUniqueValue['alias']) {
2537 $insertValue = $recWithUniqueValue['alias'];
2538 }
2539 } elseif (strpos($cfg['subst']['tokenValue'], ':') !== FALSE) {
2540 list($tokenKey, $tokenId) = explode(':', $cfg['subst']['tokenValue']);
2541 $insertValue = $tokenKey . ':' . $insertValue;
2542 }
2543 }
2544 }
2545 }
2546 // Finally, swap the soft reference token in tokenized content with the insert value:
2547 $tokenizedContent = str_replace('{softref:' . $tokenID . '}', $insertValue, $tokenizedContent);
2548 }
2549 return $tokenizedContent;
2550 }
2551
2552 /**
2553 * Process a soft reference file
2554 *
2555 * @param string $relFileName Old Relative filename
2556 * @param array $cfg soft reference configuration array
2557 * @param string $table Table for which the processing occurs
2558 * @param string $uid UID of record from table
2559 * @return string New relative filename (value to insert instead of the softref token)
2560 * @todo Define visibility
2561 */
2562 public function processSoftReferences_saveFile($relFileName, $cfg, $table, $uid) {
2563 if ($fileHeaderInfo = $this->dat['header']['files'][$cfg['file_ID']]) {
2564 // Initialize; Get directory prefix for file and find possible RTE filename
2565 $dirPrefix = PathUtility::dirname($relFileName) . '/';
2566 $rteOrigName = $this->getRTEoriginalFilename(PathUtility::basename($relFileName));
2567 // If filename looks like an RTE file, and the directory is in "uploads/", then process as a RTE file!
2568 if ($rteOrigName && GeneralUtility::isFirstPartOfStr($dirPrefix, 'uploads/')) {
2569 // RTE:
2570 // First, find unique RTE file name:
2571 if (@is_dir((PATH_site . $dirPrefix))) {
2572 // From the "original" RTE filename, produce a new "original" destination filename which is unused. Even if updated, the image should be unique. Currently the problem with this is that it leaves a lot of unused RTE images...
2573 $fileProcObj = $this->getFileProcObj();
2574 $origDestName = $fileProcObj->getUniqueName($rteOrigName, PATH_site . $dirPrefix);
2575 // Create copy file name:
2576 $pI = pathinfo($relFileName);
2577 $copyDestName = PathUtility::dirname($origDestName) . '/RTEmagicC_' . substr(PathUtility::basename($origDestName), 10) . '.' . $pI['extension'];
2578 if (!@is_file($copyDestName) && !@is_file($origDestName) && $origDestName === GeneralUtility::getFileAbsFileName($origDestName) && $copyDestName === GeneralUtility::getFileAbsFileName($copyDestName)) {
2579 if ($this->dat['header']['files'][$fileHeaderInfo['RTE_ORIG_ID']]) {
2580 if ($this->legacyImport) {
2581 $fileName = PathUtility::basename($copyDestName);
2582 $this->writeSysFileResourceForLegacyImport($fileName, $cfg['file_ID']);
2583 $relFileName = $this->filePathMap[$cfg['file_ID']] . '" data-htmlarea-file-uid="' . $fileName . '" data-htmlarea-file-table="sys_file';
2584 // Also save the original file
2585 $originalFileName = PathUtility::basename($origDestName);
2586 $this->writeSysFileResourceForLegacyImport($originalFileName, $fileHeaderInfo['RTE_ORIG_ID']);
2587 } else {
2588 // Write the copy and original RTE file to the respective filenames:
2589 $this->writeFileVerify($copyDestName, $cfg['file_ID'], TRUE);
2590 $this->writeFileVerify($origDestName, $fileHeaderInfo['RTE_ORIG_ID'], TRUE);
2591 // Return the relative path of the copy file name:
2592 return \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($copyDestName);
2593 }
2594 } else {
2595 $this->error('ERROR: Could not find original file ID');
2596 }
2597 } else {
2598 $this->error('ERROR: The destination filenames "' . $copyDestName . '" and "' . $origDestName . '" either existed or have non-valid names');
2599 }
2600 } else {
2601 $this->error('ERROR: "' . PATH_site . $dirPrefix . '" was not a directory, so could not process file "' . $relFileName . '"');
2602 }
2603 } elseif (GeneralUtility::isFirstPartOfStr($dirPrefix, $this->fileadminFolderName . '/')) {
2604 // File in fileadmin/ folder:
2605 // Create file (and possible resources)
2606 $newFileName = $this->processSoftReferences_saveFile_createRelFile($dirPrefix, PathUtility::basename($relFileName), $cfg['file_ID'], $table, $uid);
2607 if (strlen($newFileName)) {
2608 $relFileName = $newFileName;
2609 } else {
2610 $this->error('ERROR: No new file created for "' . $relFileName . '"');
2611 }
2612 } else {
2613 $this->error('ERROR: Sorry, cannot operate on non-RTE files which are outside the fileadmin folder.');
2614 }
2615 } else {
2616 $this->error('ERROR: Could not find file ID in header.');
2617 }
2618 // Return (new) filename relative to PATH_site:
2619 return $relFileName;
2620 }
2621
2622 /**
2623 * Create file in directory and return the new (unique) filename
2624 *
2625 * @param string $origDirPrefix Directory prefix, relative, with trailing slash
2626 * @param string $fileName Filename (without path)
2627 * @param string $fileID File ID from import memory
2628 * @param string $table Table for which the processing occurs
2629 * @param string $uid UID of record from table
2630 * @return string New relative filename, if any
2631 * @todo Define visibility
2632 */
2633 public function processSoftReferences_saveFile_createRelFile($origDirPrefix, $fileName, $fileID, $table, $uid) {
2634 // If the fileID map contains an entry for this fileID then just return the relative filename of that entry; we don't want to write another unique filename for this one!
2635 if (isset($this->fileIDMap[$fileID])) {
2636 return \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($this->fileIDMap[$fileID]);
2637 }
2638 if ($this->legacyImport) {
2639 // set dirPrefix to fileadmin because the right target folder is set and checked for permissions later
2640 $dirPrefix = $this->fileadminFolderName . '/';
2641 } else {
2642 // Verify FileMount access to dir-prefix. Returns the best alternative relative path if any
2643 $dirPrefix = $this->verifyFolderAccess($origDirPrefix);
2644 }
2645 if ($dirPrefix && (!$this->update || $origDirPrefix === $dirPrefix) && $this->checkOrCreateDir($dirPrefix)) {
2646 $fileHeaderInfo = $this->dat['header']['files'][$fileID];
2647 $updMode = $this->update && $this->import_mapId[$table][$uid] === $uid && $this->import_mode[$table . ':' . $uid] !== 'as_new';
2648 // Create new name for file:
2649 // Must have same ID in map array (just for security, is not really needed) and NOT be set "as_new".
2650
2651 // Write main file:
2652 if ($this->legacyImport) {
2653 $fileWritten = $this->writeSysFileResourceForLegacyImport($fileName, $fileID);
2654 if ($fileWritten) {
2655 $newName = 'file:' . $fileName;
2656 return $newName;
2657 // no support for HTML/CSS file resources attached ATM - see below
2658 }
2659 } else {
2660 if ($updMode) {
2661 $newName = PATH_site . $dirPrefix . $fileName;
2662 } else {
2663 // Create unique filename:
2664 $fileProcObj = $this->getFileProcObj();
2665 $newName = $fileProcObj->getUniqueName($fileName, PATH_site . $dirPrefix);
2666 }
2667 if ($this->writeFileVerify($newName, $fileID)) {
2668 // If the resource was an HTML/CSS file with resources attached, we will write those as well!
2669 if (is_array($fileHeaderInfo['EXT_RES_ID'])) {
2670 $tokenizedContent = $this->dat['files'][$fileID]['tokenizedContent'];
2671 $tokenSubstituted = FALSE;
2672 $fileProcObj = $this->getFileProcObj();
2673 if ($updMode) {
2674 foreach ($fileHeaderInfo['EXT_RES_ID'] as $res_fileID) {
2675 if ($this->dat['files'][$res_fileID]['filename']) {
2676 // Resolve original filename:
2677 $relResourceFileName = $this->dat['files'][$res_fileID]['parentRelFileName'];
2678 $absResourceFileName = GeneralUtility::resolveBackPath(PATH_site . $origDirPrefix . $relResourceFileName);
2679 $absResourceFileName = GeneralUtility::getFileAbsFileName($absResourceFileName);
2680 if ($absResourceFileName && GeneralUtility::isFirstPartOfStr($absResourceFileName, PATH_site . $this->fileadminFolderName . '/')) {
2681 $destDir = \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix(PathUtility::dirname($absResourceFileName) . '/');
2682 if ($this->verifyFolderAccess($destDir, TRUE) && $this->checkOrCreateDir($destDir)) {
2683 $this->writeFileVerify($absResourceFileName, $res_fileID);
2684 } else {
2685 $this->error('ERROR: Could not create file in directory "' . $destDir . '"');
2686 }
2687 } else {
2688 $this->error('ERROR: Could not resolve path for "' . $relResourceFileName . '"');
2689 }
2690 $tokenizedContent = str_replace('{EXT_RES_ID:' . $res_fileID . '}', $relResourceFileName, $tokenizedContent);
2691 $tokenSubstituted = TRUE;
2692 }
2693 }
2694 } else {
2695 // Create the resouces directory name (filename without extension, suffixed "_FILES")
2696 $resourceDir = PathUtility::dirname($newName) . '/' . preg_replace('/\\.[^.]*$/', '', PathUtility::basename($newName)) . '_FILES';
2697 if (GeneralUtility::mkdir($resourceDir)) {
2698 foreach ($fileHeaderInfo['EXT_RES_ID'] as $res_fileID) {
2699 if ($this->dat['files'][$res_fileID]['filename']) {
2700 $absResourceFileName = $fileProcObj->getUniqueName($this->dat['files'][$res_fileID]['filename'], $resourceDir);
2701 $relResourceFileName = substr($absResourceFileName, strlen(PathUtility::dirname($resourceDir)) + 1);
2702 $this->writeFileVerify($absResourceFileName, $res_fileID);
2703 $tokenizedContent = str_replace('{EXT_RES_ID:' . $res_fileID . '}', $relResourceFileName, $tokenizedContent);
2704 $tokenSubstituted = TRUE;
2705 }
2706 }
2707 }
2708 }
2709 // If substitutions has been made, write the content to the file again:
2710 if ($tokenSubstituted) {
2711 GeneralUtility::writeFile($newName, $tokenizedContent);
2712 }
2713 }
2714 return \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($newName);
2715 }
2716 }
2717 }
2718 }
2719
2720 /**
2721 * Writes a file from the import memory having $fileID to file name $fileName which must be an absolute path inside PATH_site
2722 *
2723 * @param string $fileName Absolute filename inside PATH_site to write to
2724 * @param string $fileID File ID from import memory
2725 * @param boolean $bypassMountCheck Bypasses the checking against filemounts - only for RTE files!
2726 * @return boolean Returns TRUE if it went well. Notice that the content of the file is read again, and md5 from import memory is validated.
2727 * @todo Define visibility
2728 */
2729 public function writeFileVerify($fileName, $fileID, $bypassMountCheck = FALSE) {
2730 $fileProcObj = $this->getFileProcObj();
2731 if ($fileProcObj->actionPerms['addFile']) {
2732 // Just for security, check again. Should actually not be necessary.
2733 if ($fileProcObj->checkPathAgainstMounts($fileName) || $bypassMountCheck) {
2734 $fI = GeneralUtility::split_fileref($fileName);
2735 if ($fileProcObj->checkIfAllowed($fI['fileext'], $fI['path'], $fI['file']) || $this->allowPHPScripts && $GLOBALS['BE_USER']->isAdmin()) {
2736 if (GeneralUtility::getFileAbsFileName($fileName)) {
2737 if ($this->dat['files'][$fileID]) {
2738 GeneralUtility::writeFile($fileName, $this->dat['files'][$fileID]['content']);
2739 $this->fileIDMap[$fileID] = $fileName;
2740 if (md5(GeneralUtility::getUrl($fileName)) == $this->dat['files'][$fileID]['content_md5']) {
2741 return TRUE;
2742 } else {
2743 $this->error('ERROR: File content "' . $fileName . '" was corrupted');
2744 }
2745 } else {
2746 $this->error('ERROR: File ID "' . $fileID . '" could not be found');
2747 }
2748 } else {
2749 $this->error('ERROR: Filename "' . $fileName . '" was not a valid relative file path!');
2750 }
2751 } else {
2752 $this->error('ERROR: Filename "' . $fileName . '" failed against extension check or deny-pattern!');
2753 }
2754 } else {
2755 $this->error('ERROR: Filename "' . $fileName . '" was not allowed in destination path!');
2756 }
2757 } else {
2758 $this->error('ERROR: You did not have sufficient permissions to write the file "' . $fileName . '"');
2759 }
2760 }
2761
2762
2763 /**
2764 * Writes the file with the is $fileId to the legacy import folder. The file name will used from
2765 * argument $fileName and the file was successfully created or an identical file was already found,
2766 * $fileName will held the uid of the new created file record.
2767 *
2768 * @param $fileName The file name for the new file. Value would be changed to the uid of the new created file record.
2769 * @param $fileID The id of the file in data array
2770 * @return bool
2771 */
2772 protected function writeSysFileResourceForLegacyImport(&$fileName, $fileId) {
2773 if ($this->legacyImportFolder === NULL) {
2774 return FALSE;
2775 }
2776
2777 if (!isset($this->dat['files'][$fileId])) {
2778 $this->error('ERROR: File ID "' . $fileId . '" could not be found');
2779 return FALSE;
2780 }
2781
2782 $temporaryFile = $this->writeTemporaryFileFromData($fileId, 'files');
2783 if ($temporaryFile === NULL) {
2784 // error on writing the file. Error message was already added
2785 return FALSE;
2786 }
2787
2788 $importFolder = $this->legacyImportFolder;
2789
2790 if (isset($this->dat['files'][$fileId]['relFileName'])) {
2791 $relativeFilePath = PathUtility::dirname($this->dat['files'][$fileId]['relFileName']);
2792
2793 if (!$this->legacyImportFolder->hasFolder($relativeFilePath)) {
2794 $this->legacyImportFolder->createFolder($relativeFilePath);
2795 }
2796 $importFolder = $this->legacyImportFolder->getSubfolder($relativeFilePath);
2797 }
2798
2799 $fileObject = NULL;
2800
2801 try {
2802 // check, if there is alreay the same file in the folder
2803 if ($importFolder->hasFile($fileName)) {
2804 $fileStorage = $importFolder->getStorage();
2805 $file = $fileStorage->getFile($importFolder->getIdentifier() . $fileName);
2806 if ($file->getSha1() === sha1_file($temporaryFile)) {