[TASK] Mitigate erroring builds
[Packages/TYPO3.CMS.git] / typo3 / sysext / impexp / Classes / ImportExport.php
1 <?php
2 namespace TYPO3\CMS\Impexp;
3
4 /***************************************************************
5 * Copyright notice
6 *
7 * (c) 1999-2013 Kasper Skårhøj (kasperYYYY@typo3.com)
8 * All rights reserved
9 *
10 * This script is part of the TYPO3 project. The TYPO3 project is
11 * free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2 of the License, or
14 * (at your option) any later version.
15 *
16 * The GNU General Public License can be found at
17 * http://www.gnu.org/copyleft/gpl.html.
18 * A copy is found in the text file GPL.txt and important notices to the license
19 * from the author is found in LICENSE.txt distributed with these scripts.
20 *
21 *
22 * This script is distributed in the hope that it will be useful,
23 * but WITHOUT ANY WARRANTY; without even the implied warranty of
24 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25 * GNU General Public License for more details.
26 *
27 * This copyright notice MUST APPEAR in all copies of the script!
28 ***************************************************************/
29
30 use TYPO3\CMS\Core\Exception;
31 use TYPO3\CMS\Core\Utility\GeneralUtility;
32 use TYPO3\CMS\Backend\Utility\BackendUtility;
33
34 /**
35 * EXAMPLE for using the impexp-class for exporting stuff:
36 *
37 * Create and initialize:
38 * $this->export = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance('TYPO3\\CMS\\Impexp\\ImportExport');
39 * $this->export->init();
40 * Set which tables relations we will allow:
41 * $this->export->relOnlyTables[]="tt_news"; // exclusively includes. See comment in the class
42 *
43 * Adding records:
44 * $this->export->export_addRecord("pages", $this->pageinfo);
45 * $this->export->export_addRecord("pages", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("pages", 38));
46 * $this->export->export_addRecord("pages", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("pages", 39));
47 * $this->export->export_addRecord("tt_content", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("tt_content", 12));
48 * $this->export->export_addRecord("tt_content", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("tt_content", 74));
49 * $this->export->export_addRecord("sys_template", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("sys_template", 20));
50 *
51 * Adding all the relations (recursively in 5 levels so relations has THEIR relations registered as well)
52 * for($a=0;$a<5;$a++) {
53 * $addR = $this->export->export_addDBRelations($a);
54 * if (!count($addR)) break;
55 * }
56 *
57 * Finally load all the files.
58 * $this->export->export_addFilesFromRelations(); // MUST be after the DBrelations are set so that file from ALL added records are included!
59 *
60 * Write export
61 * $out = $this->export->compileMemoryToFileContent();
62 */
63 /**
64 * T3D file Import/Export library (TYPO3 Record Document)
65 *
66 * @author Kasper Skårhøj <kasperYYYY@typo3.com>
67 */
68 class ImportExport {
69
70 /**
71 * If set, static relations (not exported) will be shown in overview as well
72 *
73 * @todo Define visibility
74 */
75 public $showStaticRelations = FALSE;
76
77 /**
78 * Name of the "fileadmin" folder where files for export/import should be located
79 *
80 * @todo Define visibility
81 */
82 public $fileadminFolderName = '';
83
84 /**
85 * Whether "import" or "export" mode of object. Set through init() function
86 *
87 * @todo Define visibility
88 */
89 public $mode = '';
90
91 /**
92 * Updates all records that has same UID instead of creating new!
93 *
94 * @todo Define visibility
95 */
96 public $update = FALSE;
97
98 /**
99 * Is set by importData() when an import has been done.
100 *
101 * @todo Define visibility
102 */
103 public $doesImport = FALSE;
104
105 /**
106 * If set to a page-record, then the preview display of the content will expect this page-record to be the target
107 * for the import and accordingly display validation information. This triggers the visual view of the
108 * import/export memory to validate if import is possible
109 *
110 * @todo Define visibility
111 */
112 public $display_import_pid_record = '';
113
114 /**
115 * Used to register the forged UID values for imported records that we want to create with the same UIDs as in the import file. Admin-only feature.
116 *
117 * @todo Define visibility
118 */
119 public $suggestedInsertUids = array();
120
121 /**
122 * Setting import modes during update state: as_new, exclude, force_uid
123 *
124 * @todo Define visibility
125 */
126 public $import_mode = array();
127
128 /**
129 * If set, PID correct is ignored globally
130 *
131 * @todo Define visibility
132 */
133 public $global_ignore_pid = FALSE;
134
135 /**
136 * If set, all UID values are forced! (update or import)
137 *
138 * @todo Define visibility
139 */
140 public $force_all_UIDS = FALSE;
141
142 /**
143 * If set, a diff-view column is added to the overview.
144 *
145 * @todo Define visibility
146 */
147 public $showDiff = FALSE;
148
149 /**
150 * If set, and if the user is admin, allow the writing of PHP scripts to fileadmin/ area.
151 *
152 * @todo Define visibility
153 */
154 public $allowPHPScripts = FALSE;
155
156 /**
157 * Disable logging when importing
158 *
159 * @todo Define visibility
160 */
161 public $enableLogging = FALSE;
162
163 /**
164 * Array of values to substitute in editable softreferences.
165 *
166 * @todo Define visibility
167 */
168 public $softrefInputValues = array();
169
170 /**
171 * Mapping between the fileID from import memory and the final filenames they are written to.
172 *
173 * @todo Define visibility
174 */
175 public $fileIDMap = array();
176
177 /**
178 * 1MB max file size
179 *
180 * @todo Define visibility
181 */
182 public $maxFileSize = 1000000;
183
184 /**
185 * 1MB max record size
186 *
187 * @todo Define visibility
188 */
189 public $maxRecordSize = 1000000;
190
191 /**
192 * 10MB max export size
193 *
194 * @todo Define visibility
195 */
196 public $maxExportSize = 10000000;
197
198 /**
199 * Add table names here which are THE ONLY ones which will be included into export if found as relations. '_ALL' will allow all tables.
200 *
201 * @todo Define visibility
202 */
203 public $relOnlyTables = array();
204
205 /**
206 * Add tables names here which should not be exported with the file. (Where relations should be mapped to same UIDs in target system).
207 *
208 * @todo Define visibility
209 */
210 public $relStaticTables = array();
211
212 /**
213 * Exclude map. Keys are table:uid pairs and if set, records are not added to the export.
214 *
215 * @todo Define visibility
216 */
217 public $excludeMap = array();
218
219 /**
220 * Soft Reference Token ID modes.
221 *
222 * @todo Define visibility
223 */
224 public $softrefCfg = array();
225
226 /**
227 * Listing extension dependencies.
228 *
229 * @todo Define visibility
230 */
231 public $extensionDependencies = array();
232
233 /**
234 * Set by user: If set, compression in t3d files is disabled
235 *
236 * @todo Define visibility
237 */
238 public $dontCompress = 0;
239
240 /**
241 * Boolean, if set, HTML file resources are included.
242 *
243 * @todo Define visibility
244 */
245 public $includeExtFileResources = 0;
246
247 /**
248 * Files with external media (HTML/css style references inside)
249 *
250 * @todo Define visibility
251 */
252 public $extFileResourceExtensions = 'html,htm,css';
253
254 /**
255 * After records are written this array is filled with [table][original_uid] = [new_uid]
256 *
257 * @todo Define visibility
258 */
259 public $import_mapId = array();
260
261 /**
262 * Keys are [tablename]:[new NEWxxx ids (or when updating it is uids)] while values are arrays with table/uid of the original record it is based on.
263 * By the array keys the new ids can be looked up inside tcemain
264 *
265 * @todo Define visibility
266 */
267 public $import_newId = array();
268
269 /**
270 * Page id map for page tree (import)
271 *
272 * @todo Define visibility
273 */
274 public $import_newId_pids = array();
275
276 /**
277 * Internal data accumulation for writing records during import
278 *
279 * @todo Define visibility
280 */
281 public $import_data = array();
282
283 /**
284 * Error log.
285 *
286 * @todo Define visibility
287 */
288 public $errorLog = array();
289
290 /**
291 * Cache for record paths
292 *
293 * @todo Define visibility
294 */
295 public $cache_getRecordPath = array();
296
297 /**
298 * Cache of checkPID values.
299 *
300 * @todo Define visibility
301 */
302 public $checkPID_cache = array();
303
304 /**
305 * Set internally if the gzcompress function exists
306 *
307 * @todo Define visibility
308 */
309 public $compress = 0;
310
311 /**
312 * Internal import/export memory
313 *
314 * @todo Define visibility
315 */
316 public $dat = array();
317
318 /**
319 * File processing object
320 *
321 * @var \TYPO3\CMS\Core\Utility\File\ExtendedFileUtility
322 * @todo Define visibility
323 */
324 public $fileProcObj = '';
325
326 /**
327 * Keys are [recordname], values are an array of fields to be included
328 * in the export
329 *
330 * @var array
331 */
332 protected $recordTypesIncludeFields = array();
333
334 /**
335 * Default array of fields to be included in the export
336 *
337 * @var array
338 */
339 protected $defaultRecordIncludeFields = array('uid', 'pid');
340
341 /**
342 * Array of current registered storage objects
343 *
344 * @var array
345 */
346 protected $storageObjects = array();
347
348 /**************************
349 * Initialize
350 *************************/
351
352 /**
353 * Init the object, both import and export
354 *
355 * @param boolean $dontCompress If set, compression in t3d files is disabled
356 * @param string $mode Mode of usage, either "import" or "export
357 * @return void
358 * @todo Define visibility
359 */
360 public function init($dontCompress = 0, $mode = '') {
361 $this->compress = function_exists('gzcompress');
362 $this->dontCompress = $dontCompress;
363 $this->mode = $mode;
364 $this->fileadminFolderName = !empty($GLOBALS['TYPO3_CONF_VARS']['BE']['fileadminDir']) ? rtrim($GLOBALS['TYPO3_CONF_VARS']['BE']['fileadminDir'], '/') : 'fileadmin';
365 }
366
367 /**************************
368 * Export / Init + Meta Data
369 *************************/
370
371 /**
372 * Set header basics
373 *
374 * @return void
375 * @todo Define visibility
376 */
377 public function setHeaderBasics() {
378 // Initializing:
379 if (is_array($this->softrefCfg)) {
380 foreach ($this->softrefCfg as $key => $value) {
381 if (!strlen($value['mode'])) {
382 unset($this->softrefCfg[$key]);
383 }
384 }
385 }
386 // Setting in header memory:
387 // Version of file format
388 $this->dat['header']['XMLversion'] = '1.0';
389 // Initialize meta data array (to put it in top of file)
390 $this->dat['header']['meta'] = array();
391 // Add list of tables to consider static
392 $this->dat['header']['relStaticTables'] = $this->relStaticTables;
393 // The list of excluded records
394 $this->dat['header']['excludeMap'] = $this->excludeMap;
395 // Soft Reference mode for elements
396 $this->dat['header']['softrefCfg'] = $this->softrefCfg;
397 // List of extensions the import depends on.
398 $this->dat['header']['extensionDependencies'] = $this->extensionDependencies;
399 }
400
401 /**
402 * Set charset
403 *
404 * @param string $charset Charset for the content in the export. During import the character set will be converted if the target system uses another charset.
405 * @return void
406 * @todo Define visibility
407 */
408 public function setCharset($charset) {
409 $this->dat['header']['charset'] = $charset;
410 }
411
412 /**
413 * Sets meta data
414 *
415 * @param string $title Title of the export
416 * @param string $description Description of the export
417 * @param string $notes Notes about the contents
418 * @param string $packager_username Backend Username of the packager (the guy making the export)
419 * @param string $packager_name Real name of the packager
420 * @param string $packager_email Email of the packager
421 * @return void
422 * @todo Define visibility
423 */
424 public function setMetaData($title, $description, $notes, $packager_username, $packager_name, $packager_email) {
425 $this->dat['header']['meta'] = array(
426 'title' => $title,
427 'description' => $description,
428 'notes' => $notes,
429 'packager_username' => $packager_username,
430 'packager_name' => $packager_name,
431 'packager_email' => $packager_email,
432 'TYPO3_version' => TYPO3_version,
433 'created' => strftime('%A %e. %B %Y', $GLOBALS['EXEC_TIME'])
434 );
435 }
436
437 /**
438 * Sets a thumbnail image to the exported file
439 *
440 * @param string $imgFilepath Filename reference, gif, jpg, png. Absolute path.
441 * @return void
442 * @todo Define visibility
443 */
444 public function addThumbnail($imgFilepath) {
445 if (@is_file($imgFilepath)) {
446 $imgInfo = @getimagesize($imgFilepath);
447 if (is_array($imgInfo)) {
448 $fileContent = GeneralUtility::getUrl($imgFilepath);
449 $this->dat['header']['thumbnail'] = array(
450 'imgInfo' => $imgInfo,
451 'content' => $fileContent,
452 'filesize' => strlen($fileContent),
453 'filemtime' => filemtime($imgFilepath),
454 'filename' => basename($imgFilepath)
455 );
456 }
457 }
458 }
459
460 /**************************
461 * Export / Init Page tree
462 *************************/
463
464 /**
465 * Sets the page-tree array in the export header and returns the array in a flattened version
466 *
467 * @param array $idH Hierarchy of ids, the page tree: array([uid] => array("uid" => [uid], "subrow" => array(.....)), [uid] => ....)
468 * @return array The hierarchical page tree converted to a one-dimensional list of pages
469 * @todo Define visibility
470 */
471 public function setPageTree($idH) {
472 $this->dat['header']['pagetree'] = $this->unsetExcludedSections($idH);
473 return $this->flatInversePageTree($this->dat['header']['pagetree']);
474 }
475
476 /**
477 * Removes entries in the page tree which are found in ->excludeMap[]
478 *
479 * @param array $idH Page uid hierarchy
480 * @return array Modified input array
481 * @access private
482 * @see setPageTree()
483 * @todo Define visibility
484 */
485 public function unsetExcludedSections($idH) {
486 if (is_array($idH)) {
487 foreach ($idH as $k => $v) {
488 if ($this->excludeMap['pages:' . $idH[$k]['uid']]) {
489 unset($idH[$k]);
490 } elseif (is_array($idH[$k]['subrow'])) {
491 $idH[$k]['subrow'] = $this->unsetExcludedSections($idH[$k]['subrow']);
492 }
493 }
494 }
495 return $idH;
496 }
497
498 /**
499 * Recursively flattening the idH array (for setPageTree() function)
500 *
501 * @param array $idH Page uid hierarchy
502 * @param array $a Accumulation array of pages (internal, don't set from outside)
503 * @return array Array with uid-uid pairs for all pages in the page tree.
504 * @see flatInversePageTree_pid()
505 * @todo Define visibility
506 */
507 public function flatInversePageTree($idH, $a = array()) {
508 if (is_array($idH)) {
509 $idH = array_reverse($idH);
510 foreach ($idH as $k => $v) {
511 $a[$v['uid']] = $v['uid'];
512 if (is_array($v['subrow'])) {
513 $a = $this->flatInversePageTree($v['subrow'], $a);
514 }
515 }
516 }
517 return $a;
518 }
519
520 /**
521 * Recursively flattening the idH array (for setPageTree() function), setting PIDs as values
522 *
523 * @param array $idH Page uid hierarchy
524 * @param array $a Accumulation array of pages (internal, don't set from outside)
525 * @param integer $pid PID value (internal)
526 * @return array Array with uid-pid pairs for all pages in the page tree.
527 * @see flatInversePageTree()
528 * @todo Define visibility
529 */
530 public function flatInversePageTree_pid($idH, $a = array(), $pid = -1) {
531 if (is_array($idH)) {
532 $idH = array_reverse($idH);
533 foreach ($idH as $k => $v) {
534 $a[$v['uid']] = $pid;
535 if (is_array($v['subrow'])) {
536 $a = $this->flatInversePageTree_pid($v['subrow'], $a, $v['uid']);
537 }
538 }
539 }
540 return $a;
541 }
542
543 /**************************
544 * Export
545 *************************/
546
547 /**
548 * Sets the fields of record types to be included in the export
549 *
550 * @param array $recordTypesIncludeFields Keys are [recordname], values are an array of fields to be included in the export
551 * @throws \TYPO3\CMS\Core\Exception if an array value is not type of array
552 * @return void
553 */
554 public function setRecordTypesIncludeFields(array $recordTypesIncludeFields) {
555 foreach ($recordTypesIncludeFields as $table => $fields) {
556 if (!is_array($fields)) {
557 throw new \TYPO3\CMS\Core\Exception('The include fields for record type ' . htmlspecialchars($table) . ' are not defined by an array.', 1391440658);
558 }
559 $this->setRecordTypeIncludeFields($table, $fields);
560 }
561 }
562
563 /**
564 * Sets the fields of a record type to be included in the export
565 *
566 * @param string $table The record type
567 * @param array $fields The fields to be included
568 * @return void
569 */
570 public function setRecordTypeIncludeFields($table, array $fields) {
571 $this->recordTypesIncludeFields[$table] = $fields;
572 }
573
574 /**
575 * Adds the record $row from $table.
576 * No checking for relations done here. Pure data.
577 *
578 * @param string $table Table name
579 * @param array $row Record row.
580 * @param integer $relationLevel (Internal) if the record is added as a relation, this is set to the "level" it was on.
581 * @return void
582 * @todo Define visibility
583 */
584 public function export_addRecord($table, $row, $relationLevel = 0) {
585 BackendUtility::workspaceOL($table, $row);
586 if ((string)$table !== '' && is_array($row) && $row['uid'] > 0 && !$this->excludeMap[($table . ':' . $row['uid'])]) {
587 if ($this->checkPID($table === 'pages' ? $row['uid'] : $row['pid'])) {
588 if (!isset($this->dat['records'][($table . ':' . $row['uid'])])) {
589 // Prepare header info:
590 $row = $this->filterRecordFields($table, $row);
591 $headerInfo = array();
592 $headerInfo['uid'] = $row['uid'];
593 $headerInfo['pid'] = $row['pid'];
594 $headerInfo['title'] = GeneralUtility::fixed_lgd_cs(BackendUtility::getRecordTitle($table, $row), 40);
595 $headerInfo['size'] = strlen(serialize($row));
596 if ($relationLevel) {
597 $headerInfo['relationLevel'] = $relationLevel;
598 }
599 // If record content is not too large in size, set the header content and add the rest:
600 if ($headerInfo['size'] < $this->maxRecordSize) {
601 // Set the header summary:
602 $this->dat['header']['records'][$table][$row['uid']] = $headerInfo;
603 // Create entry in the PID lookup:
604 $this->dat['header']['pid_lookup'][$row['pid']][$table][$row['uid']] = 1;
605 // Initialize reference index object:
606 $refIndexObj = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\Database\\ReferenceIndex');
607 // Yes to workspace overlays for exporting....
608 $refIndexObj->WSOL = TRUE;
609 $relations = $refIndexObj->getRelations($table, $row);
610 $relations = $this->fixFileIDsInRelations($relations);
611 // Data:
612 $this->dat['records'][$table . ':' . $row['uid']] = array();
613 $this->dat['records'][$table . ':' . $row['uid']]['data'] = $row;
614 $this->dat['records'][$table . ':' . $row['uid']]['rels'] = $relations;
615 $this->errorLog = array_merge($this->errorLog, $refIndexObj->errorLog);
616 // Merge error logs.
617 // Add information about the relations in the record in the header:
618 $this->dat['header']['records'][$table][$row['uid']]['rels'] = $this->flatDBrels($this->dat['records'][$table . ':' . $row['uid']]['rels']);
619 // Add information about the softrefs to header:
620 $this->dat['header']['records'][$table][$row['uid']]['softrefs'] = $this->flatSoftRefs($this->dat['records'][$table . ':' . $row['uid']]['rels']);
621 } else {
622 $this->error('Record ' . $table . ':' . $row['uid'] . ' was larger than maxRecordSize (' . GeneralUtility::formatSize($this->maxRecordSize) . ')');
623 }
624 } else {
625 $this->error('Record ' . $table . ':' . $row['uid'] . ' already added.');
626 }
627 } else {
628 $this->error('Record ' . $table . ':' . $row['uid'] . ' was outside your DB mounts!');
629 }
630 }
631 }
632
633 /**
634 * This changes the file reference ID from a hash based on the absolute file path
635 * (coming from ReferenceIndex) to a hash based on the relative file path.
636 *
637 * @param array $relations
638 * @return array
639 */
640 protected function fixFileIDsInRelations(array $relations) {
641 foreach ($relations as $field => $relation) {
642 if (isset($relation['type']) && $relation['type'] === 'file') {
643 foreach ($relation['newValueFiles'] as $key => $fileRelationData) {
644 $absoluteFilePath = $fileRelationData['ID_absFile'];
645 if (GeneralUtility::isFirstPartOfStr($absoluteFilePath, PATH_site)) {
646 $relatedFilePath = \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($absoluteFilePath);
647 $relations[$field]['newValueFiles'][$key]['ID'] = md5($relatedFilePath);
648 }
649 }
650 }
651 if ($relation['type'] == 'flex') {
652 if (is_array($relation['flexFormRels']['file'])) {
653 foreach ($relation['flexFormRels']['file'] as $key => $subList) {
654 foreach ($subList as $subKey => $fileRelationData) {
655 $absoluteFilePath = $fileRelationData['ID_absFile'];
656 if (GeneralUtility::isFirstPartOfStr($absoluteFilePath, PATH_site)) {
657 $relatedFilePath = \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($absoluteFilePath);
658 $relations[$field]['flexFormRels']['file'][$key][$subKey]['ID'] = md5($relatedFilePath);
659 }
660 }
661 }
662 }
663 }
664 }
665 return $relations;
666 }
667
668
669
670 /**
671 * This analyses the existing added records, finds all database relations to records and adds these records to the export file.
672 * This function can be called repeatedly until it returns an empty array. In principle it should not allow to infinite recursivity, but you better set a limit...
673 * Call this BEFORE the ext_addFilesFromRelations (so files from added relations are also included of course)
674 *
675 * @param integer $relationLevel Recursion level
676 * @return array overview of relations found and added: Keys [table]:[uid], values array with table and id
677 * @see export_addFilesFromRelations()
678 * @todo Define visibility
679 */
680 public function export_addDBRelations($relationLevel = 0) {
681 // Initialize:
682 $addR = array();
683 // Traverse all "rels" registered for "records"
684 if (is_array($this->dat['records'])) {
685 foreach ($this->dat['records'] as $k => $value) {
686 if (is_array($this->dat['records'][$k])) {
687 foreach ($this->dat['records'][$k]['rels'] as $fieldname => $vR) {
688 // For all DB types of relations:
689 if ($vR['type'] == 'db') {
690 foreach ($vR['itemArray'] as $fI) {
691 $this->export_addDBRelations_registerRelation($fI, $addR);
692 }
693 }
694 // For all flex/db types of relations:
695 if ($vR['type'] == 'flex') {
696 // DB relations in flex form fields:
697 if (is_array($vR['flexFormRels']['db'])) {
698 foreach ($vR['flexFormRels']['db'] as $subList) {
699 foreach ($subList as $fI) {
700 $this->export_addDBRelations_registerRelation($fI, $addR);
701 }
702 }
703 }
704 // DB oriented soft references in flex form fields:
705 if (is_array($vR['flexFormRels']['softrefs'])) {
706 foreach ($vR['flexFormRels']['softrefs'] as $subList) {
707 foreach ($subList['keys'] as $spKey => $elements) {
708 foreach ($elements as $el) {
709 if ($el['subst']['type'] === 'db' && $this->includeSoftref($el['subst']['tokenID'])) {
710 list($tempTable, $tempUid) = explode(':', $el['subst']['recordRef']);
711 $fI = array(
712 'table' => $tempTable,
713 'id' => $tempUid
714 );
715 $this->export_addDBRelations_registerRelation($fI, $addR, $el['subst']['tokenID']);
716 }
717 }
718 }
719 }
720 }
721 }
722 // In any case, if there are soft refs:
723 if (is_array($vR['softrefs']['keys'])) {
724 foreach ($vR['softrefs']['keys'] as $spKey => $elements) {
725 foreach ($elements as $el) {
726 if ($el['subst']['type'] === 'db' && $this->includeSoftref($el['subst']['tokenID'])) {
727 list($tempTable, $tempUid) = explode(':', $el['subst']['recordRef']);
728 $fI = array(
729 'table' => $tempTable,
730 'id' => $tempUid
731 );
732 $this->export_addDBRelations_registerRelation($fI, $addR, $el['subst']['tokenID']);
733 }
734 }
735 }
736 }
737 }
738 }
739 }
740 } else {
741 $this->error('There were no records available.');
742 }
743 // Now, if there were new records to add, do so:
744 if (count($addR)) {
745 foreach ($addR as $fI) {
746 // Get and set record:
747 $row = BackendUtility::getRecord($fI['table'], $fI['id']);
748 if (is_array($row)) {
749 $this->export_addRecord($fI['table'], $row, $relationLevel + 1);
750 }
751 // Set status message
752 // Relation pointers always larger than zero except certain "select" types with negative values pointing to uids - but that is not supported here.
753 if ($fI['id'] > 0) {
754 $rId = $fI['table'] . ':' . $fI['id'];
755 if (!isset($this->dat['records'][$rId])) {
756 $this->dat['records'][$rId] = 'NOT_FOUND';
757 $this->error('Relation record ' . $rId . ' was not found!');
758 }
759 }
760 }
761 }
762 // Return overview of relations found and added
763 return $addR;
764 }
765
766 /**
767 * Helper function for export_addDBRelations()
768 *
769 * @param array $fI Array with table/id keys to add
770 * @param array $addR Add array, passed by reference to be modified
771 * @param string $tokenID Softref Token ID, if applicable.
772 * @return void
773 * @see export_addDBRelations()
774 * @todo Define visibility
775 */
776 public function export_addDBRelations_registerRelation($fI, &$addR, $tokenID = '') {
777 $rId = $fI['table'] . ':' . $fI['id'];
778 if (isset($GLOBALS['TCA'][$fI['table']]) && !$this->isTableStatic($fI['table']) && !$this->isExcluded($fI['table'], $fI['id']) && (!$tokenID || $this->includeSoftref($tokenID)) && $this->inclRelation($fI['table'])) {
779 if (!isset($this->dat['records'][$rId])) {
780 // Set this record to be included since it is not already.
781 $addR[$rId] = $fI;
782 }
783 }
784 }
785
786 /**
787 * This adds all files in relations.
788 * Call this method AFTER adding all records including relations.
789 *
790 * @return void
791 * @see export_addDBRelations()
792 * @todo Define visibility
793 */
794 public function export_addFilesFromRelations() {
795 // Traverse all "rels" registered for "records"
796 if (is_array($this->dat['records'])) {
797 foreach ($this->dat['records'] as $k => $value) {
798 if (isset($this->dat['records'][$k]['rels']) && is_array($this->dat['records'][$k]['rels'])) {
799 foreach ($this->dat['records'][$k]['rels'] as $fieldname => $vR) {
800 // For all file type relations:
801 if ($vR['type'] == 'file') {
802 foreach ($vR['newValueFiles'] as $key => $fI) {
803 $this->export_addFile($fI, $k, $fieldname);
804 // Remove the absolute reference to the file so it doesn't expose absolute paths from source server:
805 unset($this->dat['records'][$k]['rels'][$fieldname]['newValueFiles'][$key]['ID_absFile']);
806 }
807 }
808 // For all flex type relations:
809 if ($vR['type'] == 'flex') {
810 if (is_array($vR['flexFormRels']['file'])) {
811 foreach ($vR['flexFormRels']['file'] as $key => $subList) {
812 foreach ($subList as $subKey => $fI) {
813 $this->export_addFile($fI, $k, $fieldname);
814 // Remove the absolute reference to the file so it doesn't expose absolute paths from source server:
815 unset($this->dat['records'][$k]['rels'][$fieldname]['flexFormRels']['file'][$key][$subKey]['ID_absFile']);
816 }
817 }
818 }
819 // DB oriented soft references in flex form fields:
820 if (is_array($vR['flexFormRels']['softrefs'])) {
821 foreach ($vR['flexFormRels']['softrefs'] as $key => $subList) {
822 foreach ($subList['keys'] as $spKey => $elements) {
823 foreach ($elements as $subKey => $el) {
824 if ($el['subst']['type'] === 'file' && $this->includeSoftref($el['subst']['tokenID'])) {
825 // Create abs path and ID for file:
826 $ID_absFile = GeneralUtility::getFileAbsFileName(PATH_site . $el['subst']['relFileName']);
827 $ID = md5($el['subst']['relFileName']);
828 if ($ID_absFile) {
829 if (!$this->dat['files'][$ID]) {
830 $fI = array(
831 'filename' => basename($ID_absFile),
832 'ID_absFile' => $ID_absFile,
833 'ID' => $ID,
834 'relFileName' => $el['subst']['relFileName']
835 );
836 $this->export_addFile($fI, '_SOFTREF_');
837 }
838 $this->dat['records'][$k]['rels'][$fieldname]['flexFormRels']['softrefs'][$key]['keys'][$spKey][$subKey]['file_ID'] = $ID;
839 }
840 }
841 }
842 }
843 }
844 }
845 }
846 // In any case, if there are soft refs:
847 if (is_array($vR['softrefs']['keys'])) {
848 foreach ($vR['softrefs']['keys'] as $spKey => $elements) {
849 foreach ($elements as $subKey => $el) {
850 if ($el['subst']['type'] === 'file' && $this->includeSoftref($el['subst']['tokenID'])) {
851 // Create abs path and ID for file:
852 $ID_absFile = GeneralUtility::getFileAbsFileName(PATH_site . $el['subst']['relFileName']);
853 $ID = md5($el['subst']['relFileName']);
854 if ($ID_absFile) {
855 if (!$this->dat['files'][$ID]) {
856 $fI = array(
857 'filename' => basename($ID_absFile),
858 'ID_absFile' => $ID_absFile,
859 'ID' => $ID,
860 'relFileName' => $el['subst']['relFileName']
861 );
862 $this->export_addFile($fI, '_SOFTREF_');
863 }
864 $this->dat['records'][$k]['rels'][$fieldname]['softrefs']['keys'][$spKey][$subKey]['file_ID'] = $ID;
865 }
866 }
867 }
868 }
869 }
870 }
871 }
872 }
873 } else {
874 $this->error('There were no records available.');
875 }
876 }
877
878 /**
879 * This adds all files from sys_file records
880 *
881 * @return void
882 */
883 public function export_addFilesFromSysFilesRecords() {
884 if (!isset($this->dat['header']['records']['sys_file']) || !is_array($this->dat['header']['records']['sys_file'])) {
885 return;
886 }
887 foreach (array_keys($this->dat['header']['records']['sys_file']) as $sysFileUid) {
888 $recordData = $this->dat['records']['sys_file:' . $sysFileUid]['data'];
889 $file = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->createFileObject($recordData);
890 $this->export_addSysFile($file);
891 }
892 }
893
894 /**
895 * Adds a files content from a sys file record to the export memory
896 *
897 * @param \TYPO3\CMS\Core\Resource\File $file
898 * @return void
899 */
900 public function export_addSysFile(\TYPO3\CMS\Core\Resource\File $file) {
901 if ($file->getProperty('size') >= $this->maxFileSize) {
902 $this->error('File ' . $file->getPublicUrl() . ' was larger (' . GeneralUtility::formatSize($file->getProperty('size')) . ') than the maxFileSize (' . GeneralUtility::formatSize($this->maxFileSize) . ')! Skipping.');
903 return;
904 }
905 try {
906 $fileContent = $file->getContents();
907 } catch (\TYPO3\CMS\Core\Resource\Exception\InsufficientFileAccessPermissionsException $e) {
908 $this->error('File ' . $file->getPublicUrl() . ': ' . $e->getMessage());
909 return;
910 } catch (\TYPO3\CMS\Core\Resource\Exception\IllegalFileExtensionException $e) {
911 $this->error('File ' . $file->getPublicUrl() . ': ' . $e->getMessage());
912 return;
913 }
914 $fileRec = array();
915 $fileRec['filesize'] = $file->getProperty('size');
916 $fileRec['filename'] = $file->getProperty('name');
917 $fileRec['filemtime'] = $file->getProperty('modification_date');
918
919 // build unique id based on the storage and the file identifier
920 $fileId = md5($file->getStorage()->getUid() . ':' . $file->getProperty('identifier_hash'));
921
922 // Setting this data in the header
923 $this->dat['header']['files_fal'][$fileId] = $fileRec;
924
925 // ... and finally add the heavy stuff:
926 $fileRec['content'] = $fileContent;
927 $fileRec['content_sha1'] = $file->getProperty('sha1');
928
929 $this->dat['files_fal'][$fileId] = $fileRec;
930 }
931
932
933 /**
934 * Adds a files content to the export memory
935 *
936 * @param array $fI File information with three keys: "filename" = filename without path, "ID_absFile" = absolute filepath to the file (including the filename), "ID" = md5 hash of "ID_absFile". "relFileName" is optional for files attached to records, but mandatory for soft referenced files (since the relFileName determines where such a file should be stored!)
937 * @param string $recordRef If the file is related to a record, this is the id on the form [table]:[id]. Information purposes only.
938 * @param string $fieldname If the file is related to a record, this is the field name it was related to. Information purposes only.
939 * @return void
940 * @todo Define visibility
941 */
942 public function export_addFile($fI, $recordRef = '', $fieldname = '') {
943 if (@is_file($fI['ID_absFile'])) {
944 if (filesize($fI['ID_absFile']) < $this->maxFileSize) {
945 $fileInfo = stat($fI['ID_absFile']);
946 $fileRec = array();
947 $fileRec['filesize'] = $fileInfo['size'];
948 $fileRec['filename'] = basename($fI['ID_absFile']);
949 $fileRec['filemtime'] = $fileInfo['mtime'];
950 //for internal type file_reference
951 $fileRec['relFileRef'] = \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($fI['ID_absFile']);
952 if ($recordRef) {
953 $fileRec['record_ref'] = $recordRef . '/' . $fieldname;
954 }
955 if ($fI['relFileName']) {
956 $fileRec['relFileName'] = $fI['relFileName'];
957 }
958 // Setting this data in the header
959 $this->dat['header']['files'][$fI['ID']] = $fileRec;
960 // ... and for the recordlisting, why not let us know WHICH relations there was...
961 if ($recordRef && $recordRef !== '_SOFTREF_') {
962 $refParts = explode(':', $recordRef, 2);
963 if (!is_array($this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'])) {
964 $this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'] = array();
965 }
966 $this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'][] = $fI['ID'];
967 }
968 // ... and finally add the heavy stuff:
969 $fileRec['content'] = GeneralUtility::getUrl($fI['ID_absFile']);
970 $fileRec['content_md5'] = md5($fileRec['content']);
971 $this->dat['files'][$fI['ID']] = $fileRec;
972 // For soft references, do further processing:
973 if ($recordRef === '_SOFTREF_') {
974 // RTE files?
975 if ($RTEoriginal = $this->getRTEoriginalFilename(basename($fI['ID_absFile']))) {
976 $RTEoriginal_absPath = dirname($fI['ID_absFile']) . '/' . $RTEoriginal;
977 if (@is_file($RTEoriginal_absPath)) {
978 $RTEoriginal_ID = md5($RTEoriginal_absPath);
979 $fileInfo = stat($RTEoriginal_absPath);
980 $fileRec = array();
981 $fileRec['filesize'] = $fileInfo['size'];
982 $fileRec['filename'] = basename($RTEoriginal_absPath);
983 $fileRec['filemtime'] = $fileInfo['mtime'];
984 $fileRec['record_ref'] = '_RTE_COPY_ID:' . $fI['ID'];
985 $this->dat['header']['files'][$fI['ID']]['RTE_ORIG_ID'] = $RTEoriginal_ID;
986 // Setting this data in the header
987 $this->dat['header']['files'][$RTEoriginal_ID] = $fileRec;
988 // ... and finally add the heavy stuff:
989 $fileRec['content'] = GeneralUtility::getUrl($RTEoriginal_absPath);
990 $fileRec['content_md5'] = md5($fileRec['content']);
991 $this->dat['files'][$RTEoriginal_ID] = $fileRec;
992 } else {
993 $this->error('RTE original file "' . \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($RTEoriginal_absPath) . '" was not found!');
994 }
995 }
996 // Files with external media?
997 // This is only done with files grabbed by a softreference parser since it is deemed improbable that hard-referenced files should undergo this treatment.
998 $html_fI = pathinfo(basename($fI['ID_absFile']));
999 if ($this->includeExtFileResources && GeneralUtility::inList($this->extFileResourceExtensions, strtolower($html_fI['extension']))) {
1000 $uniquePrefix = '###' . md5($GLOBALS['EXEC_TIME']) . '###';
1001 if (strtolower($html_fI['extension']) === 'css') {
1002 $prefixedMedias = explode($uniquePrefix, preg_replace('/(url[[:space:]]*\\([[:space:]]*["\']?)([^"\')]*)(["\']?[[:space:]]*\\))/i', '\\1' . $uniquePrefix . '\\2' . $uniquePrefix . '\\3', $fileRec['content']));
1003 } else {
1004 // html, htm:
1005 $htmlParser = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\Html\\HtmlParser');
1006 $prefixedMedias = explode($uniquePrefix, $htmlParser->prefixResourcePath($uniquePrefix, $fileRec['content'], array(), $uniquePrefix));
1007 }
1008 $htmlResourceCaptured = FALSE;
1009 foreach ($prefixedMedias as $k => $v) {
1010 if ($k % 2) {
1011 $EXTres_absPath = GeneralUtility::resolveBackPath(dirname($fI['ID_absFile']) . '/' . $v);
1012 $EXTres_absPath = GeneralUtility::getFileAbsFileName($EXTres_absPath);
1013 if ($EXTres_absPath && GeneralUtility::isFirstPartOfStr($EXTres_absPath, PATH_site . $this->fileadminFolderName . '/') && @is_file($EXTres_absPath)) {
1014 $htmlResourceCaptured = TRUE;
1015 $EXTres_ID = md5($EXTres_absPath);
1016 $this->dat['header']['files'][$fI['ID']]['EXT_RES_ID'][] = $EXTres_ID;
1017 $prefixedMedias[$k] = '{EXT_RES_ID:' . $EXTres_ID . '}';
1018 // Add file to memory if it is not set already:
1019 if (!isset($this->dat['header']['files'][$EXTres_ID])) {
1020 $fileInfo = stat($EXTres_absPath);
1021 $fileRec = array();
1022 $fileRec['filesize'] = $fileInfo['size'];
1023 $fileRec['filename'] = basename($EXTres_absPath);
1024 $fileRec['filemtime'] = $fileInfo['mtime'];
1025 $fileRec['record_ref'] = '_EXT_PARENT_:' . $fI['ID'];
1026 // Media relative to the HTML file.
1027 $fileRec['parentRelFileName'] = $v;
1028 // Setting this data in the header
1029 $this->dat['header']['files'][$EXTres_ID] = $fileRec;
1030 // ... and finally add the heavy stuff:
1031 $fileRec['content'] = GeneralUtility::getUrl($EXTres_absPath);
1032 $fileRec['content_md5'] = md5($fileRec['content']);
1033 $this->dat['files'][$EXTres_ID] = $fileRec;
1034 }
1035 }
1036 }
1037 }
1038 if ($htmlResourceCaptured) {
1039 $this->dat['files'][$fI['ID']]['tokenizedContent'] = implode('', $prefixedMedias);
1040 }
1041 }
1042 }
1043 } else {
1044 $this->error($fI['ID_absFile'] . ' was larger (' . GeneralUtility::formatSize(filesize($fI['ID_absFile'])) . ') than the maxFileSize (' . GeneralUtility::formatSize($this->maxFileSize) . ')! Skipping.');
1045 }
1046 } else {
1047 $this->error($fI['ID_absFile'] . ' was not a file! Skipping.');
1048 }
1049 }
1050
1051 /**
1052 * DB relations flattend to 1-dim array.
1053 * The list will be unique, no table/uid combination will appear twice.
1054 *
1055 * @param array $dbrels 2-dim Array of database relations organized by table key
1056 * @return array 1-dim array where entries are table:uid and keys are array with table/id
1057 * @todo Define visibility
1058 */
1059 public function flatDBrels($dbrels) {
1060 $list = array();
1061 foreach ($dbrels as $dat) {
1062 if ($dat['type'] == 'db') {
1063 foreach ($dat['itemArray'] as $i) {
1064 $list[$i['table'] . ':' . $i['id']] = $i;
1065 }
1066 }
1067 if ($dat['type'] == 'flex' && is_array($dat['flexFormRels']['db'])) {
1068 foreach ($dat['flexFormRels']['db'] as $subList) {
1069 foreach ($subList as $i) {
1070 $list[$i['table'] . ':' . $i['id']] = $i;
1071 }
1072 }
1073 }
1074 }
1075 return $list;
1076 }
1077
1078 /**
1079 * Soft References flattend to 1-dim array.
1080 *
1081 * @param array $dbrels 2-dim Array of database relations organized by table key
1082 * @return array 1-dim array where entries are arrays with properties of the soft link found and keys are a unique combination of field, spKey, structure path if applicable and token ID
1083 * @todo Define visibility
1084 */
1085 public function flatSoftRefs($dbrels) {
1086 $list = array();
1087 foreach ($dbrels as $field => $dat) {
1088 if (is_array($dat['softrefs']['keys'])) {
1089 foreach ($dat['softrefs']['keys'] as $spKey => $elements) {
1090 if (is_array($elements)) {
1091 foreach ($elements as $subKey => $el) {
1092 $lKey = $field . ':' . $spKey . ':' . $subKey;
1093 $list[$lKey] = array_merge(array('field' => $field, 'spKey' => $spKey), $el);
1094 // Add file_ID key to header - slightly "risky" way of doing this because if the calculation changes for the same value in $this->records[...] this will not work anymore!
1095 if ($el['subst'] && $el['subst']['relFileName']) {
1096 $list[$lKey]['file_ID'] = md5(PATH_site . $el['subst']['relFileName']);
1097 }
1098 }
1099 }
1100 }
1101 }
1102 if ($dat['type'] == 'flex' && is_array($dat['flexFormRels']['softrefs'])) {
1103 foreach ($dat['flexFormRels']['softrefs'] as $structurePath => $subSoftrefs) {
1104 if (is_array($subSoftrefs['keys'])) {
1105 foreach ($subSoftrefs['keys'] as $spKey => $elements) {
1106 foreach ($elements as $subKey => $el) {
1107 $lKey = $field . ':' . $structurePath . ':' . $spKey . ':' . $subKey;
1108 $list[$lKey] = array_merge(array('field' => $field, 'spKey' => $spKey, 'structurePath' => $structurePath), $el);
1109 // Add file_ID key to header - slightly "risky" way of doing this because if the calculation changes for the same value in $this->records[...] this will not work anymore!
1110 if ($el['subst'] && $el['subst']['relFileName']) {
1111 $list[$lKey]['file_ID'] = md5(PATH_site . $el['subst']['relFileName']);
1112 }
1113 }
1114 }
1115 }
1116 }
1117 }
1118 }
1119 return $list;
1120 }
1121
1122 /**
1123 * If include fields for a specific record type are set, the data
1124 * are filtered out with fields are not included in the fields.
1125 *
1126 * @param string $table The record type to be filtered
1127 * @param array $row The data to be filtered
1128 * @return array The filtered record row
1129 */
1130 protected function filterRecordFields($table, array $row) {
1131 if (isset($this->recordTypesIncludeFields[$table])) {
1132 $includeFields = array_unique(array_merge(
1133 $this->recordTypesIncludeFields[$table],
1134 $this->defaultRecordIncludeFields
1135 ));
1136 $newRow = array();
1137 foreach ($row as $key => $value) {
1138 if (in_array($key, $includeFields)) {
1139 $newRow[$key] = $value;
1140 }
1141 }
1142 } else {
1143 $newRow = $row;
1144 }
1145 return $newRow;
1146 }
1147
1148
1149 /**************************
1150 * File Output
1151 *************************/
1152
1153 /**
1154 * This compiles and returns the data content for an exported file
1155 *
1156 * @param string $type Type of output; "xml" gives xml, otherwise serialized array, possibly compressed.
1157 * @return string The output file stream
1158 * @todo Define visibility
1159 */
1160 public function compileMemoryToFileContent($type = '') {
1161 if ($type == 'xml') {
1162 $out = $this->createXML();
1163 } else {
1164 $compress = $this->doOutputCompress();
1165 $out = '';
1166 // adding header:
1167 $out .= $this->addFilePart(serialize($this->dat['header']), $compress);
1168 // adding records:
1169 $out .= $this->addFilePart(serialize($this->dat['records']), $compress);
1170 // adding files:
1171 $out .= $this->addFilePart(serialize($this->dat['files']), $compress);
1172 // adding files_fal:
1173 $out .= $this->addFilePart(serialize($this->dat['files_fal']), $compress);
1174 }
1175 return $out;
1176 }
1177
1178 /**
1179 * Creates XML string from input array
1180 *
1181 * @return string XML content
1182 * @todo Define visibility
1183 */
1184 public function createXML() {
1185 // Options:
1186 $options = array(
1187 'alt_options' => array(
1188 '/header' => array(
1189 'disableTypeAttrib' => TRUE,
1190 'clearStackPath' => TRUE,
1191 'parentTagMap' => array(
1192 'files' => 'file',
1193 'files_fal' => 'file',
1194 'records' => 'table',
1195 'table' => 'rec',
1196 'rec:rels' => 'relations',
1197 'relations' => 'element',
1198 'filerefs' => 'file',
1199 'pid_lookup' => 'page_contents',
1200 'header:relStaticTables' => 'static_tables',
1201 'static_tables' => 'tablename',
1202 'excludeMap' => 'item',
1203 'softrefCfg' => 'softrefExportMode',
1204 'extensionDependencies' => 'extkey',
1205 'softrefs' => 'softref_element'
1206 ),
1207 'alt_options' => array(
1208 '/pagetree' => array(
1209 'disableTypeAttrib' => TRUE,
1210 'useIndexTagForNum' => 'node',
1211 'parentTagMap' => array(
1212 'node:subrow' => 'node'
1213 )
1214 ),
1215 '/pid_lookup/page_contents' => array(
1216 'disableTypeAttrib' => TRUE,
1217 'parentTagMap' => array(
1218 'page_contents' => 'table'
1219 ),
1220 'grandParentTagMap' => array(
1221 'page_contents/table' => 'item'
1222 )
1223 )
1224 )
1225 ),
1226 '/records' => array(
1227 'disableTypeAttrib' => TRUE,
1228 'parentTagMap' => array(
1229 'records' => 'tablerow',
1230 'tablerow:data' => 'fieldlist',
1231 'tablerow:rels' => 'related',
1232 'related' => 'field',
1233 'field:itemArray' => 'relations',
1234 'field:newValueFiles' => 'filerefs',
1235 'field:flexFormRels' => 'flexform',
1236 'relations' => 'element',
1237 'filerefs' => 'file',
1238 'flexform:db' => 'db_relations',
1239 'flexform:file' => 'file_relations',
1240 'flexform:softrefs' => 'softref_relations',
1241 'softref_relations' => 'structurePath',
1242 'db_relations' => 'path',
1243 'file_relations' => 'path',
1244 'path' => 'element',
1245 'keys' => 'softref_key',
1246 'softref_key' => 'softref_element'
1247 ),
1248 'alt_options' => array(
1249 '/records/tablerow/fieldlist' => array(
1250 'useIndexTagForAssoc' => 'field'
1251 )
1252 )
1253 ),
1254 '/files' => array(
1255 'disableTypeAttrib' => TRUE,
1256 'parentTagMap' => array(
1257 'files' => 'file'
1258 )
1259 ),
1260 '/files_fal' => array(
1261 'disableTypeAttrib' => TRUE,
1262 'parentTagMap' => array(
1263 'files_fal' => 'file'
1264 )
1265 )
1266 )
1267 );
1268 // Creating XML file from $outputArray:
1269 $charset = $this->dat['header']['charset'] ?: 'utf-8';
1270 $XML = '<?xml version="1.0" encoding="' . $charset . '" standalone="yes" ?>' . LF;
1271 $XML .= GeneralUtility::array2xml($this->dat, '', 0, 'T3RecordDocument', 0, $options);
1272 return $XML;
1273 }
1274
1275 /**
1276 * Returns TRUE if the output should be compressed.
1277 *
1278 * @return boolean TRUE if compression is possible AND requested.
1279 * @todo Define visibility
1280 */
1281 public function doOutputCompress() {
1282 return $this->compress && !$this->dontCompress;
1283 }
1284
1285 /**
1286 * Returns a content part for a filename being build.
1287 *
1288 * @param array $data Data to store in part
1289 * @param boolean $compress Compress file?
1290 * @return string Content stream.
1291 * @todo Define visibility
1292 */
1293 public function addFilePart($data, $compress = FALSE) {
1294 if ($compress) {
1295 $data = gzcompress($data);
1296 }
1297 return md5($data) . ':' . ($compress ? '1' : '0') . ':' . str_pad(strlen($data), 10, '0', STR_PAD_LEFT) . ':' . $data . ':';
1298 }
1299
1300 /***********************
1301 * Import
1302 ***********************/
1303
1304 /**
1305 * Initialize all settings for the import
1306 *
1307 * @return void
1308 */
1309 protected function initializeImport() {
1310 // Set this flag to indicate that an import is being/has been done.
1311 $this->doesImport = 1;
1312 // Initialize:
1313 // These vars MUST last for the whole section not being cleared. They are used by the method setRelations() which are called at the end of the import session.
1314 $this->import_mapId = array();
1315 $this->import_newId = array();
1316 $this->import_newId_pids = array();
1317 // Temporary files stack initialized:
1318 $this->unlinkFiles = array();
1319 $this->alternativeFileName = array();
1320 $this->alternativeFilePath = array();
1321
1322 $this->initializeStorageObjects();
1323 }
1324
1325 /**
1326 * Initialize the all present storage objects
1327 *
1328 * @return void
1329 */
1330 protected function initializeStorageObjects() {
1331 /** @var $storageRepository \TYPO3\CMS\Core\Resource\StorageRepository */
1332 $storageRepository = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\Resource\\StorageRepository');
1333 $this->storageObjects = $storageRepository->findAll();
1334 }
1335
1336
1337 /**
1338 * Imports the internal data array to $pid.
1339 *
1340 * @param integer $pid Page ID in which to import the content
1341 * @return void
1342 */
1343 public function importData($pid) {
1344
1345 $this->initializeImport();
1346
1347 // Write sys_file_storages first
1348 $this->writeSysFileStorageRecords();
1349 // Write sys_file records and write the binary file data
1350 $this->writeSysFileRecords();
1351 // Write records, first pages, then the rest
1352 // Fields with "hard" relations to database, files and flexform fields are kept empty during this run
1353 $this->writeRecords_pages($pid);
1354 $this->writeRecords_records($pid);
1355 // Finally all the file and DB record references must be fixed. This is done after all records have supposedly been written to database:
1356 // $this->import_mapId will indicate two things: 1) that a record WAS written to db and 2) that it has got a new id-number.
1357 $this->setRelations();
1358 // And when all DB relations are in place, we can fix file and DB relations in flexform fields (since data structures often depends on relations to a DS record):
1359 $this->setFlexFormRelations();
1360 // Unlink temporary files:
1361 $this->unlinkTempFiles();
1362 // Finally, traverse all records and process softreferences with substitution attributes.
1363 $this->processSoftReferences();
1364 }
1365
1366 /**
1367 * Imports the sys_file_storage records from internal data array.
1368 *
1369 * @return void
1370 */
1371 protected function writeSysFileStorageRecords() {
1372 if (!isset($this->dat['header']['records']['sys_file_storage'])) {
1373 return;
1374 }
1375 $sysFileStorageUidsToBeResetToDefaultStorage = array();
1376 foreach (array_keys($this->dat['header']['records']['sys_file_storage']) as $sysFileStorageUid) {
1377 $storageRecord = $this->dat['records']['sys_file_storage:' . $sysFileStorageUid]['data'];
1378 // continue with Local, writable and online storage only
1379 if ($storageRecord['driver'] === 'Local' && $storageRecord['is_writable'] && $storageRecord['is_online']) {
1380 $useThisStorageUidInsteadOfTheOneInImport = 0;
1381 /** @var $localStorage \TYPO3\CMS\Core\Resource\ResourceStorage */
1382 foreach ($this->storageObjects as $localStorage) {
1383 // check the available storage for Local, writable and online ones
1384 if ($localStorage->getDriverType() === 'Local' && $localStorage->isWritable() && $localStorage->isOnline()) {
1385 // check if there is already a identical storage present (same pathType and basePath)
1386 $storageRecordConfiguration = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->convertFlexFormDataToConfigurationArray($storageRecord['configuration']);
1387 $localStorageRecordConfiguration = $localStorage->getConfiguration();
1388 if ($storageRecordConfiguration['pathType'] == $localStorageRecordConfiguration['pathType'] && $storageRecordConfiguration['basePath'] == $localStorageRecordConfiguration['basePath']) {
1389 // same storage is already present
1390 $useThisStorageUidInsteadOfTheOneInImport = $localStorage->getUid();
1391 break;
1392 }
1393 }
1394 }
1395 if ($useThisStorageUidInsteadOfTheOneInImport > 0) {
1396 // same storage is already present; map the to be imported one to the present one
1397 $this->import_mapId['sys_file_storage'][$sysFileStorageUid] = $useThisStorageUidInsteadOfTheOneInImport;
1398 } else {
1399 // Local, writable and online storage. Is allowed to be used to later write files in.
1400 $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);
1401 }
1402 } else {
1403 // Storage with non Local drivers could be imported but must not be used to saves files in, because you
1404 // could not be sure, that this is supported. The default storage will be used in this case.
1405 // It could happen that non writable and non online storage will be created as dupes because you could not
1406 // check the detailed configuration options at this point
1407 $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);
1408 $sysFileStorageUidsToBeResetToDefaultStorage[] = $sysFileStorageUid;
1409 }
1410
1411 }
1412
1413 // Importing the added ones
1414 $tce = $this->getNewTCE();
1415 // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.
1416 $tce->reverseOrder = 1;
1417 $tce->isImporting = TRUE;
1418 $tce->start($this->import_data, array());
1419 $tce->process_datamap();
1420 $this->addToMapId($tce->substNEWwithIDs);
1421
1422 $defaultStorageUid = NULL;
1423 // get default storage
1424 $defaultStorage = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->getDefaultStorage();
1425 if ($defaultStorage !== NULL) {
1426 $defaultStorageUid = $defaultStorage->getUid();
1427 }
1428 foreach ($sysFileStorageUidsToBeResetToDefaultStorage as $sysFileStorageUidToBeResetToDefaultStorage) {
1429 $this->import_mapId['sys_file_storage'][$sysFileStorageUidToBeResetToDefaultStorage] = $defaultStorageUid;
1430 }
1431
1432 // unset the sys_file_storage records to prevent a import in writeRecords_records
1433 unset($this->dat['header']['records']['sys_file_storage']);
1434 }
1435
1436 /**
1437 * Imports the sys_file records and the binary files data from internal data array.
1438 *
1439 * @return void
1440 */
1441 protected function writeSysFileRecords() {
1442 if (!isset($this->dat['header']['records']['sys_file'])) {
1443 return;
1444 }
1445
1446 // fetch fresh storage records from database
1447 $storageRecords = $this->fetchStorageRecords();
1448
1449 $defaultStorage = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->getDefaultStorage();
1450
1451 foreach (array_keys($this->dat['header']['records']['sys_file']) as $sysFileUid) {
1452 $fileRecord = $this->dat['records']['sys_file:' . $sysFileUid]['data'];
1453
1454 // save file to disk
1455 $fileId = md5($fileRecord['storage'] . ':' . $fileRecord['identifier_hash']);
1456 $temporaryFile = $this->writeTemporaryFileFromData($fileId);
1457 if ($temporaryFile === NULL) {
1458 // error on writing the file. Error message was already added
1459 continue;
1460 }
1461
1462 $originalStorageUid = $fileRecord['storage'];
1463 $useStorageFromStorageRecords = FALSE;
1464
1465 // replace storage id, if a alternative one was registered
1466 if (isset($this->import_mapId['sys_file_storage'][$fileRecord['storage']])) {
1467 $fileRecord['storage'] = $this->import_mapId['sys_file_storage'][$fileRecord['storage']];
1468 $useStorageFromStorageRecords = TRUE;
1469 }
1470
1471 if (empty($fileRecord['storage'])) {
1472 // no storage for the file is defined, mostly because of a missing default storage.
1473 $this->error('Error: No storage for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $originalStorageUid . '"');
1474 continue;
1475 }
1476
1477 // using a storage from the local storage is only allowed, if the uid is present in the
1478 // mapping. Only in this case we could be sure, that it's a local, online and writable storage.
1479 if ($useStorageFromStorageRecords && isset($storageRecords[$fileRecord['storage']])) {
1480 /** @var $storage \TYPO3\CMS\Core\Resource\ResourceStorage */
1481 $storage = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->getStorageObject($fileRecord['storage'], $storageRecords[$fileRecord['storage']]);
1482 } elseif ($defaultStorage !== NULL) {
1483 $storage = $defaultStorage;
1484 } else {
1485 $this->error('Error: No storage available for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1486 continue;
1487 }
1488
1489 $newFile = NULL;
1490
1491 // check, if there is a identical file
1492 try {
1493 if ($storage->hasFile($fileRecord['identifier'])) {
1494 $file = $storage->getFile($fileRecord['identifier']);
1495 if ($file->getSha1() === $fileRecord['sha1']) {
1496 $newFile = $file;
1497 }
1498 }
1499 } catch (Exception $e) {}
1500
1501 if ($newFile === NULL) {
1502
1503 $folderName = dirname(ltrim($fileRecord['identifier'], '/'));
1504 if (!$storage->hasFolder($folderName)) {
1505 try {
1506 $storage->createFolder($folderName);
1507 } catch (Exception $e) {
1508 $this->error('Error: Folder could not be created for file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1509 continue;
1510 }
1511 }
1512
1513 $importFolder = $storage->getFolder($folderName);
1514
1515 try {
1516 /** @var $file \TYPO3\CMS\Core\Resource\File */
1517 $newFile = $storage->addFile($temporaryFile, $importFolder, $fileRecord['name']);
1518 } catch (Exception $e) {
1519 $this->error('Error: File could not be added to the storage: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1520 continue;
1521 }
1522
1523 if ($newFile->getSha1() !== $fileRecord['sha1']) {
1524 $this->error('Error: The hash of the written file is not identical to the import data! File could be corrupted! File: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1525 }
1526 }
1527
1528 // save the new uid in the import id map
1529 $this->import_mapId['sys_file'][$fileRecord['uid']] = $newFile->getUid();
1530 $this->fixUidLocalInSysFileReferenceRecords($fileRecord['uid'], $newFile->getUid());
1531
1532 }
1533
1534 // unset the sys_file records to prevent a import in writeRecords_records
1535 unset($this->dat['header']['records']['sys_file']);
1536 }
1537
1538 /**
1539 * Normally the importer works like the following:
1540 * Step 1: import the records with cleared field values of relation fields (see addSingle())
1541 * Step 2: update the records with the right relation ids (see setRelations())
1542 *
1543 * In step 2 the saving fields of type "relation to sys_file_reference" checks the related sys_file_reference
1544 * record (created in step 1) with the FileExtensionFilter for matching file extensions of the related file.
1545 * To make this work correct, the uid_local of sys_file_reference records has to be not empty AND has to
1546 * relate to the correct (imported) sys_file record uid!!!
1547 *
1548 * This is fixed here.
1549 *
1550 * @param int $oldFileUid
1551 * @param int $newFileUid
1552 * @return void
1553 */
1554 protected function fixUidLocalInSysFileReferenceRecords($oldFileUid, $newFileUid) {
1555 if (!isset($this->dat['header']['records']['sys_file_reference'])) {
1556 return;
1557 }
1558
1559 foreach (array_keys($this->dat['header']['records']['sys_file_reference']) as $sysFileReferenceUid) {
1560 $fileReferenceRecord = $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'];
1561 if ($fileReferenceRecord['uid_local'] == $oldFileUid) {
1562 $fileReferenceRecord['uid_local'] = $newFileUid;
1563 $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'] = $fileReferenceRecord;
1564 }
1565 }
1566 }
1567
1568 /**
1569 * Fetched fresh storage records from database because the new imported
1570 * ones are not in cached data of the StorageRepository
1571 *
1572 * @return bool|array
1573 */
1574 protected function fetchStorageRecords() {
1575 $whereClause = \TYPO3\CMS\Backend\Utility\BackendUtility::BEenableFields('sys_file_storage');
1576 $whereClause .= \TYPO3\CMS\Backend\Utility\BackendUtility::deleteClause('sys_file_storage');
1577
1578 $rows = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows(
1579 '*',
1580 'sys_file_storage',
1581 '1=1' . $whereClause,
1582 '',
1583 '',
1584 '',
1585 'uid'
1586 );
1587
1588 return $rows;
1589 }
1590
1591 /**
1592 * Writes the file from import array to temp dir and returns the filename of it.
1593 *
1594 * @param string $fileId
1595 * @param string $dataKey
1596 * @return string Absolute filename of the temporary filename of the file
1597 */
1598 protected function writeTemporaryFileFromData($fileId, $dataKey = 'files_fal') {
1599 $temporaryFilePath = NULL;
1600 if (is_array($this->dat[$dataKey][$fileId])) {
1601 $temporaryFilePathInternal = GeneralUtility::tempnam('import_temp_');
1602 GeneralUtility::writeFile($temporaryFilePathInternal, $this->dat[$dataKey][$fileId]['content']);
1603 clearstatcache();
1604 if (@is_file($temporaryFilePathInternal)) {
1605 $this->unlinkFiles[] = $temporaryFilePathInternal;
1606 if (filesize($temporaryFilePathInternal) == $this->dat[$dataKey][$fileId]['filesize']) {
1607 $temporaryFilePath = $temporaryFilePathInternal;
1608 } else {
1609 $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' had a size (' . filesize($temporaryFilePathInternal) . ') different from the original (' . $this->dat[$dataKey][$fileId]['filesize'] . ')', 1);
1610 }
1611 } else {
1612 $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' was not written as it should have been!', 1);
1613 }
1614 } else {
1615 $this->error('Error: No file found for ID ' . $fileId, 1);
1616 }
1617 return $temporaryFilePath;
1618 }
1619
1620 /**
1621 * Writing pagetree/pages to database:
1622 *
1623 * @param integer $pid PID in which to import. If the operation is an update operation, the root of the page tree inside will be moved to this PID unless it is the same as the root page from the import
1624 * @return void
1625 * @see writeRecords_records()
1626 * @todo Define visibility
1627 */
1628 public function writeRecords_pages($pid) {
1629 // First, write page structure if any:
1630 if (is_array($this->dat['header']['records']['pages'])) {
1631 // $pageRecords is a copy of the pages array in the imported file. Records here are unset one by one when the addSingle function is called.
1632 $pageRecords = $this->dat['header']['records']['pages'];
1633 $this->import_data = array();
1634 // First add page tree if any
1635 if (is_array($this->dat['header']['pagetree'])) {
1636 $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);
1637 foreach ($pagesFromTree as $uid) {
1638 $thisRec = $this->dat['header']['records']['pages'][$uid];
1639 // PID: Set the main $pid, unless a NEW-id is found
1640 $setPid = isset($this->import_newId_pids[$thisRec['pid']]) ? $this->import_newId_pids[$thisRec['pid']] : $pid;
1641 $this->addSingle('pages', $uid, $setPid);
1642 unset($pageRecords[$uid]);
1643 }
1644 }
1645 // Then add all remaining pages not in tree on root level:
1646 if (count($pageRecords)) {
1647 $remainingPageUids = array_keys($pageRecords);
1648 foreach ($remainingPageUids as $pUid) {
1649 $this->addSingle('pages', $pUid, $pid);
1650 }
1651 }
1652 // Now write to database:
1653 $tce = $this->getNewTCE();
1654 $tce->isImporting = TRUE;
1655 $this->callHook('before_writeRecordsPages', array(
1656 'tce' => &$tce,
1657 'data' => &$this->import_data
1658 ));
1659 $tce->suggestedInsertUids = $this->suggestedInsertUids;
1660 $tce->start($this->import_data, array());
1661 $tce->process_datamap();
1662 $this->callHook('after_writeRecordsPages', array(
1663 'tce' => &$tce
1664 ));
1665 // post-processing: Registering new ids (end all tcemain sessions with this)
1666 $this->addToMapId($tce->substNEWwithIDs);
1667 // In case of an update, order pages from the page tree correctly:
1668 if ($this->update && is_array($this->dat['header']['pagetree'])) {
1669 $this->writeRecords_pages_order($pid);
1670 }
1671 }
1672 }
1673
1674 /**
1675 * Organize all updated pages in page tree so they are related like in the import file
1676 * Only used for updates and when $this->dat['header']['pagetree'] is an array.
1677 *
1678 * @param integer $pid Page id in which to import
1679 * @return void
1680 * @access private
1681 * @see writeRecords_pages(), writeRecords_records_order()
1682 * @todo Define visibility
1683 */
1684 public function writeRecords_pages_order($pid) {
1685 $cmd_data = array();
1686 // Get uid-pid relations and traverse them in order to map to possible new IDs
1687 $pidsFromTree = $this->flatInversePageTree_pid($this->dat['header']['pagetree']);
1688 foreach ($pidsFromTree as $origPid => $newPid) {
1689 if ($newPid >= 0 && $this->dontIgnorePid('pages', $origPid)) {
1690 // If the page had a new id (because it was created) use that instead!
1691 if (substr($this->import_newId_pids[$origPid], 0, 3) === 'NEW') {
1692 if ($this->import_mapId['pages'][$origPid]) {
1693 $mappedPid = $this->import_mapId['pages'][$origPid];
1694 $cmd_data['pages'][$mappedPid]['move'] = $newPid;
1695 }
1696 } else {
1697 $cmd_data['pages'][$origPid]['move'] = $newPid;
1698 }
1699 }
1700 }
1701 // Execute the move commands if any:
1702 if (count($cmd_data)) {
1703 $tce = $this->getNewTCE();
1704 $this->callHook('before_writeRecordsPagesOrder', array(
1705 'tce' => &$tce,
1706 'data' => &$cmd_data
1707 ));
1708 $tce->start(array(), $cmd_data);
1709 $tce->process_cmdmap();
1710 $this->callHook('after_writeRecordsPagesOrder', array(
1711 'tce' => &$tce
1712 ));
1713 }
1714 }
1715
1716 /**
1717 * Write all database records except pages (writtein in writeRecords_pages())
1718 *
1719 * @param integer $pid Page id in which to import
1720 * @return void
1721 * @see writeRecords_pages()
1722 * @todo Define visibility
1723 */
1724 public function writeRecords_records($pid) {
1725 // Write the rest of the records
1726 $this->import_data = array();
1727 if (is_array($this->dat['header']['records'])) {
1728 foreach ($this->dat['header']['records'] as $table => $recs) {
1729 if ($table != 'pages') {
1730 foreach ($recs as $uid => $thisRec) {
1731 // PID: Set the main $pid, unless a NEW-id is found
1732 $setPid = isset($this->import_mapId['pages'][$thisRec['pid']]) ? $this->import_mapId['pages'][$thisRec['pid']] : $pid;
1733 if (is_array($GLOBALS['TCA'][$table]) && $GLOBALS['TCA'][$table]['ctrl']['rootLevel']) {
1734 $setPid = 0;
1735 }
1736 // Add record:
1737 $this->addSingle($table, $uid, $setPid);
1738 }
1739 }
1740 }
1741 } else {
1742 $this->error('Error: No records defined in internal data array.');
1743 }
1744 // Now write to database:
1745 $tce = $this->getNewTCE();
1746 $this->callHook('before_writeRecordsRecords', array(
1747 'tce' => &$tce,
1748 'data' => &$this->import_data
1749 ));
1750 $tce->suggestedInsertUids = $this->suggestedInsertUids;
1751 // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.
1752 $tce->reverseOrder = 1;
1753 $tce->isImporting = TRUE;
1754 $tce->start($this->import_data, array());
1755 $tce->process_datamap();
1756 $this->callHook('after_writeRecordsRecords', array(
1757 'tce' => &$tce
1758 ));
1759 // post-processing: Removing files and registering new ids (end all tcemain sessions with this)
1760 $this->addToMapId($tce->substNEWwithIDs);
1761 // In case of an update, order pages from the page tree correctly:
1762 if ($this->update) {
1763 $this->writeRecords_records_order($pid);
1764 }
1765 }
1766
1767 /**
1768 * Organize all updated record to their new positions.
1769 * Only used for updates
1770 *
1771 * @param integer $mainPid Main PID into which we import.
1772 * @return void
1773 * @access private
1774 * @see writeRecords_records(), writeRecords_pages_order()
1775 * @todo Define visibility
1776 */
1777 public function writeRecords_records_order($mainPid) {
1778 $cmd_data = array();
1779 if (is_array($this->dat['header']['pagetree'])) {
1780 $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);
1781 } else {
1782 $pagesFromTree = array();
1783 }
1784 if (is_array($this->dat['header']['pid_lookup'])) {
1785 foreach ($this->dat['header']['pid_lookup'] as $pid => $recList) {
1786 $newPid = isset($this->import_mapId['pages'][$pid]) ? $this->import_mapId['pages'][$pid] : $mainPid;
1787 if (\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($newPid)) {
1788 foreach ($recList as $tableName => $uidList) {
1789 // If $mainPid===$newPid then we are on root level and we can consider to move pages as well!
1790 // (they will not be in the page tree!)
1791 if (($tableName != 'pages' || !$pagesFromTree[$pid]) && is_array($uidList)) {
1792 $uidList = array_reverse(array_keys($uidList));
1793 foreach ($uidList as $uid) {
1794 if ($this->dontIgnorePid($tableName, $uid)) {
1795 $cmd_data[$tableName][$uid]['move'] = $newPid;
1796 } else {
1797
1798 }
1799 }
1800 }
1801 }
1802 }
1803 }
1804 }
1805 // Execute the move commands if any:
1806 if (count($cmd_data)) {
1807 $tce = $this->getNewTCE();
1808 $this->callHook('before_writeRecordsRecordsOrder', array(
1809 'tce' => &$tce,
1810 'data' => &$cmd_data
1811 ));
1812 $tce->start(array(), $cmd_data);
1813 $tce->process_cmdmap();
1814 $this->callHook('after_writeRecordsRecordsOrder', array(
1815 'tce' => &$tce
1816 ));
1817 }
1818 }
1819
1820 /**
1821 * Adds a single record to the $importData array. Also copies files to tempfolder.
1822 * However all File/DB-references and flexform field contents are set to blank for now! That is done with setRelations() later
1823 *
1824 * @param string $table Table name (from import memory)
1825 * @param integer $uid Record UID (from import memory)
1826 * @param integer $pid Page id
1827 * @return void
1828 * @see writeRecords()
1829 * @todo Define visibility
1830 */
1831 public function addSingle($table, $uid, $pid) {
1832 if ($this->import_mode[$table . ':' . $uid] !== 'exclude') {
1833 $record = $this->dat['records'][$table . ':' . $uid]['data'];
1834 if (is_array($record)) {
1835 if ($this->update && $this->doesRecordExist($table, $uid) && $this->import_mode[$table . ':' . $uid] !== 'as_new') {
1836 $ID = $uid;
1837 } elseif ($table === 'sys_file_metadata' && $record['sys_language_uid'] == '0' && $this->import_mapId['sys_file'][$record['file']]) {
1838 // on adding sys_file records the belonging sys_file_metadata record was also created
1839 // if there is one the record need to be overwritten instead of creating a new one.
1840 $recordInDatabase = $GLOBALS['TYPO3_DB']->exec_SELECTgetSingleRow(
1841 'uid',
1842 'sys_file_metadata',
1843 'file = ' . $this->import_mapId['sys_file'][$record['file']] . ' AND sys_language_uid = 0 AND pid = 0'
1844 );
1845 if ($recordInDatabase !== NULL) {
1846 $this->import_mapId['sys_file_metadata'][$record['uid']] = $recordInDatabase['uid'];
1847 $ID = $recordInDatabase['uid'];
1848 } else {
1849 $ID = uniqid('NEW');
1850 }
1851
1852 } else {
1853 $ID = uniqid('NEW');
1854 }
1855 $this->import_newId[$table . ':' . $ID] = array('table' => $table, 'uid' => $uid);
1856 if ($table == 'pages') {
1857 $this->import_newId_pids[$uid] = $ID;
1858 }
1859 // Set main record data:
1860 $this->import_data[$table][$ID] = $record;
1861 $this->import_data[$table][$ID]['tx_impexp_origuid'] = $this->import_data[$table][$ID]['uid'];
1862 // Reset permission data:
1863 if ($table === 'pages') {
1864 // Have to reset the user/group IDs so pages are owned by importing user. Otherwise strange things may happen for non-admins!
1865 unset($this->import_data[$table][$ID]['perms_userid']);
1866 unset($this->import_data[$table][$ID]['perms_groupid']);
1867 }
1868 // PID and UID:
1869 unset($this->import_data[$table][$ID]['uid']);
1870 // Updates:
1871 if (\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($ID)) {
1872 unset($this->import_data[$table][$ID]['pid']);
1873 } else {
1874 // Inserts:
1875 $this->import_data[$table][$ID]['pid'] = $pid;
1876 if (($this->import_mode[$table . ':' . $uid] === 'force_uid' && $this->update || $this->force_all_UIDS) && $GLOBALS['BE_USER']->isAdmin()) {
1877 $this->import_data[$table][$ID]['uid'] = $uid;
1878 $this->suggestedInsertUids[$table . ':' . $uid] = 'DELETE';
1879 }
1880 }
1881 // Setting db/file blank:
1882 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
1883 switch ((string) $config['type']) {
1884 case 'db':
1885
1886 case 'file':
1887 // Fixed later in ->setRelations() [because we need to know ALL newly created IDs before we can map relations!]
1888 // In the meantime we set NO values for relations.
1889 //
1890 // BUT for field uid_local of table sys_file_reference the relation MUST not be cleared here,
1891 // because the value is already the uid of the right imported sys_file record.
1892 // @see fixUidLocalInSysFileReferenceRecords()
1893 // If it's empty or a uid to another record the FileExtensionFilter will throw an exception or
1894 // delete the reference record if the file extension of the related record doesn't match.
1895 if ($table !== 'sys_file_reference' && $field !== 'uid_local') {
1896 $this->import_data[$table][$ID][$field] = '';
1897 }
1898 break;
1899 case 'flex':
1900 // Fixed later in setFlexFormRelations()
1901 // In the meantime we set NO value for flexforms - this is mainly because file references inside will not be processed properly; In fact references will point to no file or existing files (in which case there will be double-references which is a big problem of course!)
1902 $this->import_data[$table][$ID][$field] = '';
1903 break;
1904 }
1905 }
1906 } elseif ($table . ':' . $uid != 'pages:0') {
1907 // On root level we don't want this error message.
1908 $this->error('Error: no record was found in data array!', 1);
1909 }
1910 }
1911 }
1912
1913 /**
1914 * Registers the substNEWids in memory.
1915 *
1916 * @param array $substNEWwithIDs From tcemain to be merged into internal mapping variable in this object
1917 * @return void
1918 * @see writeRecords()
1919 * @todo Define visibility
1920 */
1921 public function addToMapId($substNEWwithIDs) {
1922 foreach ($this->import_data as $table => $recs) {
1923 foreach ($recs as $id => $value) {
1924 $old_uid = $this->import_newId[$table . ':' . $id]['uid'];
1925 if (isset($substNEWwithIDs[$id])) {
1926 $this->import_mapId[$table][$old_uid] = $substNEWwithIDs[$id];
1927 } elseif ($this->update) {
1928 // Map same ID to same ID....
1929 $this->import_mapId[$table][$old_uid] = $id;
1930 } else {
1931 $this->error('Possible error: ' . $table . ':' . $old_uid . ' had no new id assigned to it. This indicates that the record was not added to database during import. Please check changelog!', 1);
1932 }
1933 }
1934 }
1935 }
1936
1937 /**
1938 * Returns a new $TCE object
1939 *
1940 * @return object $TCE object
1941 * @todo Define visibility
1942 */
1943 public function getNewTCE() {
1944 $tce = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\DataHandling\\DataHandler');
1945 $tce->stripslashes_values = 0;
1946 $tce->dontProcessTransformations = 1;
1947 $tce->enableLogging = $this->enableLogging;
1948 $tce->alternativeFileName = $this->alternativeFileName;
1949 $tce->alternativeFilePath = $this->alternativeFilePath;
1950 return $tce;
1951 }
1952
1953 /**
1954 * Cleaning up all the temporary files stored in typo3temp/ folder
1955 *
1956 * @return void
1957 * @todo Define visibility
1958 */
1959 public function unlinkTempFiles() {
1960 foreach ($this->unlinkFiles as $fileName) {
1961 if (GeneralUtility::isFirstPartOfStr($fileName, PATH_site . 'typo3temp/')) {
1962 GeneralUtility::unlink_tempfile($fileName);
1963 clearstatcache();
1964 if (is_file($fileName)) {
1965 $this->error('Error: ' . $fileName . ' was NOT unlinked as it should have been!', 1);
1966 }
1967 } else {
1968 $this->error('Error: ' . $fileName . ' was not in temp-path. Not removed!', 1);
1969 }
1970 }
1971 $this->unlinkFiles = array();
1972 }
1973
1974 /***************************
1975 * Import / Relations setting
1976 ***************************/
1977
1978 /**
1979 * At the end of the import process all file and DB relations should be set properly (that is relations to imported records are all re-created so imported records are correctly related again)
1980 * Relations in flexform fields are processed in setFlexFormRelations() after this function
1981 *
1982 * @return void
1983 * @see setFlexFormRelations()
1984 * @todo Define visibility
1985 */
1986 public function setRelations() {
1987 $updateData = array();
1988 // import_newId contains a register of all records that was in the import memorys "records" key
1989 foreach ($this->import_newId as $nId => $dat) {
1990 $table = $dat['table'];
1991 $uid = $dat['uid'];
1992 // original UID - NOT the new one!
1993 // If the record has been written and received a new id, then proceed:
1994 if (is_array($this->import_mapId[$table]) && isset($this->import_mapId[$table][$uid])) {
1995 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
1996 if (is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {
1997 // Traverse relation fields of each record
1998 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
1999 // uid_local of sys_file_reference needs no update because the correct reference uid was already written
2000 // @see ImportExport::fixUidLocalInSysFileReferenceRecords()
2001 if ($table === 'sys_file_reference' && $field === 'uid_local') {
2002 continue;
2003 }
2004 switch ((string) $config['type']) {
2005 case 'db':
2006 if (is_array($config['itemArray']) && count($config['itemArray'])) {
2007 $itemConfig = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
2008 $valArray = $this->setRelations_db($config['itemArray'], $itemConfig);
2009 $updateData[$table][$thisNewUid][$field] = implode(',', $valArray);
2010 }
2011 break;
2012 case 'file':
2013 if (is_array($config['newValueFiles']) && count($config['newValueFiles'])) {
2014 $valArr = array();
2015 foreach ($config['newValueFiles'] as $fI) {
2016 $valArr[] = $this->import_addFileNameToBeCopied($fI);
2017 }
2018 $updateData[$table][$thisNewUid][$field] = implode(',', $valArr);
2019 }
2020 break;
2021 }
2022 }
2023 } else {
2024 $this->error('Error: no record was found in data array!', 1);
2025 }
2026 } else {
2027 $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')', 1);
2028 }
2029 }
2030 if (count($updateData)) {
2031 $tce = $this->getNewTCE();
2032 $tce->isImporting = TRUE;
2033 $this->callHook('before_setRelation', array(
2034 'tce' => &$tce,
2035 'data' => &$updateData
2036 ));
2037 $tce->start($updateData, array());
2038 $tce->process_datamap();
2039 $this->callHook('after_setRelations', array(
2040 'tce' => &$tce
2041 ));
2042 }
2043 }
2044
2045 /**
2046 * Maps relations for database
2047 *
2048 * @param array $itemArray Array of item sets (table/uid) from a dbAnalysis object
2049 * @param array $itemConfig Array of TCA config of the field the relation to be set on
2050 * @return array Array with values [table]_[uid] or [uid] for field of type group / internal_type file_reference. These values have the regular tcemain-input group/select type which means they will automatically be processed into a uid-list or MM relations.
2051 * @todo Define visibility
2052 */
2053 public function setRelations_db($itemArray, $itemConfig) {
2054 $valArray = array();
2055 foreach ($itemArray as $relDat) {
2056 if (is_array($this->import_mapId[$relDat['table']]) && isset($this->import_mapId[$relDat['table']][$relDat['id']])) {
2057 // Since non FAL file relation type group internal_type file_reference are handled as reference to
2058 // sys_file records Datahandler requires the value as uid of the the related sys_file record only
2059 if ($itemConfig['type'] === 'group' && $itemConfig['internal_type'] === 'file_reference') {
2060 $value = $this->import_mapId[$relDat['table']][$relDat['id']];
2061 } else {
2062 $value = $relDat['table'] . '_' . $this->import_mapId[$relDat['table']][$relDat['id']];
2063 }
2064 $valArray[] = $value;
2065 } elseif ($this->isTableStatic($relDat['table']) || $this->isExcluded($relDat['table'], $relDat['id']) || $relDat['id'] < 0) {
2066 // Checking for less than zero because some select types could contain negative values, eg. fe_groups (-1, -2) and sys_language (-1 = ALL languages). This must be handled on both export and import.
2067 $valArray[] = $relDat['table'] . '_' . $relDat['id'];
2068 } else {
2069 $this->error('Lost relation: ' . $relDat['table'] . ':' . $relDat['id'], 1);
2070 }
2071 }
2072 return $valArray;
2073 }
2074
2075 /**
2076 * Writes the file from import array to temp dir and returns the filename of it.
2077 *
2078 * @param array $fI File information with three keys: "filename" = filename without path, "ID_absFile" = absolute filepath to the file (including the filename), "ID" = md5 hash of "ID_absFile
2079 * @return string Absolute filename of the temporary filename of the file. In ->alternativeFileName the original name is set.
2080 * @todo Define visibility
2081 */
2082 public function import_addFileNameToBeCopied($fI) {
2083 if (is_array($this->dat['files'][$fI['ID']])) {
2084 $tmpFile = GeneralUtility::tempnam('import_temp_');
2085 GeneralUtility::writeFile($tmpFile, $this->dat['files'][$fI['ID']]['content']);
2086 clearstatcache();
2087 if (@is_file($tmpFile)) {
2088 $this->unlinkFiles[] = $tmpFile;
2089 if (filesize($tmpFile) == $this->dat['files'][$fI['ID']]['filesize']) {
2090 $this->alternativeFileName[$tmpFile] = $fI['filename'];
2091 $this->alternativeFilePath[$tmpFile] = $this->dat['files'][$fI['ID']]['relFileRef'];
2092 return $tmpFile;
2093 } else {
2094 $this->error('Error: temporary file ' . $tmpFile . ' had a size (' . filesize($tmpFile) . ') different from the original (' . $this->dat['files'][$fI['ID']]['filesize'] . ')', 1);
2095 }
2096 } else {
2097 $this->error('Error: temporary file ' . $tmpFile . ' was not written as it should have been!', 1);
2098 }
2099 } else {
2100 $this->error('Error: No file found for ID ' . $fI['ID'], 1);
2101 }
2102 }
2103
2104 /**
2105 * After all DB relations has been set in the end of the import (see setRelations()) then it is time to correct all relations inside of FlexForm fields.
2106 * The reason for doing this after is that the setting of relations may affect (quite often!) which data structure is used for the flexforms field!
2107 *
2108 * @return void
2109 * @see setRelations()
2110 * @todo Define visibility
2111 */
2112 public function setFlexFormRelations() {
2113 $updateData = array();
2114 // import_newId contains a register of all records that was in the import memorys "records" key
2115 foreach ($this->import_newId as $nId => $dat) {
2116 $table = $dat['table'];
2117 $uid = $dat['uid'];
2118 // original UID - NOT the new one!
2119 // If the record has been written and received a new id, then proceed:
2120 if (is_array($this->import_mapId[$table]) && isset($this->import_mapId[$table][$uid])) {
2121 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
2122 if (is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {
2123 // Traverse relation fields of each record
2124 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
2125 switch ((string) $config['type']) {
2126 case 'flex':
2127 // Get XML content and set as default value (string, non-processed):
2128 $updateData[$table][$thisNewUid][$field] = $this->dat['records'][$table . ':' . $uid]['data'][$field];
2129 // If there has been registered relations inside the flex form field, run processing on the content:
2130 if (count($config['flexFormRels']['db']) || count($config['flexFormRels']['file'])) {
2131 $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');
2132 // This will fetch the new row for the element (which should be updated with any references to data structures etc.)
2133 $conf = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
2134 if (is_array($origRecordRow) && is_array($conf) && $conf['type'] === 'flex') {
2135 // Get current data structure and value array:
2136 $dataStructArray = BackendUtility::getFlexFormDS($conf, $origRecordRow, $table);
2137 $currentValueArray = GeneralUtility::xml2array($updateData[$table][$thisNewUid][$field]);
2138 // Do recursive processing of the XML data:
2139 $iteratorObj = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\DataHandling\\DataHandler');
2140 $iteratorObj->callBackObj = $this;
2141 $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData($currentValueArray['data'], array(), array(), $dataStructArray, array($table, $thisNewUid, $field, $config), 'remapListedDBRecords_flexFormCallBack');
2142 // The return value is set as an array which means it will be processed by tcemain for file and DB references!
2143 if (is_array($currentValueArray['data'])) {
2144 $updateData[$table][$thisNewUid][$field] = $currentValueArray;
2145 }
2146 }
2147 }
2148 break;
2149 }
2150 }
2151 } else {
2152 $this->error('Error: no record was found in data array!', 1);
2153 }
2154 } else {
2155 $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')', 1);
2156 }
2157 }
2158 if (count($updateData)) {
2159 $tce = $this->getNewTCE();
2160 $tce->isImporting = TRUE;
2161 $this->callHook('before_setFlexFormRelations', array(
2162 'tce' => &$tce,
2163 'data' => &$updateData
2164 ));
2165 $tce->start($updateData, array());
2166 $tce->process_datamap();
2167 $this->callHook('after_setFlexFormRelations', array(
2168 'tce' => &$tce
2169 ));
2170 }
2171 }
2172
2173 /**
2174 * Callback function for traversing the FlexForm structure in relation to remapping database relations
2175 *
2176 * @param array $pParams Set of parameters in numeric array: table, uid, field
2177 * @param array $dsConf TCA config for field (from Data Structure of course)
2178 * @param string $dataValue Field value (from FlexForm XML)
2179 * @param string $dataValue_ext1 Not used
2180 * @param string $dataValue_ext2 Not used
2181 * @param string $path Path of where the data structure of the element is found
2182 * @return array Array where the "value" key carries the value.
2183 * @see setFlexFormRelations()
2184 * @todo Define visibility
2185 */
2186 public function remapListedDBRecords_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path) {
2187 // Extract parameters:
2188 list($table, $uid, $field, $config) = $pParams;
2189 // In case the $path is used as index without a trailing slash we will remove that
2190 if (!is_array($config['flexFormRels']['db'][$path]) && is_array($config['flexFormRels']['db'][rtrim($path, '/')])) {
2191 $path = rtrim($path, '/');
2192 }
2193 if (is_array($config['flexFormRels']['db'][$path])) {
2194 $valArray = $this->setRelations_db($config['flexFormRels']['db'][$path], $dsConf);
2195 $dataValue = implode(',', $valArray);
2196 }
2197 if (is_array($config['flexFormRels']['file'][$path])) {
2198 foreach ($config['flexFormRels']['file'][$path] as $fI) {
2199 $valArr[] = $this->import_addFileNameToBeCopied($fI);
2200 }
2201 $dataValue = implode(',', $valArr);
2202 }
2203 return array('value' => $dataValue);
2204 }
2205
2206 /**************************
2207 * Import / Soft References
2208 *************************/
2209
2210 /**
2211 * Processing of soft references
2212 *
2213 * @return void
2214 * @todo Define visibility
2215 */
2216 public function processSoftReferences() {
2217 // Initialize:
2218 $inData = array();
2219 // Traverse records:
2220 if (is_array($this->dat['header']['records'])) {
2221 foreach ($this->dat['header']['records'] as $table => $recs) {
2222 foreach ($recs as $uid => $thisRec) {
2223 // If there are soft references defined, traverse those:
2224 if (isset($GLOBALS['TCA'][$table]) && is_array($thisRec['softrefs'])) {
2225 // First traversal is to collect softref configuration and split them up based on fields. This could probably also have been done with the "records" key instead of the header.
2226 $fieldsIndex = array();
2227 foreach ($thisRec['softrefs'] as $softrefDef) {
2228 // If a substitution token is set:
2229 if ($softrefDef['field'] && is_array($softrefDef['subst']) && $softrefDef['subst']['tokenID']) {
2230 $fieldsIndex[$softrefDef['field']][$softrefDef['subst']['tokenID']] = $softrefDef;
2231 }
2232 }
2233 // The new id:
2234 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
2235 // Now, if there are any fields that require substitution to be done, lets go for that:
2236 foreach ($fieldsIndex as $field => $softRefCfgs) {
2237 if (is_array($GLOBALS['TCA'][$table]['columns'][$field])) {
2238 $conf = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
2239 if ($conf['type'] === 'flex') {
2240 // This will fetch the new row for the element (which should be updated with any references to data structures etc.)
2241 $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');
2242 if (is_array($origRecordRow)) {
2243 // Get current data structure and value array:
2244 $dataStructArray = BackendUtility::getFlexFormDS($conf, $origRecordRow, $table);
2245 $currentValueArray = GeneralUtility::xml2array($origRecordRow[$field]);
2246 // Do recursive processing of the XML data:
2247 /** @var $iteratorObj \TYPO3\CMS\Core\DataHandling\DataHandler */
2248 $iteratorObj = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\DataHandling\\DataHandler');
2249 $iteratorObj->callBackObj = $this;
2250 $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData($currentValueArray['data'], array(), array(), $dataStructArray, array($table, $uid, $field, $softRefCfgs), 'processSoftReferences_flexFormCallBack');
2251 // The return value is set as an array which means it will be processed by tcemain for file and DB references!
2252 if (is_array($currentValueArray['data'])) {
2253 $inData[$table][$thisNewUid][$field] = $currentValueArray;
2254 }
2255 }
2256 } else {
2257 // Get tokenizedContent string and proceed only if that is not blank:
2258 $tokenizedContent = $this->dat['records'][$table . ':' . $uid]['rels'][$field]['softrefs']['tokenizedContent'];
2259 if (strlen($tokenizedContent) && is_array($softRefCfgs)) {
2260 $inData[$table][$thisNewUid][$field] = $this->processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid);
2261 }
2262 }
2263 }
2264 }
2265 }
2266 }
2267 }
2268 }
2269 // Now write to database:
2270 $tce = $this->getNewTCE();
2271 $tce->isImporting = TRUE;
2272 $this->callHook('before_processSoftReferences', array(
2273 'tce' => &$tce,
2274 'data' => &$inData
2275 ));
2276 $tce->enableLogging = TRUE;
2277 $tce->start($inData, array());
2278 $tce->process_datamap();
2279 $this->callHook('after_processSoftReferences', array(
2280 'tce' => &$tce
2281 ));
2282 }
2283
2284 /**
2285 * Callback function for traversing the FlexForm structure in relation to remapping softreference relations
2286 *
2287 * @param array $pParams Set of parameters in numeric array: table, uid, field
2288 * @param array $dsConf TCA config for field (from Data Structure of course)
2289 * @param string $dataValue Field value (from FlexForm XML)
2290 * @param string $dataValue_ext1 Not used
2291 * @param string $dataValue_ext2 Not used
2292 * @param string $path Path of where the data structure where the element is found
2293 * @return array Array where the "value" key carries the value.
2294 * @see setFlexFormRelations()
2295 * @todo Define visibility
2296 */
2297 public function processSoftReferences_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path) {
2298 // Extract parameters:
2299 list($table, $origUid, $field, $softRefCfgs) = $pParams;
2300 if (is_array($softRefCfgs)) {
2301 // First, find all soft reference configurations for this structure path (they are listed flat in the header):
2302 $thisSoftRefCfgList = array();
2303 foreach ($softRefCfgs as $sK => $sV) {
2304 if ($sV['structurePath'] === $path) {
2305 $thisSoftRefCfgList[$sK] = $sV;
2306 }
2307 }
2308 // If any was found, do processing:
2309 if (count($thisSoftRefCfgList)) {
2310 // Get tokenizedContent string and proceed only if that is not blank:
2311 $tokenizedContent = $this->dat['records'][$table . ':' . $origUid]['rels'][$field]['flexFormRels']['softrefs'][$path]['tokenizedContent'];
2312 if (strlen($tokenizedContent)) {
2313 $dataValue = $this->processSoftReferences_substTokens($tokenizedContent, $thisSoftRefCfgList, $table, $origUid);
2314 }
2315 }
2316 }
2317 // Return
2318 return array('value' => $dataValue);
2319 }
2320
2321 /**
2322 * Substition of softreference tokens
2323 *
2324 * @param string $tokenizedContent Content of field with soft reference tokens in.
2325 * @param array $softRefCfgs Soft reference configurations
2326 * @param string $table Table for which the processing occurs
2327 * @param string $uid UID of record from table
2328 * @return string The input content with tokens substituted according to entries in softRefCfgs
2329 * @todo Define visibility
2330 */
2331 public function processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid) {
2332 // traverse each softref type for this field:
2333 foreach ($softRefCfgs as $cfg) {
2334 // Get token ID:
2335 $tokenID = $cfg['subst']['tokenID'];
2336 // Default is current token value:
2337 $insertValue = $cfg['subst']['tokenValue'];
2338 // Based on mode:
2339 switch ((string) $this->softrefCfg[$tokenID]['mode']) {
2340 case 'exclude':
2341 // Exclude is a simple passthrough of the value
2342 break;
2343 case 'editable':
2344 // Editable always picks up the value from this input array:
2345 $insertValue = $this->softrefInputValues[$tokenID];
2346 break;
2347 default:
2348 // Mapping IDs/creating files: Based on type, look up new value:
2349 switch ((string) $cfg['subst']['type']) {
2350 case 'file':
2351 // Create / Overwrite file:
2352 $insertValue = $this->processSoftReferences_saveFile($cfg['subst']['relFileName'], $cfg, $table, $uid);
2353 break;
2354 case 'db':
2355 default:
2356 // Trying to map database element if found in the mapID array:
2357 list($tempTable, $tempUid) = explode(':', $cfg['subst']['recordRef']);
2358 if (isset($this->import_mapId[$tempTable][$tempUid])) {
2359 $insertValue = BackendUtility::wsMapId($tempTable, $this->import_mapId[$tempTable][$tempUid]);
2360 // Look if reference is to a page and the original token value was NOT an integer - then we assume is was an alias and try to look up the new one!
2361 if ($tempTable === 'pages' && !\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($cfg['subst']['tokenValue'])) {
2362 $recWithUniqueValue = BackendUtility::getRecord($tempTable, $insertValue, 'alias');
2363 if ($recWithUniqueValue['alias']) {
2364 $insertValue = $recWithUniqueValue['alias'];
2365 }
2366 } elseif (strpos($cfg['subst']['tokenValue'], ':') !== FALSE) {
2367 list($tokenKey, $tokenId) = explode(':', $cfg['subst']['tokenValue']);
2368 $insertValue = $tokenKey . ':' . $insertValue;
2369 }
2370 }
2371 }
2372 }
2373 // Finally, swap the soft reference token in tokenized content with the insert value:
2374 $tokenizedContent = str_replace('{softref:' . $tokenID . '}', $insertValue, $tokenizedContent);
2375 }
2376 return $tokenizedContent;
2377 }
2378
2379 /**
2380 * Process a soft reference file
2381 *
2382 * @param string $relFileName Old Relative filename
2383 * @param array $cfg soft reference configuration array
2384 * @param string $table Table for which the processing occurs
2385 * @param string $uid UID of record from table
2386 * @return string New relative filename (value to insert instead of the softref token)
2387 * @todo Define visibility
2388 */
2389 public function processSoftReferences_saveFile($relFileName, $cfg, $table, $uid) {
2390 if ($fileHeaderInfo = $this->dat['header']['files'][$cfg['file_ID']]) {
2391 // Initialize; Get directory prefix for file and find possible RTE filename
2392 $dirPrefix = dirname($relFileName) . '/';
2393 $rteOrigName = $this->getRTEoriginalFilename(basename($relFileName));
2394 // If filename looks like an RTE file, and the directory is in "uploads/", then process as a RTE file!
2395 if ($rteOrigName && GeneralUtility::isFirstPartOfStr($dirPrefix, 'uploads/')) {
2396 // RTE:
2397 // First, find unique RTE file name:
2398 if (@is_dir((PATH_site . $dirPrefix))) {
2399 // From the "original" RTE filename, produce a new "original" destination filename which is unused. Even if updated, the image should be unique. Currently the problem with this is that it leaves a lot of unused RTE images...
2400 $fileProcObj = $this->getFileProcObj();
2401 $origDestName = $fileProcObj->getUniqueName($rteOrigName, PATH_site . $dirPrefix);
2402 // Create copy file name:
2403 $pI = pathinfo($relFileName);
2404 $copyDestName = dirname($origDestName) . '/RTEmagicC_' . substr(basename($origDestName), 10) . '.' . $pI['extension'];
2405 if (!@is_file($copyDestName) && !@is_file($origDestName) && $origDestName === GeneralUtility::getFileAbsFileName($origDestName) && $copyDestName === GeneralUtility::getFileAbsFileName($copyDestName)) {
2406 if ($this->dat['header']['files'][$fileHeaderInfo['RTE_ORIG_ID']]) {
2407 // Write the copy and original RTE file to the respective filenames:
2408 $this->writeFileVerify($copyDestName, $cfg['file_ID'], TRUE);
2409 $this->writeFileVerify($origDestName, $fileHeaderInfo['RTE_ORIG_ID'], TRUE);
2410 // Return the relative path of the copy file name:
2411 return \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($copyDestName);
2412 } else {
2413 $this->error('ERROR: Could not find original file ID');
2414 }
2415 } else {
2416 $this->error('ERROR: The destination filenames "' . $copyDestName . '" and "' . $origDestName . '" either existed or have non-valid names');
2417 }
2418 } else {
2419 $this->error('ERROR: "' . PATH_site . $dirPrefix . '" was not a directory, so could not process file "' . $relFileName . '"');
2420 }
2421 } elseif (GeneralUtility::isFirstPartOfStr($dirPrefix, $this->fileadminFolderName . '/')) {
2422 // File in fileadmin/ folder:
2423 // Create file (and possible resources)
2424 $newFileName = $this->processSoftReferences_saveFile_createRelFile($dirPrefix, basename($relFileName), $cfg['file_ID'], $table, $uid);
2425 if (strlen($newFileName)) {
2426 $relFileName = $newFileName;
2427 } else {
2428 $this->error('ERROR: No new file created for "' . $relFileName . '"');
2429 }
2430 } else {
2431 $this->error('ERROR: Sorry, cannot operate on non-RTE files which are outside the fileadmin folder.');
2432 }
2433 } else {
2434 $this->error('ERROR: Could not find file ID in header.');
2435 }
2436 // Return (new) filename relative to PATH_site:
2437 return $relFileName;
2438 }
2439
2440 /**
2441 * Create file in directory and return the new (unique) filename
2442 *
2443 * @param string $origDirPrefix Directory prefix, relative, with trailing slash
2444 * @param string $fileName Filename (without path)
2445 * @param string $fileID File ID from import memory
2446 * @param string $table Table for which the processing occurs
2447 * @param string $uid UID of record from table
2448 * @return string New relative filename, if any
2449 * @todo Define visibility
2450 */
2451 public function processSoftReferences_saveFile_createRelFile($origDirPrefix, $fileName, $fileID, $table, $uid) {
2452 // If the fileID map contains an entry for this fileID then just return the relative filename of that entry; we don't want to write another unique filename for this one!
2453 if ($this->fileIDMap[$fileID]) {
2454 return \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($this->fileIDMap[$fileID]);
2455 }
2456 // Verify FileMount access to dir-prefix. Returns the best alternative relative path if any
2457 $dirPrefix = $this->verifyFolderAccess($origDirPrefix);
2458 if ($dirPrefix && (!$this->update || $origDirPrefix === $dirPrefix) && $this->checkOrCreateDir($dirPrefix)) {
2459 $fileHeaderInfo = $this->dat['header']['files'][$fileID];
2460 $updMode = $this->update && $this->import_mapId[$table][$uid] === $uid && $this->import_mode[$table . ':' . $uid] !== 'as_new';
2461 // Create new name for file:
2462 // Must have same ID in map array (just for security, is not really needed) and NOT be set "as_new".
2463 if ($updMode) {
2464 $newName = PATH_site . $dirPrefix . $fileName;
2465 } else {
2466 // Create unique filename:
2467 $fileProcObj = $this->getFileProcObj();
2468 $newName = $fileProcObj->getUniqueName($fileName, PATH_site . $dirPrefix);
2469 }
2470 // Write main file:
2471 if ($this->writeFileVerify($newName, $fileID)) {
2472 // If the resource was an HTML/CSS file with resources attached, we will write those as well!
2473 if (is_array($fileHeaderInfo['EXT_RES_ID'])) {
2474 $tokenizedContent = $this->dat['files'][$fileID]['tokenizedContent'];
2475 $tokenSubstituted = FALSE;
2476 $fileProcObj = $this->getFileProcObj();
2477 if ($updMode) {
2478 foreach ($fileHeaderInfo['EXT_RES_ID'] as $res_fileID) {
2479 if ($this->dat['files'][$res_fileID]['filename']) {
2480 // Resolve original filename:
2481 $relResourceFileName = $this->dat['files'][$res_fileID]['parentRelFileName'];
2482 $absResourceFileName = GeneralUtility::resolveBackPath(PATH_site . $origDirPrefix . $relResourceFileName);
2483 $absResourceFileName = GeneralUtility::getFileAbsFileName($absResourceFileName);
2484 if ($absResourceFileName && GeneralUtility::isFirstPartOfStr($absResourceFileName, PATH_site . $this->fileadminFolderName . '/')) {
2485 $destDir = \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix(dirname($absResourceFileName) . '/');
2486 if ($this->verifyFolderAccess($destDir, TRUE) && $this->checkOrCreateDir($destDir)) {
2487 $this->writeFileVerify($absResourceFileName, $res_fileID);
2488 } else {
2489 $this->error('ERROR: Could not create file in directory "' . $destDir . '"');
2490 }
2491 } else {
2492 $this->error('ERROR: Could not resolve path for "' . $relResourceFileName . '"');
2493 }
2494 $tokenizedContent = str_replace('{EXT_RES_ID:' . $res_fileID . '}', $relResourceFileName, $tokenizedContent);
2495 $tokenSubstituted = TRUE;
2496 }
2497 }
2498 } else {
2499 // Create the resouces directory name (filename without extension, suffixed "_FILES")
2500 $resourceDir = dirname($newName) . '/' . preg_replace('/\\.[^.]*$/', '', basename($newName)) . '_FILES';
2501 if (GeneralUtility::mkdir($resourceDir)) {
2502 foreach ($fileHeaderInfo['EXT_RES_ID'] as $res_fileID) {
2503 if ($this->dat['files'][$res_fileID]['filename']) {
2504 $absResourceFileName = $fileProcObj->getUniqueName($this->dat['files'][$res_fileID]['filename'], $resourceDir);
2505 $relResourceFileName = substr($absResourceFileName, strlen(dirname($resourceDir)) + 1);
2506 $this->writeFileVerify($absResourceFileName, $res_fileID);
2507 $tokenizedContent = str_replace('{EXT_RES_ID:' . $res_fileID . '}', $relResourceFileName, $tokenizedContent);
2508 $tokenSubstituted = TRUE;
2509 }
2510 }
2511 }
2512 }
2513 // If substitutions has been made, write the content to the file again:
2514 if ($tokenSubstituted) {
2515 GeneralUtility::writeFile($newName, $tokenizedContent);
2516 }
2517 }
2518 return \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($newName);
2519 }
2520 }
2521 }
2522
2523 /**
2524 * Writes a file from the import memory having $fileID to file name $fileName which must be an absolute path inside PATH_site
2525 *
2526 * @param string $fileName Absolute filename inside PATH_site to write to
2527 * @param string $fileID File ID from import memory
2528 * @param boolean $bypassMountCheck Bypasses the checking against filemounts - only for RTE files!
2529 * @return boolean Returns TRUE if it went well. Notice that the content of the file is read again, and md5 from import memory is validated.
2530 * @todo Define visibility
2531 */
2532 public function writeFileVerify($fileName, $fileID, $bypassMountCheck = FALSE) {
2533 $fileProcObj = $this->getFileProcObj();
2534 if ($fileProcObj->actionPerms['addFile']) {
2535 // Just for security, check again. Should actually not be necessary.
2536 if ($fileProcObj->checkPathAgainstMounts($fileName) || $bypassMountCheck) {
2537 $fI = GeneralUtility::split_fileref($fileName);
2538 if ($fileProcObj->checkIfAllowed($fI['fileext'], $fI['path'], $fI['file']) || $this->allowPHPScripts && $GLOBALS['BE_USER']->isAdmin()) {
2539 if (GeneralUtility::getFileAbsFileName($fileName)) {
2540 if ($this->dat['files'][$fileID]) {
2541 GeneralUtility::writeFile($fileName, $this->dat['files'][$fileID]['content']);
2542 $this->fileIDMap[$fileID] = $fileName;
2543 if (md5(GeneralUtility::getUrl($fileName)) == $this->dat['files'][$fileID]['content_md5']) {
2544 return TRUE;
2545 } else {
2546 $this->error('ERROR: File content "' . $fileName . '" was corrupted');
2547 }
2548 } else {
2549 $this->error('ERROR: File ID "' . $fileID . '" could not be found');
2550 }
2551 } else {
2552 $this->error('ERROR: Filename "' . $fileName . '" was not a valid relative file path!');
2553 }
2554 } else {
2555 $this->error('ERROR: Filename "' . $fileName . '" failed against extension check or deny-pattern!');
2556 }
2557 } else {
2558 $this->error('ERROR: Filename "' . $fileName . '" was not allowed in destination path!');
2559 }
2560 } else {
2561 $this->error('ERROR: You did not have sufficient permissions to write the file "' . $fileName . '"');
2562 }
2563 }
2564
2565 /**
2566 * Returns TRUE if directory exists and if it doesn't it will create directory and return TRUE if that succeeded.
2567 *
2568 * @param string $dirPrefix Directory to create. Having a trailing slash. Must be in fileadmin/. Relative to PATH_site
2569 * @return boolean TRUE, if directory exists (was created)
2570 * @todo Define visibility
2571 */
2572 public function checkOrCreateDir($dirPrefix) {
2573 // Split dir path and remove first directory (which should be "fileadmin")
2574 $filePathParts = explode('/', $dirPrefix);
2575 $firstDir = array_shift($filePathParts);
2576 if ($firstDir === $this->fileadminFolderName && GeneralUtility::getFileAbsFileName($dirPrefix)) {
2577 $pathAcc = '';
2578 foreach ($filePathParts as $dirname) {
2579 $pathAcc .= '/' . $dirname;
2580 if (strlen($dirname)) {
2581 if (!@is_dir((PATH_site . $this->fileadminFolderName . $pathAcc))) {
2582 if (!GeneralUtility::mkdir((PATH_site . $this->fileadminFolderName . $pathAcc))) {
2583 $this->error('ERROR: Directory could not be created....B');
2584 return FALSE;
2585 }
2586 }
2587 } elseif ($dirPrefix === $this->fileadminFolderName . $pathAcc) {
2588 return TRUE;
2589 } else {
2590 $this->error('ERROR: Directory could not be created....A');
2591 }
2592 }
2593 }
2594 }
2595
2596 /**
2597 * Verifies that the input path (relative to PATH_site) is found in the backend users filemounts.
2598 * If it doesn't it will try to find another relative filemount for the user and return an alternative path prefix for the file.
2599 *
2600 * @param string $dirPrefix Path relative to PATH_site
2601 * @param boolean $noAlternative If set, Do not look for alternative path! Just return FALSE
2602 * @return string If a path is available that will be returned, otherwise FALSE.
2603 * @todo Define visibility
2604 */
2605 public function verifyFolderAccess($dirPrefix, $noAlternative = FALSE) {
2606 $fileProcObj = $this->getFileProcObj();
2607 // Check, if dirPrefix is inside a valid Filemount for user:
2608 $result = $fileProcObj->checkPathAgainstMounts(PATH_site . $dirPrefix);
2609 // If not, try to find another relative filemount and use that instead:
2610 if (!$result) {
2611 if ($noAlternative) {
2612 return FALSE;
2613 }
2614 // Find first web folder:
2615 $result = $fileProcObj->findFirstWebFolder();
2616 // If that succeeded, return the path to it:
2617 if ($result) {
2618 // Remove the "fileadmin/" prefix of input path - and append the rest to the return value:
2619 if (GeneralUtility::isFirstPartOfStr($dirPrefix, $this->fileadminFolderName . '/')) {
2620 $dirPrefix = substr($dirPrefix, strlen($this->fileadminFolderName . '/'));
2621 }
2622 return \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($fileProcObj->mounts[$result]['path'] . $dirPrefix);
2623 }
2624 } else {
2625 return $dirPrefix;
2626 }
2627 }
2628
2629 /**************************
2630 * File Input
2631 *************************/
2632
2633 /**
2634 * Loads the header section/all of the $filename into memory
2635 *
2636 * @param string $filename Filename, absolute
2637 * @param boolean $all If set, all information is loaded (header, records and files). Otherwise the default is to read only the header information
2638 * @return boolean TRUE if the operation went well
2639 * @todo Define visibility
2640 */
2641 public function loadFile($filename, $all = 0) {
2642 if (@is_file($filename)) {
2643 $fI = pathinfo($filename);
2644 if (strtolower($fI['extension']) == 'xml') {
2645 // XML:
2646 $xmlContent = GeneralUtility::getUrl($filename);
2647 if (strlen($xmlContent)) {
2648 $this->dat = GeneralUtility::xml2array($xmlContent, '', TRUE);
2649 if (is_array($this->dat)) {
2650 if ($this->dat['_DOCUMENT_TAG'] === 'T3RecordDocument' && is_array($this->dat['header']) && is_array($this->dat['records'])) {
2651 $this->loadInit();
2652 return TRUE;
2653 } else {
2654 $this->error('XML file did not contain proper XML for TYPO3 Import');
2655 }
2656 } else {
2657 $this->error('XML could not be parsed: ' . $this->dat);
2658 }
2659 } else {
2660 $this->error('Error opening file: ' . $filename);
2661 }
2662 } else {
2663 // T3D
2664 if ($fd = fopen($filename, 'rb')) {
2665 $this->dat['header'] = $this->getNextFilePart($fd, 1, 'header');
2666 if ($all) {
2667 $this->dat['records'] = $this->getNextFilePart($fd, 1, 'records');
2668 $this->dat['files'] = $this->getNextFilePart($fd, 1, 'files');
2669 $this->dat['files_fal'] = $this->getNextFilePart($fd, 1, 'files_fal');
2670 }
2671 $this->loadInit();
2672 return TRUE;
2673 } else {
2674 $this->error('Error opening file: ' . $filename);
2675 }
2676 fclose($fd);
2677 }
2678 } else {
2679 $this->error('Filename not found: ' . $filename);
2680 }
2681 return FALSE;
2682 }
2683
2684 /**
2685 * Returns the next content part form the fileresource (t3d), $fd
2686 *
2687 * @param pointer $fd File pointer
2688 * @param boolean $unserialize If set, the returned content is unserialized into an array, otherwise you get the raw string
2689 * @param string $name For error messages this indicates the section of the problem.
2690 * @return string Data string
2691 * @access private
2692 * @see loadFile()
2693 * @todo Define visibility
2694 */
2695 public function getNextFilePart($fd, $unserialize = 0, $name = '') {
2696 $initStrLen = 32 + 1 + 1 + 1 + 10 + 1;
2697 // Getting header data
2698 $initStr = fread($fd, $initStrLen);
2699 $initStrDat = explode(':', $initStr);
2700 if (strstr($initStrDat[0], 'Warning') == FALSE) {
2701 if ((string)$initStrDat[3] === '') {
2702 $datString = fread($fd, (int)$initStrDat[2]);
2703 fread($fd, 1);
2704 if (md5($datString) === $initStrDat[0]) {
2705 if ($initStrDat[1]) {
2706 if ($this->compress) {
2707 $datString = gzuncompress($datString);
2708 } else {
2709 $this->error('Content read error: This file requires decompression, but this server does not offer gzcompress()/gzuncompress() functions.', 1);
2710 }
2711 }
2712 return $unserialize ? unserialize($datString) : $datString;
2713 } else {
2714 $this->error('MD5 check failed (' . $name . ')');
2715 }
2716 } else {
2717 $this->error('File read error: InitString had a wrong length. (' . $name . ')');
2718 }
2719 } else {
2720 $this->error('File read error: Warning message in file. (' . $initStr . fgets($fd) . ')');
2721 }
2722 }
2723
2724 /**
2725 * Loads T3D file content into the $this->dat array
2726 * (This function can be used to test the output strings from ->compileMemoryToFileContent())
2727 *
2728 * @param string $filecontent File content
2729 * @return void
2730 * @todo Define visibility
2731 */
2732 public function loadContent($filecontent) {
2733 $pointer = 0;
2734 $this->dat['header'] = $this->getNextContentPart($filecontent, $pointer, 1, 'header');
2735 $this->dat['records'] = $this->getNextContentPart($filecontent, $pointer, 1, 'records');
2736 $this->dat['files'] = $this->getNextContentPart($filecontent, $pointer, 1, 'files');
2737 $this->loadInit();
2738 }
2739
2740 /**
2741 * Returns the next content part from the $filecontent
2742 *
2743 * @param string $filecontent File content string
2744 * @param integer $pointer File pointer (where to read from)
2745 * @param boolean $unserialize If set, the returned content is unserialized into an array, otherwise you get the raw string
2746 * @param string $name For error messages this indicates the section of the problem.
2747 * @return string Data string
2748 * @todo Define visibility
2749 */
2750 public function getNextContentPart($filecontent, &$pointer, $unserialize = 0, $name = '') {
2751 $initStrLen = 32 + 1 + 1 + 1 + 10 + 1;
2752 // getting header data
2753 $initStr = substr($filecontent, $pointer, $initStrLen);
2754 $pointer += $initStrLen;
2755 $initStrDat = explode(':', $initStr);
2756 if ((string)$initStrDat[3] === '') {
2757 $datString = substr($filecontent, $pointer, (int)$initStrDat[2]);
2758 $pointer += (int)$initStrDat[2] + 1;
2759 if (md5($datString) === $initStrDat[0]) {
2760 if ($initStrDat[1]) {
2761 if ($this->compress) {
2762 $datString = gzuncompress($datString);
2763 } else {
2764 $this->error('Content read error: This file requires decompression, but this server does not offer gzcompress()/gzuncompress() functions.', 1);
2765 }
2766 }
2767 return $unserialize ? unserialize($datString) : $datString;
2768 } else {
2769 $this->error('MD5 check failed (' . $name . ')');
2770 }
2771 } else {
2772 $this->error('Content read error: InitString had a wrong length. (' . $name . ')');
2773 }
2774 }
2775
2776 /**
2777 * Setting up the object based on the recently loaded ->dat array
2778 *
2779 * @return void
2780 * @todo Define visibility
2781 */