e6e375bb693fb00b8d720b7d4ac9689448ca0b8c
[Packages/TYPO3.CMS.git] / typo3 / sysext / impexp / Classes / ImportExport.php
1 <?php
2 namespace TYPO3\CMS\Impexp;
3
4 /***************************************************************
5 * Copyright notice
6 *
7 * (c) 1999-2013 Kasper Skårhøj (kasperYYYY@typo3.com)
8 * All rights reserved
9 *
10 * This script is part of the TYPO3 project. The TYPO3 project is
11 * free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2 of the License, or
14 * (at your option) any later version.
15 *
16 * The GNU General Public License can be found at
17 * http://www.gnu.org/copyleft/gpl.html.
18 * A copy is found in the text file GPL.txt and important notices to the license
19 * from the author is found in LICENSE.txt distributed with these scripts.
20 *
21 *
22 * This script is distributed in the hope that it will be useful,
23 * but WITHOUT ANY WARRANTY; without even the implied warranty of
24 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25 * GNU General Public License for more details.
26 *
27 * This copyright notice MUST APPEAR in all copies of the script!
28 ***************************************************************/
29
30 use TYPO3\CMS\Core\Exception;
31 use TYPO3\CMS\Core\Utility\GeneralUtility;
32 use TYPO3\CMS\Backend\Utility\BackendUtility;
33
34 /**
35 * EXAMPLE for using the impexp-class for exporting stuff:
36 *
37 * Create and initialize:
38 * $this->export = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance('TYPO3\\CMS\\Impexp\\ImportExport');
39 * $this->export->init();
40 * Set which tables relations we will allow:
41 * $this->export->relOnlyTables[]="tt_news"; // exclusively includes. See comment in the class
42 *
43 * Adding records:
44 * $this->export->export_addRecord("pages", $this->pageinfo);
45 * $this->export->export_addRecord("pages", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("pages", 38));
46 * $this->export->export_addRecord("pages", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("pages", 39));
47 * $this->export->export_addRecord("tt_content", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("tt_content", 12));
48 * $this->export->export_addRecord("tt_content", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("tt_content", 74));
49 * $this->export->export_addRecord("sys_template", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("sys_template", 20));
50 *
51 * Adding all the relations (recursively in 5 levels so relations has THEIR relations registered as well)
52 * for($a=0;$a<5;$a++) {
53 * $addR = $this->export->export_addDBRelations($a);
54 * if (!count($addR)) break;
55 * }
56 *
57 * Finally load all the files.
58 * $this->export->export_addFilesFromRelations(); // MUST be after the DBrelations are set so that file from ALL added records are included!
59 *
60 * Write export
61 * $out = $this->export->compileMemoryToFileContent();
62 */
63 /**
64 * T3D file Import/Export library (TYPO3 Record Document)
65 *
66 * @author Kasper Skårhøj <kasperYYYY@typo3.com>
67 */
68 class ImportExport {
69
70 /**
71 * If set, static relations (not exported) will be shown in overview as well
72 *
73 * @todo Define visibility
74 */
75 public $showStaticRelations = FALSE;
76
77 /**
78 * Name of the "fileadmin" folder where files for export/import should be located
79 *
80 * @todo Define visibility
81 */
82 public $fileadminFolderName = '';
83
84 /**
85 * Whether "import" or "export" mode of object. Set through init() function
86 *
87 * @todo Define visibility
88 */
89 public $mode = '';
90
91 /**
92 * Updates all records that has same UID instead of creating new!
93 *
94 * @todo Define visibility
95 */
96 public $update = FALSE;
97
98 /**
99 * Is set by importData() when an import has been done.
100 *
101 * @todo Define visibility
102 */
103 public $doesImport = FALSE;
104
105 /**
106 * If set to a page-record, then the preview display of the content will expect this page-record to be the target
107 * for the import and accordingly display validation information. This triggers the visual view of the
108 * import/export memory to validate if import is possible
109 *
110 * @todo Define visibility
111 */
112 public $display_import_pid_record = '';
113
114 /**
115 * Used to register the forged UID values for imported records that we want to create with the same UIDs as in the import file. Admin-only feature.
116 *
117 * @todo Define visibility
118 */
119 public $suggestedInsertUids = array();
120
121 /**
122 * Setting import modes during update state: as_new, exclude, force_uid
123 *
124 * @todo Define visibility
125 */
126 public $import_mode = array();
127
128 /**
129 * If set, PID correct is ignored globally
130 *
131 * @todo Define visibility
132 */
133 public $global_ignore_pid = FALSE;
134
135 /**
136 * If set, all UID values are forced! (update or import)
137 *
138 * @todo Define visibility
139 */
140 public $force_all_UIDS = FALSE;
141
142 /**
143 * If set, a diff-view column is added to the overview.
144 *
145 * @todo Define visibility
146 */
147 public $showDiff = FALSE;
148
149 /**
150 * If set, and if the user is admin, allow the writing of PHP scripts to fileadmin/ area.
151 *
152 * @todo Define visibility
153 */
154 public $allowPHPScripts = FALSE;
155
156 /**
157 * Disable logging when importing
158 *
159 * @todo Define visibility
160 */
161 public $enableLogging = FALSE;
162
163 /**
164 * Array of values to substitute in editable softreferences.
165 *
166 * @todo Define visibility
167 */
168 public $softrefInputValues = array();
169
170 /**
171 * Mapping between the fileID from import memory and the final filenames they are written to.
172 *
173 * @todo Define visibility
174 */
175 public $fileIDMap = array();
176
177 /**
178 * 1MB max file size
179 *
180 * @todo Define visibility
181 */
182 public $maxFileSize = 1000000;
183
184 /**
185 * 1MB max record size
186 *
187 * @todo Define visibility
188 */
189 public $maxRecordSize = 1000000;
190
191 /**
192 * 10MB max export size
193 *
194 * @todo Define visibility
195 */
196 public $maxExportSize = 10000000;
197
198 /**
199 * Add table names here which are THE ONLY ones which will be included into export if found as relations. '_ALL' will allow all tables.
200 *
201 * @todo Define visibility
202 */
203 public $relOnlyTables = array();
204
205 /**
206 * Add tables names here which should not be exported with the file. (Where relations should be mapped to same UIDs in target system).
207 *
208 * @todo Define visibility
209 */
210 public $relStaticTables = array();
211
212 /**
213 * Exclude map. Keys are table:uid pairs and if set, records are not added to the export.
214 *
215 * @todo Define visibility
216 */
217 public $excludeMap = array();
218
219 /**
220 * Soft Reference Token ID modes.
221 *
222 * @todo Define visibility
223 */
224 public $softrefCfg = array();
225
226 /**
227 * Listing extension dependencies.
228 *
229 * @todo Define visibility
230 */
231 public $extensionDependencies = array();
232
233 /**
234 * Set by user: If set, compression in t3d files is disabled
235 *
236 * @todo Define visibility
237 */
238 public $dontCompress = 0;
239
240 /**
241 * Boolean, if set, HTML file resources are included.
242 *
243 * @todo Define visibility
244 */
245 public $includeExtFileResources = 0;
246
247 /**
248 * Files with external media (HTML/css style references inside)
249 *
250 * @todo Define visibility
251 */
252 public $extFileResourceExtensions = 'html,htm,css';
253
254 /**
255 * After records are written this array is filled with [table][original_uid] = [new_uid]
256 *
257 * @todo Define visibility
258 */
259 public $import_mapId = array();
260
261 /**
262 * Keys are [tablename]:[new NEWxxx ids (or when updating it is uids)] while values are arrays with table/uid of the original record it is based on.
263 * By the array keys the new ids can be looked up inside tcemain
264 *
265 * @todo Define visibility
266 */
267 public $import_newId = array();
268
269 /**
270 * Page id map for page tree (import)
271 *
272 * @todo Define visibility
273 */
274 public $import_newId_pids = array();
275
276 /**
277 * Internal data accumulation for writing records during import
278 *
279 * @todo Define visibility
280 */
281 public $import_data = array();
282
283 /**
284 * Error log.
285 *
286 * @todo Define visibility
287 */
288 public $errorLog = array();
289
290 /**
291 * Cache for record paths
292 *
293 * @todo Define visibility
294 */
295 public $cache_getRecordPath = array();
296
297 /**
298 * Cache of checkPID values.
299 *
300 * @todo Define visibility
301 */
302 public $checkPID_cache = array();
303
304 /**
305 * Set internally if the gzcompress function exists
306 *
307 * @todo Define visibility
308 */
309 public $compress = 0;
310
311 /**
312 * Internal import/export memory
313 *
314 * @todo Define visibility
315 */
316 public $dat = array();
317
318 /**
319 * File processing object
320 *
321 * @var \TYPO3\CMS\Core\Utility\File\ExtendedFileUtility
322 * @todo Define visibility
323 */
324 public $fileProcObj = '';
325
326 /**
327 * Keys are [recordname], values are an array of fields to be included
328 * in the export
329 *
330 * @var array
331 */
332 protected $recordTypesIncludeFields = array();
333
334 /**
335 * Default array of fields to be included in the export
336 *
337 * @var array
338 */
339 protected $defaultRecordIncludeFields = array('uid', 'pid');
340
341 /**
342 * Array of current registered storage objects
343 *
344 * @var array
345 */
346 protected $storageObjects = array();
347
348 /**************************
349 * Initialize
350 *************************/
351
352 /**
353 * Init the object, both import and export
354 *
355 * @param boolean $dontCompress If set, compression in t3d files is disabled
356 * @param string $mode Mode of usage, either "import" or "export
357 * @return void
358 * @todo Define visibility
359 */
360 public function init($dontCompress = 0, $mode = '') {
361 $this->compress = function_exists('gzcompress');
362 $this->dontCompress = $dontCompress;
363 $this->mode = $mode;
364 $this->fileadminFolderName = !empty($GLOBALS['TYPO3_CONF_VARS']['BE']['fileadminDir']) ? rtrim($GLOBALS['TYPO3_CONF_VARS']['BE']['fileadminDir'], '/') : 'fileadmin';
365 }
366
367 /**************************
368 * Export / Init + Meta Data
369 *************************/
370
371 /**
372 * Set header basics
373 *
374 * @return void
375 * @todo Define visibility
376 */
377 public function setHeaderBasics() {
378 // Initializing:
379 if (is_array($this->softrefCfg)) {
380 foreach ($this->softrefCfg as $key => $value) {
381 if (!strlen($value['mode'])) {
382 unset($this->softrefCfg[$key]);
383 }
384 }
385 }
386 // Setting in header memory:
387 // Version of file format
388 $this->dat['header']['XMLversion'] = '1.0';
389 // Initialize meta data array (to put it in top of file)
390 $this->dat['header']['meta'] = array();
391 // Add list of tables to consider static
392 $this->dat['header']['relStaticTables'] = $this->relStaticTables;
393 // The list of excluded records
394 $this->dat['header']['excludeMap'] = $this->excludeMap;
395 // Soft Reference mode for elements
396 $this->dat['header']['softrefCfg'] = $this->softrefCfg;
397 // List of extensions the import depends on.
398 $this->dat['header']['extensionDependencies'] = $this->extensionDependencies;
399 }
400
401 /**
402 * Set charset
403 *
404 * @param string $charset Charset for the content in the export. During import the character set will be converted if the target system uses another charset.
405 * @return void
406 * @todo Define visibility
407 */
408 public function setCharset($charset) {
409 $this->dat['header']['charset'] = $charset;
410 }
411
412 /**
413 * Sets meta data
414 *
415 * @param string $title Title of the export
416 * @param string $description Description of the export
417 * @param string $notes Notes about the contents
418 * @param string $packager_username Backend Username of the packager (the guy making the export)
419 * @param string $packager_name Real name of the packager
420 * @param string $packager_email Email of the packager
421 * @return void
422 * @todo Define visibility
423 */
424 public function setMetaData($title, $description, $notes, $packager_username, $packager_name, $packager_email) {
425 $this->dat['header']['meta'] = array(
426 'title' => $title,
427 'description' => $description,
428 'notes' => $notes,
429 'packager_username' => $packager_username,
430 'packager_name' => $packager_name,
431 'packager_email' => $packager_email,
432 'TYPO3_version' => TYPO3_version,
433 'created' => strftime('%A %e. %B %Y', $GLOBALS['EXEC_TIME'])
434 );
435 }
436
437 /**
438 * Sets a thumbnail image to the exported file
439 *
440 * @param string $imgFilepath Filename reference, gif, jpg, png. Absolute path.
441 * @return void
442 * @todo Define visibility
443 */
444 public function addThumbnail($imgFilepath) {
445 if (@is_file($imgFilepath)) {
446 $imgInfo = @getimagesize($imgFilepath);
447 if (is_array($imgInfo)) {
448 $fileContent = GeneralUtility::getUrl($imgFilepath);
449 $this->dat['header']['thumbnail'] = array(
450 'imgInfo' => $imgInfo,
451 'content' => $fileContent,
452 'filesize' => strlen($fileContent),
453 'filemtime' => filemtime($imgFilepath),
454 'filename' => basename($imgFilepath)
455 );
456 }
457 }
458 }
459
460 /**************************
461 * Export / Init Page tree
462 *************************/
463
464 /**
465 * Sets the page-tree array in the export header and returns the array in a flattened version
466 *
467 * @param array $idH Hierarchy of ids, the page tree: array([uid] => array("uid" => [uid], "subrow" => array(.....)), [uid] => ....)
468 * @return array The hierarchical page tree converted to a one-dimensional list of pages
469 * @todo Define visibility
470 */
471 public function setPageTree($idH) {
472 $this->dat['header']['pagetree'] = $this->unsetExcludedSections($idH);
473 return $this->flatInversePageTree($this->dat['header']['pagetree']);
474 }
475
476 /**
477 * Removes entries in the page tree which are found in ->excludeMap[]
478 *
479 * @param array $idH Page uid hierarchy
480 * @return array Modified input array
481 * @access private
482 * @see setPageTree()
483 * @todo Define visibility
484 */
485 public function unsetExcludedSections($idH) {
486 if (is_array($idH)) {
487 foreach ($idH as $k => $v) {
488 if ($this->excludeMap['pages:' . $idH[$k]['uid']]) {
489 unset($idH[$k]);
490 } elseif (is_array($idH[$k]['subrow'])) {
491 $idH[$k]['subrow'] = $this->unsetExcludedSections($idH[$k]['subrow']);
492 }
493 }
494 }
495 return $idH;
496 }
497
498 /**
499 * Recursively flattening the idH array (for setPageTree() function)
500 *
501 * @param array $idH Page uid hierarchy
502 * @param array $a Accumulation array of pages (internal, don't set from outside)
503 * @return array Array with uid-uid pairs for all pages in the page tree.
504 * @see flatInversePageTree_pid()
505 * @todo Define visibility
506 */
507 public function flatInversePageTree($idH, $a = array()) {
508 if (is_array($idH)) {
509 $idH = array_reverse($idH);
510 foreach ($idH as $k => $v) {
511 $a[$v['uid']] = $v['uid'];
512 if (is_array($v['subrow'])) {
513 $a = $this->flatInversePageTree($v['subrow'], $a);
514 }
515 }
516 }
517 return $a;
518 }
519
520 /**
521 * Recursively flattening the idH array (for setPageTree() function), setting PIDs as values
522 *
523 * @param array $idH Page uid hierarchy
524 * @param array $a Accumulation array of pages (internal, don't set from outside)
525 * @param integer $pid PID value (internal)
526 * @return array Array with uid-pid pairs for all pages in the page tree.
527 * @see flatInversePageTree()
528 * @todo Define visibility
529 */
530 public function flatInversePageTree_pid($idH, $a = array(), $pid = -1) {
531 if (is_array($idH)) {
532 $idH = array_reverse($idH);
533 foreach ($idH as $k => $v) {
534 $a[$v['uid']] = $pid;
535 if (is_array($v['subrow'])) {
536 $a = $this->flatInversePageTree_pid($v['subrow'], $a, $v['uid']);
537 }
538 }
539 }
540 return $a;
541 }
542
543 /**************************
544 * Export
545 *************************/
546
547 /**
548 * Sets the fields of record types to be included in the export
549 *
550 * @param array $recordTypesIncludeFields Keys are [recordname], values are an array of fields to be included in the export
551 * @throws \TYPO3\CMS\Core\Exception if an array value is not type of array
552 * @return void
553 */
554 public function setRecordTypesIncludeFields(array $recordTypesIncludeFields) {
555 foreach ($recordTypesIncludeFields as $table => $fields) {
556 if (!is_array($fields)) {
557 throw new \TYPO3\CMS\Core\Exception('The include fields for record type ' . htmlspecialchars($table) . ' are not defined by an array.', 1391440658);
558 }
559 $this->setRecordTypeIncludeFields($table, $fields);
560 }
561 }
562
563 /**
564 * Sets the fields of a record type to be included in the export
565 *
566 * @param string $table The record type
567 * @param array $fields The fields to be included
568 * @return void
569 */
570 public function setRecordTypeIncludeFields($table, array $fields) {
571 $this->recordTypesIncludeFields[$table] = $fields;
572 }
573
574 /**
575 * Adds the record $row from $table.
576 * No checking for relations done here. Pure data.
577 *
578 * @param string $table Table name
579 * @param array $row Record row.
580 * @param integer $relationLevel (Internal) if the record is added as a relation, this is set to the "level" it was on.
581 * @return void
582 * @todo Define visibility
583 */
584 public function export_addRecord($table, $row, $relationLevel = 0) {
585 BackendUtility::workspaceOL($table, $row);
586 if ((string)$table !== '' && is_array($row) && $row['uid'] > 0 && !$this->excludeMap[($table . ':' . $row['uid'])]) {
587 if ($this->checkPID($table === 'pages' ? $row['uid'] : $row['pid'])) {
588 if (!isset($this->dat['records'][($table . ':' . $row['uid'])])) {
589 // Prepare header info:
590 $row = $this->filterRecordFields($table, $row);
591 $headerInfo = array();
592 $headerInfo['uid'] = $row['uid'];
593 $headerInfo['pid'] = $row['pid'];
594 $headerInfo['title'] = GeneralUtility::fixed_lgd_cs(BackendUtility::getRecordTitle($table, $row), 40);
595 $headerInfo['size'] = strlen(serialize($row));
596 if ($relationLevel) {
597 $headerInfo['relationLevel'] = $relationLevel;
598 }
599 // If record content is not too large in size, set the header content and add the rest:
600 if ($headerInfo['size'] < $this->maxRecordSize) {
601 // Set the header summary:
602 $this->dat['header']['records'][$table][$row['uid']] = $headerInfo;
603 // Create entry in the PID lookup:
604 $this->dat['header']['pid_lookup'][$row['pid']][$table][$row['uid']] = 1;
605 // Initialize reference index object:
606 $refIndexObj = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\Database\\ReferenceIndex');
607 // Yes to workspace overlays for exporting....
608 $refIndexObj->WSOL = TRUE;
609 // Data:
610 $this->dat['records'][$table . ':' . $row['uid']] = array();
611 $this->dat['records'][$table . ':' . $row['uid']]['data'] = $row;
612 $this->dat['records'][$table . ':' . $row['uid']]['rels'] = $refIndexObj->getRelations($table, $row);
613 $this->errorLog = array_merge($this->errorLog, $refIndexObj->errorLog);
614 // Merge error logs.
615 // Add information about the relations in the record in the header:
616 $this->dat['header']['records'][$table][$row['uid']]['rels'] = $this->flatDBrels($this->dat['records'][$table . ':' . $row['uid']]['rels']);
617 // Add information about the softrefs to header:
618 $this->dat['header']['records'][$table][$row['uid']]['softrefs'] = $this->flatSoftRefs($this->dat['records'][$table . ':' . $row['uid']]['rels']);
619 } else {
620 $this->error('Record ' . $table . ':' . $row['uid'] . ' was larger than maxRecordSize (' . GeneralUtility::formatSize($this->maxRecordSize) . ')');
621 }
622 } else {
623 $this->error('Record ' . $table . ':' . $row['uid'] . ' already added.');
624 }
625 } else {
626 $this->error('Record ' . $table . ':' . $row['uid'] . ' was outside your DB mounts!');
627 }
628 }
629 }
630
631 /**
632 * This analyses the existing added records, finds all database relations to records and adds these records to the export file.
633 * This function can be called repeatedly until it returns an empty array. In principle it should not allow to infinite recursivity, but you better set a limit...
634 * Call this BEFORE the ext_addFilesFromRelations (so files from added relations are also included of course)
635 *
636 * @param integer $relationLevel Recursion level
637 * @return array overview of relations found and added: Keys [table]:[uid], values array with table and id
638 * @see export_addFilesFromRelations()
639 * @todo Define visibility
640 */
641 public function export_addDBRelations($relationLevel = 0) {
642 // Initialize:
643 $addR = array();
644 // Traverse all "rels" registered for "records"
645 if (is_array($this->dat['records'])) {
646 foreach ($this->dat['records'] as $k => $value) {
647 if (is_array($this->dat['records'][$k])) {
648 foreach ($this->dat['records'][$k]['rels'] as $fieldname => $vR) {
649 // For all DB types of relations:
650 if ($vR['type'] == 'db') {
651 foreach ($vR['itemArray'] as $fI) {
652 $this->export_addDBRelations_registerRelation($fI, $addR);
653 }
654 }
655 // For all flex/db types of relations:
656 if ($vR['type'] == 'flex') {
657 // DB relations in flex form fields:
658 if (is_array($vR['flexFormRels']['db'])) {
659 foreach ($vR['flexFormRels']['db'] as $subList) {
660 foreach ($subList as $fI) {
661 $this->export_addDBRelations_registerRelation($fI, $addR);
662 }
663 }
664 }
665 // DB oriented soft references in flex form fields:
666 if (is_array($vR['flexFormRels']['softrefs'])) {
667 foreach ($vR['flexFormRels']['softrefs'] as $subList) {
668 foreach ($subList['keys'] as $spKey => $elements) {
669 foreach ($elements as $el) {
670 if ($el['subst']['type'] === 'db' && $this->includeSoftref($el['subst']['tokenID'])) {
671 list($tempTable, $tempUid) = explode(':', $el['subst']['recordRef']);
672 $fI = array(
673 'table' => $tempTable,
674 'id' => $tempUid
675 );
676 $this->export_addDBRelations_registerRelation($fI, $addR, $el['subst']['tokenID']);
677 }
678 }
679 }
680 }
681 }
682 }
683 // In any case, if there are soft refs:
684 if (is_array($vR['softrefs']['keys'])) {
685 foreach ($vR['softrefs']['keys'] as $spKey => $elements) {
686 foreach ($elements as $el) {
687 if ($el['subst']['type'] === 'db' && $this->includeSoftref($el['subst']['tokenID'])) {
688 list($tempTable, $tempUid) = explode(':', $el['subst']['recordRef']);
689 $fI = array(
690 'table' => $tempTable,
691 'id' => $tempUid
692 );
693 $this->export_addDBRelations_registerRelation($fI, $addR, $el['subst']['tokenID']);
694 }
695 }
696 }
697 }
698 }
699 }
700 }
701 } else {
702 $this->error('There were no records available.');
703 }
704 // Now, if there were new records to add, do so:
705 if (count($addR)) {
706 foreach ($addR as $fI) {
707 // Get and set record:
708 $row = BackendUtility::getRecord($fI['table'], $fI['id']);
709 if (is_array($row)) {
710 $this->export_addRecord($fI['table'], $row, $relationLevel + 1);
711 }
712 // Set status message
713 // Relation pointers always larger than zero except certain "select" types with negative values pointing to uids - but that is not supported here.
714 if ($fI['id'] > 0) {
715 $rId = $fI['table'] . ':' . $fI['id'];
716 if (!isset($this->dat['records'][$rId])) {
717 $this->dat['records'][$rId] = 'NOT_FOUND';
718 $this->error('Relation record ' . $rId . ' was not found!');
719 }
720 }
721 }
722 }
723 // Return overview of relations found and added
724 return $addR;
725 }
726
727 /**
728 * Helper function for export_addDBRelations()
729 *
730 * @param array $fI Array with table/id keys to add
731 * @param array $addR Add array, passed by reference to be modified
732 * @param string $tokenID Softref Token ID, if applicable.
733 * @return void
734 * @see export_addDBRelations()
735 * @todo Define visibility
736 */
737 public function export_addDBRelations_registerRelation($fI, &$addR, $tokenID = '') {
738 $rId = $fI['table'] . ':' . $fI['id'];
739 if (isset($GLOBALS['TCA'][$fI['table']]) && !$this->isTableStatic($fI['table']) && !$this->isExcluded($fI['table'], $fI['id']) && (!$tokenID || $this->includeSoftref($tokenID)) && $this->inclRelation($fI['table'])) {
740 if (!isset($this->dat['records'][$rId])) {
741 // Set this record to be included since it is not already.
742 $addR[$rId] = $fI;
743 }
744 }
745 }
746
747 /**
748 * This adds all files in relations.
749 * Call this method AFTER adding all records including relations.
750 *
751 * @return void
752 * @see export_addDBRelations()
753 * @todo Define visibility
754 */
755 public function export_addFilesFromRelations() {
756 // Traverse all "rels" registered for "records"
757 if (is_array($this->dat['records'])) {
758 foreach ($this->dat['records'] as $k => $value) {
759 if (isset($this->dat['records'][$k]['rels']) && is_array($this->dat['records'][$k]['rels'])) {
760 foreach ($this->dat['records'][$k]['rels'] as $fieldname => $vR) {
761 // For all file type relations:
762 if ($vR['type'] == 'file') {
763 foreach ($vR['newValueFiles'] as $key => $fI) {
764 $this->export_addFile($fI, $k, $fieldname);
765 // Remove the absolute reference to the file so it doesn't expose absolute paths from source server:
766 unset($this->dat['records'][$k]['rels'][$fieldname]['newValueFiles'][$key]['ID_absFile']);
767 }
768 }
769 // For all flex type relations:
770 if ($vR['type'] == 'flex') {
771 if (is_array($vR['flexFormRels']['file'])) {
772 foreach ($vR['flexFormRels']['file'] as $key => $subList) {
773 foreach ($subList as $subKey => $fI) {
774 $this->export_addFile($fI, $k, $fieldname);
775 // Remove the absolute reference to the file so it doesn't expose absolute paths from source server:
776 unset($this->dat['records'][$k]['rels'][$fieldname]['flexFormRels']['file'][$key][$subKey]['ID_absFile']);
777 }
778 }
779 }
780 // DB oriented soft references in flex form fields:
781 if (is_array($vR['flexFormRels']['softrefs'])) {
782 foreach ($vR['flexFormRels']['softrefs'] as $key => $subList) {
783 foreach ($subList['keys'] as $spKey => $elements) {
784 foreach ($elements as $subKey => $el) {
785 if ($el['subst']['type'] === 'file' && $this->includeSoftref($el['subst']['tokenID'])) {
786 // Create abs path and ID for file:
787 $ID_absFile = GeneralUtility::getFileAbsFileName(PATH_site . $el['subst']['relFileName']);
788 $ID = md5($ID_absFile);
789 if ($ID_absFile) {
790 if (!$this->dat['files'][$ID]) {
791 $fI = array(
792 'filename' => basename($ID_absFile),
793 'ID_absFile' => $ID_absFile,
794 'ID' => $ID,
795 'relFileName' => $el['subst']['relFileName']
796 );
797 $this->export_addFile($fI, '_SOFTREF_');
798 }
799 $this->dat['records'][$k]['rels'][$fieldname]['flexFormRels']['softrefs'][$key]['keys'][$spKey][$subKey]['file_ID'] = $ID;
800 }
801 }
802 }
803 }
804 }
805 }
806 }
807 // In any case, if there are soft refs:
808 if (is_array($vR['softrefs']['keys'])) {
809 foreach ($vR['softrefs']['keys'] as $spKey => $elements) {
810 foreach ($elements as $subKey => $el) {
811 if ($el['subst']['type'] === 'file' && $this->includeSoftref($el['subst']['tokenID'])) {
812 // Create abs path and ID for file:
813 $ID_absFile = GeneralUtility::getFileAbsFileName(PATH_site . $el['subst']['relFileName']);
814 $ID = md5($ID_absFile);
815 if ($ID_absFile) {
816 if (!$this->dat['files'][$ID]) {
817 $fI = array(
818 'filename' => basename($ID_absFile),
819 'ID_absFile' => $ID_absFile,
820 'ID' => $ID,
821 'relFileName' => $el['subst']['relFileName']
822 );
823 $this->export_addFile($fI, '_SOFTREF_');
824 }
825 $this->dat['records'][$k]['rels'][$fieldname]['softrefs']['keys'][$spKey][$subKey]['file_ID'] = $ID;
826 }
827 }
828 }
829 }
830 }
831 }
832 }
833 }
834 } else {
835 $this->error('There were no records available.');
836 }
837 }
838
839 /**
840 * This adds all files from sys_file records
841 *
842 * @return void
843 */
844 public function export_addFilesFromSysFilesRecords() {
845 if (!isset($this->dat['header']['records']['sys_file']) || !is_array($this->dat['header']['records']['sys_file'])) {
846 return;
847 }
848 foreach (array_keys($this->dat['header']['records']['sys_file']) as $sysFileUid) {
849 $recordData = $this->dat['records']['sys_file:' . $sysFileUid]['data'];
850 $file = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->createFileObject($recordData);
851 $this->export_addSysFile($file);
852 }
853 }
854
855 /**
856 * Adds a files content from a sys file record to the export memory
857 *
858 * @param \TYPO3\CMS\Core\Resource\File $file
859 * @return void
860 */
861 public function export_addSysFile(\TYPO3\CMS\Core\Resource\File $file) {
862 if ($file->getProperty('size') >= $this->maxFileSize) {
863 $this->error('File ' . $file->getPublicUrl() . ' was larger (' . GeneralUtility::formatSize($file->getProperty('size')) . ') than the maxFileSize (' . GeneralUtility::formatSize($this->maxFileSize) . ')! Skipping.');
864 return;
865 }
866 try {
867 $fileContent = $file->getContents();
868 } catch (\TYPO3\CMS\Core\Resource\Exception\InsufficientFileAccessPermissionsException $e) {
869 $this->error('File ' . $file->getPublicUrl() . ': ' . $e->getMessage());
870 return;
871 } catch (\TYPO3\CMS\Core\Resource\Exception\IllegalFileExtensionException $e) {
872 $this->error('File ' . $file->getPublicUrl() . ': ' . $e->getMessage());
873 return;
874 }
875 $fileRec = array();
876 $fileRec['filesize'] = $file->getProperty('size');
877 $fileRec['filename'] = $file->getProperty('name');
878 $fileRec['filemtime'] = $file->getProperty('modification_date');
879
880 // build unique id based on the storage and the file identifier
881 $fileId = md5($file->getStorage()->getUid() . ':' . $file->getProperty('identifier_hash'));
882
883 // Setting this data in the header
884 $this->dat['header']['files_fal'][$fileId] = $fileRec;
885
886 // ... and finally add the heavy stuff:
887 $fileRec['content'] = $fileContent;
888 $fileRec['content_sha1'] = $file->getProperty('sha1');
889
890 $this->dat['files_fal'][$fileId] = $fileRec;
891 }
892
893
894 /**
895 * Adds a files content to the export memory
896 *
897 * @param array $fI File information with three keys: "filename" = filename without path, "ID_absFile" = absolute filepath to the file (including the filename), "ID" = md5 hash of "ID_absFile". "relFileName" is optional for files attached to records, but mandatory for soft referenced files (since the relFileName determines where such a file should be stored!)
898 * @param string $recordRef If the file is related to a record, this is the id on the form [table]:[id]. Information purposes only.
899 * @param string $fieldname If the file is related to a record, this is the field name it was related to. Information purposes only.
900 * @return void
901 * @todo Define visibility
902 */
903 public function export_addFile($fI, $recordRef = '', $fieldname = '') {
904 if (@is_file($fI['ID_absFile'])) {
905 if (filesize($fI['ID_absFile']) < $this->maxFileSize) {
906 $fileInfo = stat($fI['ID_absFile']);
907 $fileRec = array();
908 $fileRec['filesize'] = $fileInfo['size'];
909 $fileRec['filename'] = basename($fI['ID_absFile']);
910 $fileRec['filemtime'] = $fileInfo['mtime'];
911 //for internal type file_reference
912 $fileRec['relFileRef'] = \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($fI['ID_absFile']);
913 if ($recordRef) {
914 $fileRec['record_ref'] = $recordRef . '/' . $fieldname;
915 }
916 if ($fI['relFileName']) {
917 $fileRec['relFileName'] = $fI['relFileName'];
918 }
919 // Setting this data in the header
920 $this->dat['header']['files'][$fI['ID']] = $fileRec;
921 // ... and for the recordlisting, why not let us know WHICH relations there was...
922 if ($recordRef && $recordRef !== '_SOFTREF_') {
923 $refParts = explode(':', $recordRef, 2);
924 if (!is_array($this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'])) {
925 $this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'] = array();
926 }
927 $this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'][] = $fI['ID'];
928 }
929 // ... and finally add the heavy stuff:
930 $fileRec['content'] = GeneralUtility::getUrl($fI['ID_absFile']);
931 $fileRec['content_md5'] = md5($fileRec['content']);
932 $this->dat['files'][$fI['ID']] = $fileRec;
933 // For soft references, do further processing:
934 if ($recordRef === '_SOFTREF_') {
935 // RTE files?
936 if ($RTEoriginal = $this->getRTEoriginalFilename(basename($fI['ID_absFile']))) {
937 $RTEoriginal_absPath = dirname($fI['ID_absFile']) . '/' . $RTEoriginal;
938 if (@is_file($RTEoriginal_absPath)) {
939 $RTEoriginal_ID = md5($RTEoriginal_absPath);
940 $fileInfo = stat($RTEoriginal_absPath);
941 $fileRec = array();
942 $fileRec['filesize'] = $fileInfo['size'];
943 $fileRec['filename'] = basename($RTEoriginal_absPath);
944 $fileRec['filemtime'] = $fileInfo['mtime'];
945 $fileRec['record_ref'] = '_RTE_COPY_ID:' . $fI['ID'];
946 $this->dat['header']['files'][$fI['ID']]['RTE_ORIG_ID'] = $RTEoriginal_ID;
947 // Setting this data in the header
948 $this->dat['header']['files'][$RTEoriginal_ID] = $fileRec;
949 // ... and finally add the heavy stuff:
950 $fileRec['content'] = GeneralUtility::getUrl($RTEoriginal_absPath);
951 $fileRec['content_md5'] = md5($fileRec['content']);
952 $this->dat['files'][$RTEoriginal_ID] = $fileRec;
953 } else {
954 $this->error('RTE original file "' . \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($RTEoriginal_absPath) . '" was not found!');
955 }
956 }
957 // Files with external media?
958 // This is only done with files grabbed by a softreference parser since it is deemed improbable that hard-referenced files should undergo this treatment.
959 $html_fI = pathinfo(basename($fI['ID_absFile']));
960 if ($this->includeExtFileResources && GeneralUtility::inList($this->extFileResourceExtensions, strtolower($html_fI['extension']))) {
961 $uniquePrefix = '###' . md5($GLOBALS['EXEC_TIME']) . '###';
962 if (strtolower($html_fI['extension']) === 'css') {
963 $prefixedMedias = explode($uniquePrefix, preg_replace('/(url[[:space:]]*\\([[:space:]]*["\']?)([^"\')]*)(["\']?[[:space:]]*\\))/i', '\\1' . $uniquePrefix . '\\2' . $uniquePrefix . '\\3', $fileRec['content']));
964 } else {
965 // html, htm:
966 $htmlParser = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\Html\\HtmlParser');
967 $prefixedMedias = explode($uniquePrefix, $htmlParser->prefixResourcePath($uniquePrefix, $fileRec['content'], array(), $uniquePrefix));
968 }
969 $htmlResourceCaptured = FALSE;
970 foreach ($prefixedMedias as $k => $v) {
971 if ($k % 2) {
972 $EXTres_absPath = GeneralUtility::resolveBackPath(dirname($fI['ID_absFile']) . '/' . $v);
973 $EXTres_absPath = GeneralUtility::getFileAbsFileName($EXTres_absPath);
974 if ($EXTres_absPath && GeneralUtility::isFirstPartOfStr($EXTres_absPath, PATH_site . $this->fileadminFolderName . '/') && @is_file($EXTres_absPath)) {
975 $htmlResourceCaptured = TRUE;
976 $EXTres_ID = md5($EXTres_absPath);
977 $this->dat['header']['files'][$fI['ID']]['EXT_RES_ID'][] = $EXTres_ID;
978 $prefixedMedias[$k] = '{EXT_RES_ID:' . $EXTres_ID . '}';
979 // Add file to memory if it is not set already:
980 if (!isset($this->dat['header']['files'][$EXTres_ID])) {
981 $fileInfo = stat($EXTres_absPath);
982 $fileRec = array();
983 $fileRec['filesize'] = $fileInfo['size'];
984 $fileRec['filename'] = basename($EXTres_absPath);
985 $fileRec['filemtime'] = $fileInfo['mtime'];
986 $fileRec['record_ref'] = '_EXT_PARENT_:' . $fI['ID'];
987 // Media relative to the HTML file.
988 $fileRec['parentRelFileName'] = $v;
989 // Setting this data in the header
990 $this->dat['header']['files'][$EXTres_ID] = $fileRec;
991 // ... and finally add the heavy stuff:
992 $fileRec['content'] = GeneralUtility::getUrl($EXTres_absPath);
993 $fileRec['content_md5'] = md5($fileRec['content']);
994 $this->dat['files'][$EXTres_ID] = $fileRec;
995 }
996 }
997 }
998 }
999 if ($htmlResourceCaptured) {
1000 $this->dat['files'][$fI['ID']]['tokenizedContent'] = implode('', $prefixedMedias);
1001 }
1002 }
1003 }
1004 } else {
1005 $this->error($fI['ID_absFile'] . ' was larger (' . GeneralUtility::formatSize(filesize($fI['ID_absFile'])) . ') than the maxFileSize (' . GeneralUtility::formatSize($this->maxFileSize) . ')! Skipping.');
1006 }
1007 } else {
1008 $this->error($fI['ID_absFile'] . ' was not a file! Skipping.');
1009 }
1010 }
1011
1012 /**
1013 * DB relations flattend to 1-dim array.
1014 * The list will be unique, no table/uid combination will appear twice.
1015 *
1016 * @param array $dbrels 2-dim Array of database relations organized by table key
1017 * @return array 1-dim array where entries are table:uid and keys are array with table/id
1018 * @todo Define visibility
1019 */
1020 public function flatDBrels($dbrels) {
1021 $list = array();
1022 foreach ($dbrels as $dat) {
1023 if ($dat['type'] == 'db') {
1024 foreach ($dat['itemArray'] as $i) {
1025 $list[$i['table'] . ':' . $i['id']] = $i;
1026 }
1027 }
1028 if ($dat['type'] == 'flex' && is_array($dat['flexFormRels']['db'])) {
1029 foreach ($dat['flexFormRels']['db'] as $subList) {
1030 foreach ($subList as $i) {
1031 $list[$i['table'] . ':' . $i['id']] = $i;
1032 }
1033 }
1034 }
1035 }
1036 return $list;
1037 }
1038
1039 /**
1040 * Soft References flattend to 1-dim array.
1041 *
1042 * @param array $dbrels 2-dim Array of database relations organized by table key
1043 * @return array 1-dim array where entries are arrays with properties of the soft link found and keys are a unique combination of field, spKey, structure path if applicable and token ID
1044 * @todo Define visibility
1045 */
1046 public function flatSoftRefs($dbrels) {
1047 $list = array();
1048 foreach ($dbrels as $field => $dat) {
1049 if (is_array($dat['softrefs']['keys'])) {
1050 foreach ($dat['softrefs']['keys'] as $spKey => $elements) {
1051 if (is_array($elements)) {
1052 foreach ($elements as $subKey => $el) {
1053 $lKey = $field . ':' . $spKey . ':' . $subKey;
1054 $list[$lKey] = array_merge(array('field' => $field, 'spKey' => $spKey), $el);
1055 // Add file_ID key to header - slightly "risky" way of doing this because if the calculation changes for the same value in $this->records[...] this will not work anymore!
1056 if ($el['subst'] && $el['subst']['relFileName']) {
1057 $list[$lKey]['file_ID'] = md5(PATH_site . $el['subst']['relFileName']);
1058 }
1059 }
1060 }
1061 }
1062 }
1063 if ($dat['type'] == 'flex' && is_array($dat['flexFormRels']['softrefs'])) {
1064 foreach ($dat['flexFormRels']['softrefs'] as $structurePath => $subSoftrefs) {
1065 if (is_array($subSoftrefs['keys'])) {
1066 foreach ($subSoftrefs['keys'] as $spKey => $elements) {
1067 foreach ($elements as $subKey => $el) {
1068 $lKey = $field . ':' . $structurePath . ':' . $spKey . ':' . $subKey;
1069 $list[$lKey] = array_merge(array('field' => $field, 'spKey' => $spKey, 'structurePath' => $structurePath), $el);
1070 // Add file_ID key to header - slightly "risky" way of doing this because if the calculation changes for the same value in $this->records[...] this will not work anymore!
1071 if ($el['subst'] && $el['subst']['relFileName']) {
1072 $list[$lKey]['file_ID'] = md5(PATH_site . $el['subst']['relFileName']);
1073 }
1074 }
1075 }
1076 }
1077 }
1078 }
1079 }
1080 return $list;
1081 }
1082
1083 /**
1084 * If include fields for a specific record type are set, the data
1085 * are filtered out with fields are not included in the fields.
1086 *
1087 * @param string $table The record type to be filtered
1088 * @param array $row The data to be filtered
1089 * @return array The filtered record row
1090 */
1091 protected function filterRecordFields($table, array $row) {
1092 if (isset($this->recordTypesIncludeFields[$table])) {
1093 $includeFields = array_unique(array_merge(
1094 $this->recordTypesIncludeFields[$table],
1095 $this->defaultRecordIncludeFields
1096 ));
1097 $newRow = array();
1098 foreach ($row as $key => $value) {
1099 if (in_array($key, $includeFields)) {
1100 $newRow[$key] = $value;
1101 }
1102 }
1103 } else {
1104 $newRow = $row;
1105 }
1106 return $newRow;
1107 }
1108
1109
1110 /**************************
1111 * File Output
1112 *************************/
1113
1114 /**
1115 * This compiles and returns the data content for an exported file
1116 *
1117 * @param string $type Type of output; "xml" gives xml, otherwise serialized array, possibly compressed.
1118 * @return string The output file stream
1119 * @todo Define visibility
1120 */
1121 public function compileMemoryToFileContent($type = '') {
1122 if ($type == 'xml') {
1123 $out = $this->createXML();
1124 } else {
1125 $compress = $this->doOutputCompress();
1126 $out = '';
1127 // adding header:
1128 $out .= $this->addFilePart(serialize($this->dat['header']), $compress);
1129 // adding records:
1130 $out .= $this->addFilePart(serialize($this->dat['records']), $compress);
1131 // adding files:
1132 $out .= $this->addFilePart(serialize($this->dat['files']), $compress);
1133 // adding files_fal:
1134 $out .= $this->addFilePart(serialize($this->dat['files_fal']), $compress);
1135 }
1136 return $out;
1137 }
1138
1139 /**
1140 * Creates XML string from input array
1141 *
1142 * @return string XML content
1143 * @todo Define visibility
1144 */
1145 public function createXML() {
1146 // Options:
1147 $options = array(
1148 'alt_options' => array(
1149 '/header' => array(
1150 'disableTypeAttrib' => TRUE,
1151 'clearStackPath' => TRUE,
1152 'parentTagMap' => array(
1153 'files' => 'file',
1154 'files_fal' => 'file',
1155 'records' => 'table',
1156 'table' => 'rec',
1157 'rec:rels' => 'relations',
1158 'relations' => 'element',
1159 'filerefs' => 'file',
1160 'pid_lookup' => 'page_contents',
1161 'header:relStaticTables' => 'static_tables',
1162 'static_tables' => 'tablename',
1163 'excludeMap' => 'item',
1164 'softrefCfg' => 'softrefExportMode',
1165 'extensionDependencies' => 'extkey',
1166 'softrefs' => 'softref_element'
1167 ),
1168 'alt_options' => array(
1169 '/pagetree' => array(
1170 'disableTypeAttrib' => TRUE,
1171 'useIndexTagForNum' => 'node',
1172 'parentTagMap' => array(
1173 'node:subrow' => 'node'
1174 )
1175 ),
1176 '/pid_lookup/page_contents' => array(
1177 'disableTypeAttrib' => TRUE,
1178 'parentTagMap' => array(
1179 'page_contents' => 'table'
1180 ),
1181 'grandParentTagMap' => array(
1182 'page_contents/table' => 'item'
1183 )
1184 )
1185 )
1186 ),
1187 '/records' => array(
1188 'disableTypeAttrib' => TRUE,
1189 'parentTagMap' => array(
1190 'records' => 'tablerow',
1191 'tablerow:data' => 'fieldlist',
1192 'tablerow:rels' => 'related',
1193 'related' => 'field',
1194 'field:itemArray' => 'relations',
1195 'field:newValueFiles' => 'filerefs',
1196 'field:flexFormRels' => 'flexform',
1197 'relations' => 'element',
1198 'filerefs' => 'file',
1199 'flexform:db' => 'db_relations',
1200 'flexform:file' => 'file_relations',
1201 'flexform:softrefs' => 'softref_relations',
1202 'softref_relations' => 'structurePath',
1203 'db_relations' => 'path',
1204 'file_relations' => 'path',
1205 'path' => 'element',
1206 'keys' => 'softref_key',
1207 'softref_key' => 'softref_element'
1208 ),
1209 'alt_options' => array(
1210 '/records/tablerow/fieldlist' => array(
1211 'useIndexTagForAssoc' => 'field'
1212 )
1213 )
1214 ),
1215 '/files' => array(
1216 'disableTypeAttrib' => TRUE,
1217 'parentTagMap' => array(
1218 'files' => 'file'
1219 )
1220 ),
1221 '/files_fal' => array(
1222 'disableTypeAttrib' => TRUE,
1223 'parentTagMap' => array(
1224 'files_fal' => 'file'
1225 )
1226 )
1227 )
1228 );
1229 // Creating XML file from $outputArray:
1230 $charset = $this->dat['header']['charset'] ?: 'utf-8';
1231 $XML = '<?xml version="1.0" encoding="' . $charset . '" standalone="yes" ?>' . LF;
1232 $XML .= GeneralUtility::array2xml($this->dat, '', 0, 'T3RecordDocument', 0, $options);
1233 return $XML;
1234 }
1235
1236 /**
1237 * Returns TRUE if the output should be compressed.
1238 *
1239 * @return boolean TRUE if compression is possible AND requested.
1240 * @todo Define visibility
1241 */
1242 public function doOutputCompress() {
1243 return $this->compress && !$this->dontCompress;
1244 }
1245
1246 /**
1247 * Returns a content part for a filename being build.
1248 *
1249 * @param array $data Data to store in part
1250 * @param boolean $compress Compress file?
1251 * @return string Content stream.
1252 * @todo Define visibility
1253 */
1254 public function addFilePart($data, $compress = FALSE) {
1255 if ($compress) {
1256 $data = gzcompress($data);
1257 }
1258 return md5($data) . ':' . ($compress ? '1' : '0') . ':' . str_pad(strlen($data), 10, '0', STR_PAD_LEFT) . ':' . $data . ':';
1259 }
1260
1261 /***********************
1262 * Import
1263 ***********************/
1264
1265 /**
1266 * Initialize all settings for the import
1267 *
1268 * @return void
1269 */
1270 protected function initializeImport() {
1271 // Set this flag to indicate that an import is being/has been done.
1272 $this->doesImport = 1;
1273 // Initialize:
1274 // These vars MUST last for the whole section not being cleared. They are used by the method setRelations() which are called at the end of the import session.
1275 $this->import_mapId = array();
1276 $this->import_newId = array();
1277 $this->import_newId_pids = array();
1278 // Temporary files stack initialized:
1279 $this->unlinkFiles = array();
1280 $this->alternativeFileName = array();
1281 $this->alternativeFilePath = array();
1282
1283 $this->initializeStorageObjects();
1284 }
1285
1286 /**
1287 * Initialize the all present storage objects
1288 *
1289 * @return void
1290 */
1291 protected function initializeStorageObjects() {
1292 /** @var $storageRepository \TYPO3\CMS\Core\Resource\StorageRepository */
1293 $storageRepository = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\Resource\\StorageRepository');
1294 $this->storageObjects = $storageRepository->findAll();
1295 }
1296
1297
1298 /**
1299 * Imports the internal data array to $pid.
1300 *
1301 * @param integer $pid Page ID in which to import the content
1302 * @return void
1303 */
1304 public function importData($pid) {
1305
1306 $this->initializeImport();
1307
1308 // Write sys_file_storages first
1309 $this->writeSysFileStorageRecords();
1310 // Write sys_file records and write the binary file data
1311 $this->writeSysFileRecords();
1312 // Write records, first pages, then the rest
1313 // Fields with "hard" relations to database, files and flexform fields are kept empty during this run
1314 $this->writeRecords_pages($pid);
1315 $this->writeRecords_records($pid);
1316 // Finally all the file and DB record references must be fixed. This is done after all records have supposedly been written to database:
1317 // $this->import_mapId will indicate two things: 1) that a record WAS written to db and 2) that it has got a new id-number.
1318 $this->setRelations();
1319 // And when all DB relations are in place, we can fix file and DB relations in flexform fields (since data structures often depends on relations to a DS record):
1320 $this->setFlexFormRelations();
1321 // Unlink temporary files:
1322 $this->unlinkTempFiles();
1323 // Finally, traverse all records and process softreferences with substitution attributes.
1324 $this->processSoftReferences();
1325 }
1326
1327 /**
1328 * Imports the sys_file_storage records from internal data array.
1329 *
1330 * @return void
1331 */
1332 protected function writeSysFileStorageRecords() {
1333 if (!isset($this->dat['header']['records']['sys_file_storage'])) {
1334 return;
1335 }
1336 $sysFileStorageUidsToBeResetToDefaultStorage = array();
1337 foreach (array_keys($this->dat['header']['records']['sys_file_storage']) as $sysFileStorageUid) {
1338 $storageRecord = $this->dat['records']['sys_file_storage:' . $sysFileStorageUid]['data'];
1339 // continue with Local, writable and online storage only
1340 if ($storageRecord['driver'] === 'Local' && $storageRecord['is_writable'] && $storageRecord['is_online']) {
1341 $useThisStorageUidInsteadOfTheOneInImport = 0;
1342 /** @var $localStorage \TYPO3\CMS\Core\Resource\ResourceStorage */
1343 foreach ($this->storageObjects as $localStorage) {
1344 // check the available storage for Local, writable and online ones
1345 if ($localStorage->getDriverType() === 'Local' && $localStorage->isWritable() && $localStorage->isOnline()) {
1346 // check if there is already a identical storage present (same pathType and basePath)
1347 $storageRecordConfiguration = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->convertFlexFormDataToConfigurationArray($storageRecord['configuration']);
1348 $localStorageRecordConfiguration = $localStorage->getConfiguration();
1349 if ($storageRecordConfiguration['pathType'] == $localStorageRecordConfiguration['pathType'] && $storageRecordConfiguration['basePath'] == $localStorageRecordConfiguration['basePath']) {
1350 // same storage is already present
1351 $useThisStorageUidInsteadOfTheOneInImport = $localStorage->getUid();
1352 break;
1353 }
1354 }
1355 }
1356 if ($useThisStorageUidInsteadOfTheOneInImport > 0) {
1357 // same storage is already present; map the to be imported one to the present one
1358 $this->import_mapId['sys_file_storage'][$sysFileStorageUid] = $useThisStorageUidInsteadOfTheOneInImport;
1359 } else {
1360 // Local, writable and online storage. Is allowed to be used to later write files in.
1361 $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);
1362 }
1363 } else {
1364 // Storage with non Local drivers could be imported but must not be used to saves files in, because you
1365 // could not be sure, that this is supported. The default storage will be used in this case.
1366 // It could happen that non writable and non online storage will be created as dupes because you could not
1367 // check the detailed configuration options at this point
1368 $this->addSingle('sys_file_storage', $sysFileStorageUid, 0);
1369 $sysFileStorageUidsToBeResetToDefaultStorage[] = $sysFileStorageUid;
1370 }
1371
1372 }
1373
1374 // Importing the added ones
1375 $tce = $this->getNewTCE();
1376 // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.
1377 $tce->reverseOrder = 1;
1378 $tce->isImporting = TRUE;
1379 $tce->start($this->import_data, array());
1380 $tce->process_datamap();
1381 $this->addToMapId($tce->substNEWwithIDs);
1382
1383 $defaultStorageUid = NULL;
1384 // get default storage
1385 $defaultStorage = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->getDefaultStorage();
1386 if ($defaultStorage !== NULL) {
1387 $defaultStorageUid = $defaultStorage->getUid();
1388 }
1389 foreach ($sysFileStorageUidsToBeResetToDefaultStorage as $sysFileStorageUidToBeResetToDefaultStorage) {
1390 $this->import_mapId['sys_file_storage'][$sysFileStorageUidToBeResetToDefaultStorage] = $defaultStorageUid;
1391 }
1392
1393 // unset the sys_file_storage records to prevent a import in writeRecords_records
1394 unset($this->dat['header']['records']['sys_file_storage']);
1395 }
1396
1397 /**
1398 * Imports the sys_file records and the binary files data from internal data array.
1399 *
1400 * @return void
1401 */
1402 protected function writeSysFileRecords() {
1403 if (!isset($this->dat['header']['records']['sys_file'])) {
1404 return;
1405 }
1406
1407 // fetch fresh storage records from database
1408 $storageRecords = $this->fetchStorageRecords();
1409
1410 $defaultStorage = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->getDefaultStorage();
1411
1412 foreach (array_keys($this->dat['header']['records']['sys_file']) as $sysFileUid) {
1413 $fileRecord = $this->dat['records']['sys_file:' . $sysFileUid]['data'];
1414
1415 // save file to disk
1416 $fileId = md5($fileRecord['storage'] . ':' . $fileRecord['identifier_hash']);
1417 $temporaryFile = $this->writeTemporaryFileFromData($fileId);
1418 if ($temporaryFile === NULL) {
1419 // error on writing the file. Error message was already added
1420 continue;
1421 }
1422
1423 $originalStorageUid = $fileRecord['storage'];
1424 $useStorageFromStorageRecords = FALSE;
1425
1426 // replace storage id, if a alternative one was registered
1427 if (isset($this->import_mapId['sys_file_storage'][$fileRecord['storage']])) {
1428 $fileRecord['storage'] = $this->import_mapId['sys_file_storage'][$fileRecord['storage']];
1429 $useStorageFromStorageRecords = TRUE;
1430 }
1431
1432 if (empty($fileRecord['storage'])) {
1433 // no storage for the file is defined, mostly because of a missing default storage.
1434 $this->error('Error: No storage for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $originalStorageUid . '"');
1435 continue;
1436 }
1437
1438 // using a storage from the local storage is only allowed, if the uid is present in the
1439 // mapping. Only in this case we could be sure, that it's a local, online and writable storage.
1440 if ($useStorageFromStorageRecords && isset($storageRecords[$fileRecord['storage']])) {
1441 /** @var $storage \TYPO3\CMS\Core\Resource\ResourceStorage */
1442 $storage = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->getStorageObject($fileRecord['storage'], $storageRecords[$fileRecord['storage']]);
1443 } elseif ($defaultStorage !== NULL) {
1444 $storage = $defaultStorage;
1445 } else {
1446 $this->error('Error: No storage available for the file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1447 continue;
1448 }
1449
1450 $newFile = NULL;
1451
1452 // check, if there is a identical file
1453 try {
1454 if ($storage->hasFile($fileRecord['identifier'])) {
1455 $file = $storage->getFile($fileRecord['identifier']);
1456 if ($file->getSha1() === $fileRecord['sha1']) {
1457 $newFile = $file;
1458 }
1459 }
1460 } catch (Exception $e) {}
1461
1462 if ($newFile === NULL) {
1463
1464 $folderName = dirname(ltrim($fileRecord['identifier'], '/'));
1465 if (!$storage->hasFolder($folderName)) {
1466 try {
1467 $storage->createFolder($folderName);
1468 } catch (Exception $e) {
1469 $this->error('Error: Folder could not be created for file "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1470 continue;
1471 }
1472 }
1473
1474 $importFolder = $storage->getFolder($folderName);
1475
1476 try {
1477 /** @var $file \TYPO3\CMS\Core\Resource\File */
1478 $newFile = $storage->addFile($temporaryFile, $importFolder, $fileRecord['name']);
1479 } catch (Exception $e) {
1480 $this->error('Error: File could not be added to the storage: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1481 continue;
1482 }
1483
1484 if ($newFile->getSha1() !== $fileRecord['sha1']) {
1485 $this->error('Error: The hash of the written file is not identical to the import data! File could be corrupted! File: "' . $fileRecord['identifier'] . '" with storage uid "' . $fileRecord['storage'] . '"');
1486 }
1487 }
1488
1489 // save the new uid in the import id map
1490 $this->import_mapId['sys_file'][$fileRecord['uid']] = $newFile->getUid();
1491 $this->fixUidLocalInSysFileReferenceRecords($fileRecord['uid'], $newFile->getUid());
1492
1493 }
1494
1495 // unset the sys_file records to prevent a import in writeRecords_records
1496 unset($this->dat['header']['records']['sys_file']);
1497 }
1498
1499 /**
1500 * Normally the importer works like the following:
1501 * Step 1: import the records with cleared field values of relation fields (see addSingle())
1502 * Step 2: update the records with the right relation ids (see setRelations())
1503 *
1504 * In step 2 the saving fields of type "relation to sys_file_reference" checks the related sys_file_reference
1505 * record (created in step 1) with the FileExtensionFilter for matching file extensions of the related file.
1506 * To make this work correct, the uid_local of sys_file_reference records has to be not empty AND has to
1507 * relate to the correct (imported) sys_file record uid!!!
1508 *
1509 * This is fixed here.
1510 *
1511 * @param int $oldFileUid
1512 * @param int $newFileUid
1513 * @return void
1514 */
1515 protected function fixUidLocalInSysFileReferenceRecords($oldFileUid, $newFileUid) {
1516 if (!isset($this->dat['header']['records']['sys_file_reference'])) {
1517 return;
1518 }
1519
1520 foreach (array_keys($this->dat['header']['records']['sys_file_reference']) as $sysFileReferenceUid) {
1521 $fileReferenceRecord = $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'];
1522 if ($fileReferenceRecord['uid_local'] == $oldFileUid) {
1523 $fileReferenceRecord['uid_local'] = $newFileUid;
1524 $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'] = $fileReferenceRecord;
1525 }
1526 }
1527 }
1528
1529 /**
1530 * Fetched fresh storage records from database because the new imported
1531 * ones are not in cached data of the StorageRepository
1532 *
1533 * @return bool|array
1534 */
1535 protected function fetchStorageRecords() {
1536 $whereClause = \TYPO3\CMS\Backend\Utility\BackendUtility::BEenableFields('sys_file_storage');
1537 $whereClause .= \TYPO3\CMS\Backend\Utility\BackendUtility::deleteClause('sys_file_storage');
1538
1539 $rows = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows(
1540 '*',
1541 'sys_file_storage',
1542 '1=1' . $whereClause,
1543 '',
1544 '',
1545 '',
1546 'uid'
1547 );
1548
1549 return $rows;
1550 }
1551
1552 /**
1553 * Writes the file from import array to temp dir and returns the filename of it.
1554 *
1555 * @param string $fileId
1556 * @param string $dataKey
1557 * @return string Absolute filename of the temporary filename of the file
1558 */
1559 protected function writeTemporaryFileFromData($fileId, $dataKey = 'files_fal') {
1560 $temporaryFilePath = NULL;
1561 if (is_array($this->dat[$dataKey][$fileId])) {
1562 $temporaryFilePathInternal = GeneralUtility::tempnam('import_temp_');
1563 GeneralUtility::writeFile($temporaryFilePathInternal, $this->dat[$dataKey][$fileId]['content']);
1564 clearstatcache();
1565 if (@is_file($temporaryFilePathInternal)) {
1566 $this->unlinkFiles[] = $temporaryFilePathInternal;
1567 if (filesize($temporaryFilePathInternal) == $this->dat[$dataKey][$fileId]['filesize']) {
1568 $temporaryFilePath = $temporaryFilePathInternal;
1569 } else {
1570 $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' had a size (' . filesize($temporaryFilePathInternal) . ') different from the original (' . $this->dat[$dataKey][$fileId]['filesize'] . ')', 1);
1571 }
1572 } else {
1573 $this->error('Error: temporary file ' . $temporaryFilePathInternal . ' was not written as it should have been!', 1);
1574 }
1575 } else {
1576 $this->error('Error: No file found for ID ' . $fileId, 1);
1577 }
1578 return $temporaryFilePath;
1579 }
1580
1581 /**
1582 * Writing pagetree/pages to database:
1583 *
1584 * @param integer $pid PID in which to import. If the operation is an update operation, the root of the page tree inside will be moved to this PID unless it is the same as the root page from the import
1585 * @return void
1586 * @see writeRecords_records()
1587 * @todo Define visibility
1588 */
1589 public function writeRecords_pages($pid) {
1590 // First, write page structure if any:
1591 if (is_array($this->dat['header']['records']['pages'])) {
1592 // $pageRecords is a copy of the pages array in the imported file. Records here are unset one by one when the addSingle function is called.
1593 $pageRecords = $this->dat['header']['records']['pages'];
1594 $this->import_data = array();
1595 // First add page tree if any
1596 if (is_array($this->dat['header']['pagetree'])) {
1597 $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);
1598 foreach ($pagesFromTree as $uid) {
1599 $thisRec = $this->dat['header']['records']['pages'][$uid];
1600 // PID: Set the main $pid, unless a NEW-id is found
1601 $setPid = isset($this->import_newId_pids[$thisRec['pid']]) ? $this->import_newId_pids[$thisRec['pid']] : $pid;
1602 $this->addSingle('pages', $uid, $setPid);
1603 unset($pageRecords[$uid]);
1604 }
1605 }
1606 // Then add all remaining pages not in tree on root level:
1607 if (count($pageRecords)) {
1608 $remainingPageUids = array_keys($pageRecords);
1609 foreach ($remainingPageUids as $pUid) {
1610 $this->addSingle('pages', $pUid, $pid);
1611 }
1612 }
1613 // Now write to database:
1614 $tce = $this->getNewTCE();
1615 $tce->isImporting = TRUE;
1616 $this->callHook('before_writeRecordsPages', array(
1617 'tce' => &$tce,
1618 'data' => &$this->import_data
1619 ));
1620 $tce->suggestedInsertUids = $this->suggestedInsertUids;
1621 $tce->start($this->import_data, array());
1622 $tce->process_datamap();
1623 $this->callHook('after_writeRecordsPages', array(
1624 'tce' => &$tce
1625 ));
1626 // post-processing: Registering new ids (end all tcemain sessions with this)
1627 $this->addToMapId($tce->substNEWwithIDs);
1628 // In case of an update, order pages from the page tree correctly:
1629 if ($this->update && is_array($this->dat['header']['pagetree'])) {
1630 $this->writeRecords_pages_order($pid);
1631 }
1632 }
1633 }
1634
1635 /**
1636 * Organize all updated pages in page tree so they are related like in the import file
1637 * Only used for updates and when $this->dat['header']['pagetree'] is an array.
1638 *
1639 * @param integer $pid Page id in which to import
1640 * @return void
1641 * @access private
1642 * @see writeRecords_pages(), writeRecords_records_order()
1643 * @todo Define visibility
1644 */
1645 public function writeRecords_pages_order($pid) {
1646 $cmd_data = array();
1647 // Get uid-pid relations and traverse them in order to map to possible new IDs
1648 $pidsFromTree = $this->flatInversePageTree_pid($this->dat['header']['pagetree']);
1649 foreach ($pidsFromTree as $origPid => $newPid) {
1650 if ($newPid >= 0 && $this->dontIgnorePid('pages', $origPid)) {
1651 // If the page had a new id (because it was created) use that instead!
1652 if (substr($this->import_newId_pids[$origPid], 0, 3) === 'NEW') {
1653 if ($this->import_mapId['pages'][$origPid]) {
1654 $mappedPid = $this->import_mapId['pages'][$origPid];
1655 $cmd_data['pages'][$mappedPid]['move'] = $newPid;
1656 }
1657 } else {
1658 $cmd_data['pages'][$origPid]['move'] = $newPid;
1659 }
1660 }
1661 }
1662 // Execute the move commands if any:
1663 if (count($cmd_data)) {
1664 $tce = $this->getNewTCE();
1665 $this->callHook('before_writeRecordsPagesOrder', array(
1666 'tce' => &$tce,
1667 'data' => &$cmd_data
1668 ));
1669 $tce->start(array(), $cmd_data);
1670 $tce->process_cmdmap();
1671 $this->callHook('after_writeRecordsPagesOrder', array(
1672 'tce' => &$tce
1673 ));
1674 }
1675 }
1676
1677 /**
1678 * Write all database records except pages (writtein in writeRecords_pages())
1679 *
1680 * @param integer $pid Page id in which to import
1681 * @return void
1682 * @see writeRecords_pages()
1683 * @todo Define visibility
1684 */
1685 public function writeRecords_records($pid) {
1686 // Write the rest of the records
1687 $this->import_data = array();
1688 if (is_array($this->dat['header']['records'])) {
1689 foreach ($this->dat['header']['records'] as $table => $recs) {
1690 if ($table != 'pages') {
1691 foreach ($recs as $uid => $thisRec) {
1692 // PID: Set the main $pid, unless a NEW-id is found
1693 $setPid = isset($this->import_mapId['pages'][$thisRec['pid']]) ? $this->import_mapId['pages'][$thisRec['pid']] : $pid;
1694 if (is_array($GLOBALS['TCA'][$table]) && $GLOBALS['TCA'][$table]['ctrl']['rootLevel']) {
1695 $setPid = 0;
1696 }
1697 // Add record:
1698 $this->addSingle($table, $uid, $setPid);
1699 }
1700 }
1701 }
1702 } else {
1703 $this->error('Error: No records defined in internal data array.');
1704 }
1705 // Now write to database:
1706 $tce = $this->getNewTCE();
1707 $this->callHook('before_writeRecordsRecords', array(
1708 'tce' => &$tce,
1709 'data' => &$this->import_data
1710 ));
1711 $tce->suggestedInsertUids = $this->suggestedInsertUids;
1712 // Because all records are being submitted in their correct order with positive pid numbers - and so we should reverse submission order internally.
1713 $tce->reverseOrder = 1;
1714 $tce->isImporting = TRUE;
1715 $tce->start($this->import_data, array());
1716 $tce->process_datamap();
1717 $this->callHook('after_writeRecordsRecords', array(
1718 'tce' => &$tce
1719 ));
1720 // post-processing: Removing files and registering new ids (end all tcemain sessions with this)
1721 $this->addToMapId($tce->substNEWwithIDs);
1722 // In case of an update, order pages from the page tree correctly:
1723 if ($this->update) {
1724 $this->writeRecords_records_order($pid);
1725 }
1726 }
1727
1728 /**
1729 * Organize all updated record to their new positions.
1730 * Only used for updates
1731 *
1732 * @param integer $mainPid Main PID into which we import.
1733 * @return void
1734 * @access private
1735 * @see writeRecords_records(), writeRecords_pages_order()
1736 * @todo Define visibility
1737 */
1738 public function writeRecords_records_order($mainPid) {
1739 $cmd_data = array();
1740 if (is_array($this->dat['header']['pagetree'])) {
1741 $pagesFromTree = $this->flatInversePageTree($this->dat['header']['pagetree']);
1742 } else {
1743 $pagesFromTree = array();
1744 }
1745 if (is_array($this->dat['header']['pid_lookup'])) {
1746 foreach ($this->dat['header']['pid_lookup'] as $pid => $recList) {
1747 $newPid = isset($this->import_mapId['pages'][$pid]) ? $this->import_mapId['pages'][$pid] : $mainPid;
1748 if (\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($newPid)) {
1749 foreach ($recList as $tableName => $uidList) {
1750 // If $mainPid===$newPid then we are on root level and we can consider to move pages as well!
1751 // (they will not be in the page tree!)
1752 if (($tableName != 'pages' || !$pagesFromTree[$pid]) && is_array($uidList)) {
1753 $uidList = array_reverse(array_keys($uidList));
1754 foreach ($uidList as $uid) {
1755 if ($this->dontIgnorePid($tableName, $uid)) {
1756 $cmd_data[$tableName][$uid]['move'] = $newPid;
1757 } else {
1758
1759 }
1760 }
1761 }
1762 }
1763 }
1764 }
1765 }
1766 // Execute the move commands if any:
1767 if (count($cmd_data)) {
1768 $tce = $this->getNewTCE();
1769 $this->callHook('before_writeRecordsRecordsOrder', array(
1770 'tce' => &$tce,
1771 'data' => &$cmd_data
1772 ));
1773 $tce->start(array(), $cmd_data);
1774 $tce->process_cmdmap();
1775 $this->callHook('after_writeRecordsRecordsOrder', array(
1776 'tce' => &$tce
1777 ));
1778 }
1779 }
1780
1781 /**
1782 * Adds a single record to the $importData array. Also copies files to tempfolder.
1783 * However all File/DB-references and flexform field contents are set to blank for now! That is done with setRelations() later
1784 *
1785 * @param string $table Table name (from import memory)
1786 * @param integer $uid Record UID (from import memory)
1787 * @param integer $pid Page id
1788 * @return void
1789 * @see writeRecords()
1790 * @todo Define visibility
1791 */
1792 public function addSingle($table, $uid, $pid) {
1793 if ($this->import_mode[$table . ':' . $uid] !== 'exclude') {
1794 $record = $this->dat['records'][$table . ':' . $uid]['data'];
1795 if (is_array($record)) {
1796 if ($this->update && $this->doesRecordExist($table, $uid) && $this->import_mode[$table . ':' . $uid] !== 'as_new') {
1797 $ID = $uid;
1798 } elseif ($table === 'sys_file_metadata' && $record['sys_language_uid'] == '0' && $this->import_mapId['sys_file'][$record['file']]) {
1799 // on adding sys_file records the belonging sys_file_metadata record was also created
1800 // if there is one the record need to be overwritten instead of creating a new one.
1801 $recordInDatabase = $GLOBALS['TYPO3_DB']->exec_SELECTgetSingleRow(
1802 'uid',
1803 'sys_file_metadata',
1804 'file = ' . $this->import_mapId['sys_file'][$record['file']] . ' AND sys_language_uid = 0 AND pid = 0'
1805 );
1806 if ($recordInDatabase !== NULL) {
1807 $this->import_mapId['sys_file_metadata'][$record['uid']] = $recordInDatabase['uid'];
1808 $ID = $recordInDatabase['uid'];
1809 } else {
1810 $ID = uniqid('NEW');
1811 }
1812
1813 } else {
1814 $ID = uniqid('NEW');
1815 }
1816 $this->import_newId[$table . ':' . $ID] = array('table' => $table, 'uid' => $uid);
1817 if ($table == 'pages') {
1818 $this->import_newId_pids[$uid] = $ID;
1819 }
1820 // Set main record data:
1821 $this->import_data[$table][$ID] = $record;
1822 $this->import_data[$table][$ID]['tx_impexp_origuid'] = $this->import_data[$table][$ID]['uid'];
1823 // Reset permission data:
1824 if ($table === 'pages') {
1825 // Have to reset the user/group IDs so pages are owned by importing user. Otherwise strange things may happen for non-admins!
1826 unset($this->import_data[$table][$ID]['perms_userid']);
1827 unset($this->import_data[$table][$ID]['perms_groupid']);
1828 }
1829 // PID and UID:
1830 unset($this->import_data[$table][$ID]['uid']);
1831 // Updates:
1832 if (\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($ID)) {
1833 unset($this->import_data[$table][$ID]['pid']);
1834 } else {
1835 // Inserts:
1836 $this->import_data[$table][$ID]['pid'] = $pid;
1837 if (($this->import_mode[$table . ':' . $uid] === 'force_uid' && $this->update || $this->force_all_UIDS) && $GLOBALS['BE_USER']->isAdmin()) {
1838 $this->import_data[$table][$ID]['uid'] = $uid;
1839 $this->suggestedInsertUids[$table . ':' . $uid] = 'DELETE';
1840 }
1841 }
1842 // Setting db/file blank:
1843 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
1844 switch ((string) $config['type']) {
1845 case 'db':
1846
1847 case 'file':
1848 // Fixed later in ->setRelations() [because we need to know ALL newly created IDs before we can map relations!]
1849 // In the meantime we set NO values for relations.
1850 //
1851 // BUT for field uid_local of table sys_file_reference the relation MUST not be cleared here,
1852 // because the value is already the uid of the right imported sys_file record.
1853 // @see fixUidLocalInSysFileReferenceRecords()
1854 // If it's empty or a uid to another record the FileExtensionFilter will throw an exception or
1855 // delete the reference record if the file extension of the related record doesn't match.
1856 if ($table !== 'sys_file_reference' && $field !== 'uid_local') {
1857 $this->import_data[$table][$ID][$field] = '';
1858 }
1859 break;
1860 case 'flex':
1861 // Fixed later in setFlexFormRelations()
1862 // In the meantime we set NO value for flexforms - this is mainly because file references inside will not be processed properly; In fact references will point to no file or existing files (in which case there will be double-references which is a big problem of course!)
1863 $this->import_data[$table][$ID][$field] = '';
1864 break;
1865 }
1866 }
1867 } elseif ($table . ':' . $uid != 'pages:0') {
1868 // On root level we don't want this error message.
1869 $this->error('Error: no record was found in data array!', 1);
1870 }
1871 }
1872 }
1873
1874 /**
1875 * Registers the substNEWids in memory.
1876 *
1877 * @param array $substNEWwithIDs From tcemain to be merged into internal mapping variable in this object
1878 * @return void
1879 * @see writeRecords()
1880 * @todo Define visibility
1881 */
1882 public function addToMapId($substNEWwithIDs) {
1883 foreach ($this->import_data as $table => $recs) {
1884 foreach ($recs as $id => $value) {
1885 $old_uid = $this->import_newId[$table . ':' . $id]['uid'];
1886 if (isset($substNEWwithIDs[$id])) {
1887 $this->import_mapId[$table][$old_uid] = $substNEWwithIDs[$id];
1888 } elseif ($this->update) {
1889 // Map same ID to same ID....
1890 $this->import_mapId[$table][$old_uid] = $id;
1891 } else {
1892 $this->error('Possible error: ' . $table . ':' . $old_uid . ' had no new id assigned to it. This indicates that the record was not added to database during import. Please check changelog!', 1);
1893 }
1894 }
1895 }
1896 }
1897
1898 /**
1899 * Returns a new $TCE object
1900 *
1901 * @return object $TCE object
1902 * @todo Define visibility
1903 */
1904 public function getNewTCE() {
1905 $tce = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\DataHandling\\DataHandler');
1906 $tce->stripslashes_values = 0;
1907 $tce->dontProcessTransformations = 1;
1908 $tce->enableLogging = $this->enableLogging;
1909 $tce->alternativeFileName = $this->alternativeFileName;
1910 $tce->alternativeFilePath = $this->alternativeFilePath;
1911 return $tce;
1912 }
1913
1914 /**
1915 * Cleaning up all the temporary files stored in typo3temp/ folder
1916 *
1917 * @return void
1918 * @todo Define visibility
1919 */
1920 public function unlinkTempFiles() {
1921 foreach ($this->unlinkFiles as $fileName) {
1922 if (GeneralUtility::isFirstPartOfStr($fileName, PATH_site . 'typo3temp/')) {
1923 GeneralUtility::unlink_tempfile($fileName);
1924 clearstatcache();
1925 if (is_file($fileName)) {
1926 $this->error('Error: ' . $fileName . ' was NOT unlinked as it should have been!', 1);
1927 }
1928 } else {
1929 $this->error('Error: ' . $fileName . ' was not in temp-path. Not removed!', 1);
1930 }
1931 }
1932 $this->unlinkFiles = array();
1933 }
1934
1935 /***************************
1936 * Import / Relations setting
1937 ***************************/
1938
1939 /**
1940 * At the end of the import process all file and DB relations should be set properly (that is relations to imported records are all re-created so imported records are correctly related again)
1941 * Relations in flexform fields are processed in setFlexFormRelations() after this function
1942 *
1943 * @return void
1944 * @see setFlexFormRelations()
1945 * @todo Define visibility
1946 */
1947 public function setRelations() {
1948 $updateData = array();
1949 // import_newId contains a register of all records that was in the import memorys "records" key
1950 foreach ($this->import_newId as $nId => $dat) {
1951 $table = $dat['table'];
1952 $uid = $dat['uid'];
1953 // original UID - NOT the new one!
1954 // If the record has been written and received a new id, then proceed:
1955 if (is_array($this->import_mapId[$table]) && isset($this->import_mapId[$table][$uid])) {
1956 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
1957 if (is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {
1958 // Traverse relation fields of each record
1959 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
1960 // uid_local of sys_file_reference needs no update because the correct reference uid was already written
1961 // @see ImportExport::fixUidLocalInSysFileReferenceRecords()
1962 if ($table === 'sys_file_reference' && $field === 'uid_local') {
1963 continue;
1964 }
1965 switch ((string) $config['type']) {
1966 case 'db':
1967 if (is_array($config['itemArray']) && count($config['itemArray'])) {
1968 $valArray = $this->setRelations_db($config['itemArray']);
1969 $updateData[$table][$thisNewUid][$field] = implode(',', $valArray);
1970 }
1971 break;
1972 case 'file':
1973 if (is_array($config['newValueFiles']) && count($config['newValueFiles'])) {
1974 $valArr = array();
1975 foreach ($config['newValueFiles'] as $fI) {
1976 $valArr[] = $this->import_addFileNameToBeCopied($fI);
1977 }
1978 $updateData[$table][$thisNewUid][$field] = implode(',', $valArr);
1979 }
1980 break;
1981 }
1982 }
1983 } else {
1984 $this->error('Error: no record was found in data array!', 1);
1985 }
1986 } else {
1987 $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')', 1);
1988 }
1989 }
1990 if (count($updateData)) {
1991 $tce = $this->getNewTCE();
1992 $tce->isImporting = TRUE;
1993 $this->callHook('before_setRelation', array(
1994 'tce' => &$tce,
1995 'data' => &$updateData
1996 ));
1997 $tce->start($updateData, array());
1998 $tce->process_datamap();
1999 $this->callHook('after_setRelations', array(
2000 'tce' => &$tce
2001 ));
2002 }
2003 }
2004
2005 /**
2006 * Maps relations for database
2007 *
2008 * @param array $itemArray Array of item sets (table/uid) from a dbAnalysis object
2009 * @return array Array with values [table]_[uid]. These values have the regular tcemain-input group/select type which means they will automatically be processed into a uid-list or MM relations.
2010 * @todo Define visibility
2011 */
2012 public function setRelations_db($itemArray) {
2013 $valArray = array();
2014 foreach ($itemArray as $relDat) {
2015 if (is_array($this->import_mapId[$relDat['table']]) && isset($this->import_mapId[$relDat['table']][$relDat['id']])) {
2016 $valArray[] = $relDat['table'] . '_' . $this->import_mapId[$relDat['table']][$relDat['id']];
2017 } elseif ($this->isTableStatic($relDat['table']) || $this->isExcluded($relDat['table'], $relDat['id']) || $relDat['id'] < 0) {
2018 // Checking for less than zero because some select types could contain negative values, eg. fe_groups (-1, -2) and sys_language (-1 = ALL languages). This must be handled on both export and import.
2019 $valArray[] = $relDat['table'] . '_' . $relDat['id'];
2020 } else {
2021 $this->error('Lost relation: ' . $relDat['table'] . ':' . $relDat['id'], 1);
2022 }
2023 }
2024 return $valArray;
2025 }
2026
2027 /**
2028 * Writes the file from import array to temp dir and returns the filename of it.
2029 *
2030 * @param array $fI File information with three keys: "filename" = filename without path, "ID_absFile" = absolute filepath to the file (including the filename), "ID" = md5 hash of "ID_absFile
2031 * @return string Absolute filename of the temporary filename of the file. In ->alternativeFileName the original name is set.
2032 * @todo Define visibility
2033 */
2034 public function import_addFileNameToBeCopied($fI) {
2035 if (is_array($this->dat['files'][$fI['ID']])) {
2036 $tmpFile = GeneralUtility::tempnam('import_temp_');
2037 GeneralUtility::writeFile($tmpFile, $this->dat['files'][$fI['ID']]['content']);
2038 clearstatcache();
2039 if (@is_file($tmpFile)) {
2040 $this->unlinkFiles[] = $tmpFile;
2041 if (filesize($tmpFile) == $this->dat['files'][$fI['ID']]['filesize']) {
2042 $this->alternativeFileName[$tmpFile] = $fI['filename'];
2043 $this->alternativeFilePath[$tmpFile] = $this->dat['files'][$fI['ID']]['relFileRef'];
2044 return $tmpFile;
2045 } else {
2046 $this->error('Error: temporary file ' . $tmpFile . ' had a size (' . filesize($tmpFile) . ') different from the original (' . $this->dat['files'][$fI['ID']]['filesize'] . ')', 1);
2047 }
2048 } else {
2049 $this->error('Error: temporary file ' . $tmpFile . ' was not written as it should have been!', 1);
2050 }
2051 } else {
2052 $this->error('Error: No file found for ID ' . $fI['ID'], 1);
2053 }
2054 }
2055
2056 /**
2057 * After all DB relations has been set in the end of the import (see setRelations()) then it is time to correct all relations inside of FlexForm fields.
2058 * The reason for doing this after is that the setting of relations may affect (quite often!) which data structure is used for the flexforms field!
2059 *
2060 * @return void
2061 * @see setRelations()
2062 * @todo Define visibility
2063 */
2064 public function setFlexFormRelations() {
2065 $updateData = array();
2066 // import_newId contains a register of all records that was in the import memorys "records" key
2067 foreach ($this->import_newId as $nId => $dat) {
2068 $table = $dat['table'];
2069 $uid = $dat['uid'];
2070 // original UID - NOT the new one!
2071 // If the record has been written and received a new id, then proceed:
2072 if (is_array($this->import_mapId[$table]) && isset($this->import_mapId[$table][$uid])) {
2073 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
2074 if (is_array($this->dat['records'][$table . ':' . $uid]['rels'])) {
2075 // Traverse relation fields of each record
2076 foreach ($this->dat['records'][$table . ':' . $uid]['rels'] as $field => $config) {
2077 switch ((string) $config['type']) {
2078 case 'flex':
2079 // Get XML content and set as default value (string, non-processed):
2080 $updateData[$table][$thisNewUid][$field] = $this->dat['records'][$table . ':' . $uid]['data'][$field];
2081 // If there has been registered relations inside the flex form field, run processing on the content:
2082 if (count($config['flexFormRels']['db']) || count($config['flexFormRels']['file'])) {
2083 $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');
2084 // This will fetch the new row for the element (which should be updated with any references to data structures etc.)
2085 $conf = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
2086 if (is_array($origRecordRow) && is_array($conf) && $conf['type'] === 'flex') {
2087 // Get current data structure and value array:
2088 $dataStructArray = BackendUtility::getFlexFormDS($conf, $origRecordRow, $table);
2089 $currentValueArray = GeneralUtility::xml2array($updateData[$table][$thisNewUid][$field]);
2090 // Do recursive processing of the XML data:
2091 $iteratorObj = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\DataHandling\\DataHandler');
2092 $iteratorObj->callBackObj = $this;
2093 $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData($currentValueArray['data'], array(), array(), $dataStructArray, array($table, $thisNewUid, $field, $config), 'remapListedDBRecords_flexFormCallBack');
2094 // The return value is set as an array which means it will be processed by tcemain for file and DB references!
2095 if (is_array($currentValueArray['data'])) {
2096 $updateData[$table][$thisNewUid][$field] = $currentValueArray;
2097 }
2098 }
2099 }
2100 break;
2101 }
2102 }
2103 } else {
2104 $this->error('Error: no record was found in data array!', 1);
2105 }
2106 } else {
2107 $this->error('Error: this records is NOT created it seems! (' . $table . ':' . $uid . ')', 1);
2108 }
2109 }
2110 if (count($updateData)) {
2111 $tce = $this->getNewTCE();
2112 $tce->isImporting = TRUE;
2113 $this->callHook('before_setFlexFormRelations', array(
2114 'tce' => &$tce,
2115 'data' => &$updateData
2116 ));
2117 $tce->start($updateData, array());
2118 $tce->process_datamap();
2119 $this->callHook('after_setFlexFormRelations', array(
2120 'tce' => &$tce
2121 ));
2122 }
2123 }
2124
2125 /**
2126 * Callback function for traversing the FlexForm structure in relation to remapping database relations
2127 *
2128 * @param array $pParams Set of parameters in numeric array: table, uid, field
2129 * @param array $dsConf TCA config for field (from Data Structure of course)
2130 * @param string $dataValue Field value (from FlexForm XML)
2131 * @param string $dataValue_ext1 Not used
2132 * @param string $dataValue_ext2 Not used
2133 * @param string $path Path of where the data structure of the element is found
2134 * @return array Array where the "value" key carries the value.
2135 * @see setFlexFormRelations()
2136 * @todo Define visibility
2137 */
2138 public function remapListedDBRecords_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path) {
2139 // Extract parameters:
2140 list($table, $uid, $field, $config) = $pParams;
2141 // In case the $path is used as index without a trailing slash we will remove that
2142 if (!is_array($config['flexFormRels']['db'][$path]) && is_array($config['flexFormRels']['db'][rtrim($path, '/')])) {
2143 $path = rtrim($path, '/');
2144 }
2145 if (is_array($config['flexFormRels']['db'][$path])) {
2146 $valArray = $this->setRelations_db($config['flexFormRels']['db'][$path]);
2147 $dataValue = implode(',', $valArray);
2148 }
2149 if (is_array($config['flexFormRels']['file'][$path])) {
2150 foreach ($config['flexFormRels']['file'][$path] as $fI) {
2151 $valArr[] = $this->import_addFileNameToBeCopied($fI);
2152 }
2153 $dataValue = implode(',', $valArr);
2154 }
2155 return array('value' => $dataValue);
2156 }
2157
2158 /**************************
2159 * Import / Soft References
2160 *************************/
2161
2162 /**
2163 * Processing of soft references
2164 *
2165 * @return void
2166 * @todo Define visibility
2167 */
2168 public function processSoftReferences() {
2169 // Initialize:
2170 $inData = array();
2171 // Traverse records:
2172 if (is_array($this->dat['header']['records'])) {
2173 foreach ($this->dat['header']['records'] as $table => $recs) {
2174 foreach ($recs as $uid => $thisRec) {
2175 // If there are soft references defined, traverse those:
2176 if (isset($GLOBALS['TCA'][$table]) && is_array($thisRec['softrefs'])) {
2177 // First traversal is to collect softref configuration and split them up based on fields. This could probably also have been done with the "records" key instead of the header.
2178 $fieldsIndex = array();
2179 foreach ($thisRec['softrefs'] as $softrefDef) {
2180 // If a substitution token is set:
2181 if ($softrefDef['field'] && is_array($softrefDef['subst']) && $softrefDef['subst']['tokenID']) {
2182 $fieldsIndex[$softrefDef['field']][$softrefDef['subst']['tokenID']] = $softrefDef;
2183 }
2184 }
2185 // The new id:
2186 $thisNewUid = BackendUtility::wsMapId($table, $this->import_mapId[$table][$uid]);
2187 // Now, if there are any fields that require substitution to be done, lets go for that:
2188 foreach ($fieldsIndex as $field => $softRefCfgs) {
2189 if (is_array($GLOBALS['TCA'][$table]['columns'][$field])) {
2190 $conf = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
2191 if ($conf['type'] === 'flex') {
2192 // This will fetch the new row for the element (which should be updated with any references to data structures etc.)
2193 $origRecordRow = BackendUtility::getRecord($table, $thisNewUid, '*');
2194 if (is_array($origRecordRow)) {
2195 // Get current data structure and value array:
2196 $dataStructArray = BackendUtility::getFlexFormDS($conf, $origRecordRow, $table);
2197 $currentValueArray = GeneralUtility::xml2array($origRecordRow[$field]);
2198 // Do recursive processing of the XML data:
2199 /** @var $iteratorObj \TYPO3\CMS\Core\DataHandling\DataHandler */
2200 $iteratorObj = GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\DataHandling\\DataHandler');
2201 $iteratorObj->callBackObj = $this;
2202 $currentValueArray['data'] = $iteratorObj->checkValue_flex_procInData($currentValueArray['data'], array(), array(), $dataStructArray, array($table, $uid, $field, $softRefCfgs), 'processSoftReferences_flexFormCallBack');
2203 // The return value is set as an array which means it will be processed by tcemain for file and DB references!
2204 if (is_array($currentValueArray['data'])) {
2205 $inData[$table][$thisNewUid][$field] = $currentValueArray;
2206 }
2207 }
2208 } else {
2209 // Get tokenizedContent string and proceed only if that is not blank:
2210 $tokenizedContent = $this->dat['records'][$table . ':' . $uid]['rels'][$field]['softrefs']['tokenizedContent'];
2211 if (strlen($tokenizedContent) && is_array($softRefCfgs)) {
2212 $inData[$table][$thisNewUid][$field] = $this->processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid);
2213 }
2214 }
2215 }
2216 }
2217 }
2218 }
2219 }
2220 }
2221 // Now write to database:
2222 $tce = $this->getNewTCE();
2223 $tce->isImporting = TRUE;
2224 $this->callHook('before_processSoftReferences', array(
2225 'tce' => &$tce,
2226 'data' => &$inData
2227 ));
2228 $tce->enableLogging = TRUE;
2229 $tce->start($inData, array());
2230 $tce->process_datamap();
2231 $this->callHook('after_processSoftReferences', array(
2232 'tce' => &$tce
2233 ));
2234 }
2235
2236 /**
2237 * Callback function for traversing the FlexForm structure in relation to remapping softreference relations
2238 *
2239 * @param array $pParams Set of parameters in numeric array: table, uid, field
2240 * @param array $dsConf TCA config for field (from Data Structure of course)
2241 * @param string $dataValue Field value (from FlexForm XML)
2242 * @param string $dataValue_ext1 Not used
2243 * @param string $dataValue_ext2 Not used
2244 * @param string $path Path of where the data structure where the element is found
2245 * @return array Array where the "value" key carries the value.
2246 * @see setFlexFormRelations()
2247 * @todo Define visibility
2248 */
2249 public function processSoftReferences_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $path) {
2250 // Extract parameters:
2251 list($table, $origUid, $field, $softRefCfgs) = $pParams;
2252 if (is_array($softRefCfgs)) {
2253 // First, find all soft reference configurations for this structure path (they are listed flat in the header):
2254 $thisSoftRefCfgList = array();
2255 foreach ($softRefCfgs as $sK => $sV) {
2256 if ($sV['structurePath'] === $path) {
2257 $thisSoftRefCfgList[$sK] = $sV;
2258 }
2259 }
2260 // If any was found, do processing:
2261 if (count($thisSoftRefCfgList)) {
2262 // Get tokenizedContent string and proceed only if that is not blank:
2263 $tokenizedContent = $this->dat['records'][$table . ':' . $origUid]['rels'][$field]['flexFormRels']['softrefs'][$path]['tokenizedContent'];
2264 if (strlen($tokenizedContent)) {
2265 $dataValue = $this->processSoftReferences_substTokens($tokenizedContent, $thisSoftRefCfgList, $table, $origUid);
2266 }
2267 }
2268 }
2269 // Return
2270 return array('value' => $dataValue);
2271 }
2272
2273 /**
2274 * Substition of softreference tokens
2275 *
2276 * @param string $tokenizedContent Content of field with soft reference tokens in.
2277 * @param array $softRefCfgs Soft reference configurations
2278 * @param string $table Table for which the processing occurs
2279 * @param string $uid UID of record from table
2280 * @return string The input content with tokens substituted according to entries in softRefCfgs
2281 * @todo Define visibility
2282 */
2283 public function processSoftReferences_substTokens($tokenizedContent, $softRefCfgs, $table, $uid) {
2284 // traverse each softref type for this field:
2285 foreach ($softRefCfgs as $cfg) {
2286 // Get token ID:
2287 $tokenID = $cfg['subst']['tokenID'];
2288 // Default is current token value:
2289 $insertValue = $cfg['subst']['tokenValue'];
2290 // Based on mode:
2291 switch ((string) $this->softrefCfg[$tokenID]['mode']) {
2292 case 'exclude':
2293 // Exclude is a simple passthrough of the value
2294 break;
2295 case 'editable':
2296 // Editable always picks up the value from this input array:
2297 $insertValue = $this->softrefInputValues[$tokenID];
2298 break;
2299 default:
2300 // Mapping IDs/creating files: Based on type, look up new value:
2301 switch ((string) $cfg['subst']['type']) {
2302 case 'file':
2303 // Create / Overwrite file:
2304 $insertValue = $this->processSoftReferences_saveFile($cfg['subst']['relFileName'], $cfg, $table, $uid);
2305 break;
2306 case 'db':
2307 default:
2308 // Trying to map database element if found in the mapID array:
2309 list($tempTable, $tempUid) = explode(':', $cfg['subst']['recordRef']);
2310 if (isset($this->import_mapId[$tempTable][$tempUid])) {
2311 $insertValue = BackendUtility::wsMapId($tempTable, $this->import_mapId[$tempTable][$tempUid]);
2312 // Look if reference is to a page and the original token value was NOT an integer - then we assume is was an alias and try to look up the new one!
2313 if ($tempTable === 'pages' && !\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($cfg['subst']['tokenValue'])) {
2314 $recWithUniqueValue = BackendUtility::getRecord($tempTable, $insertValue, 'alias');
2315 if ($recWithUniqueValue['alias']) {
2316 $insertValue = $recWithUniqueValue['alias'];
2317 }
2318 } elseif (strpos($cfg['subst']['tokenValue'], ':') !== FALSE) {
2319 list($tokenKey, $tokenId) = explode(':', $cfg['subst']['tokenValue']);
2320 $insertValue = $tokenKey . ':' . $insertValue;
2321 }
2322 }
2323 }
2324 }
2325 // Finally, swap the soft reference token in tokenized content with the insert value:
2326 $tokenizedContent = str_replace('{softref:' . $tokenID . '}', $insertValue, $tokenizedContent);
2327 }
2328 return $tokenizedContent;
2329 }
2330
2331 /**
2332 * Process a soft reference file
2333 *
2334 * @param string $relFileName Old Relative filename
2335 * @param array $cfg soft reference configuration array
2336 * @param string $table Table for which the processing occurs
2337 * @param string $uid UID of record from table
2338 * @return string New relative filename (value to insert instead of the softref token)
2339 * @todo Define visibility
2340 */
2341 public function processSoftReferences_saveFile($relFileName, $cfg, $table, $uid) {
2342 if ($fileHeaderInfo = $this->dat['header']['files'][$cfg['file_ID']]) {
2343 // Initialize; Get directory prefix for file and find possible RTE filename
2344 $dirPrefix = dirname($relFileName) . '/';
2345 $rteOrigName = $this->getRTEoriginalFilename(basename($relFileName));
2346 // If filename looks like an RTE file, and the directory is in "uploads/", then process as a RTE file!
2347 if ($rteOrigName && GeneralUtility::isFirstPartOfStr($dirPrefix, 'uploads/')) {
2348 // RTE:
2349 // First, find unique RTE file name:
2350 if (@is_dir((PATH_site . $dirPrefix))) {
2351 // From the "original" RTE filename, produce a new "original" destination filename which is unused. Even if updated, the image should be unique. Currently the problem with this is that it leaves a lot of unused RTE images...
2352 $fileProcObj = $this->getFileProcObj();
2353 $origDestName = $fileProcObj->getUniqueName($rteOrigName, PATH_site . $dirPrefix);
2354 // Create copy file name:
2355 $pI = pathinfo($relFileName);
2356 $copyDestName = dirname($origDestName) . '/RTEmagicC_' . substr(basename($origDestName), 10) . '.' . $pI['extension'];
2357 if (!@is_file($copyDestName) && !@is_file($origDestName) && $origDestName === GeneralUtility::getFileAbsFileName($origDestName) && $copyDestName === GeneralUtility::getFileAbsFileName($copyDestName)) {
2358 if ($this->dat['header']['files'][$fileHeaderInfo['RTE_ORIG_ID']]) {
2359 // Write the copy and original RTE file to the respective filenames:
2360 $this->writeFileVerify($copyDestName, $cfg['file_ID'], TRUE);
2361 $this->writeFileVerify($origDestName, $fileHeaderInfo['RTE_ORIG_ID'], TRUE);
2362 // Return the relative path of the copy file name:
2363 return \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($copyDestName);
2364 } else {
2365 $this->error('ERROR: Could not find original file ID');
2366 }
2367 } else {
2368 $this->error('ERROR: The destination filenames "' . $copyDestName . '" and "' . $origDestName . '" either existed or have non-valid names');
2369 }
2370 } else {
2371 $this->error('ERROR: "' . PATH_site . $dirPrefix . '" was not a directory, so could not process file "' . $relFileName . '"');
2372 }
2373 } elseif (GeneralUtility::isFirstPartOfStr($dirPrefix, $this->fileadminFolderName . '/')) {
2374 // File in fileadmin/ folder:
2375 // Create file (and possible resources)
2376 $newFileName = $this->processSoftReferences_saveFile_createRelFile($dirPrefix, basename($relFileName), $cfg['file_ID'], $table, $uid);
2377 if (strlen($newFileName)) {
2378 $relFileName = $newFileName;
2379 } else {
2380 $this->error('ERROR: No new file created for "' . $relFileName . '"');
2381 }
2382 } else {
2383 $this->error('ERROR: Sorry, cannot operate on non-RTE files which are outside the fileadmin folder.');
2384 }
2385 } else {
2386 $this->error('ERROR: Could not find file ID in header.');
2387 }
2388 // Return (new) filename relative to PATH_site:
2389 return $relFileName;
2390 }
2391
2392 /**
2393 * Create file in directory and return the new (unique) filename
2394 *
2395 * @param string $origDirPrefix Directory prefix, relative, with trailing slash
2396 * @param string $fileName Filename (without path)
2397 * @param string $fileID File ID from import memory
2398 * @param string $table Table for which the processing occurs
2399 * @param string $uid UID of record from table
2400 * @return string New relative filename, if any
2401 * @todo Define visibility
2402 */
2403 public function processSoftReferences_saveFile_createRelFile($origDirPrefix, $fileName, $fileID, $table, $uid) {
2404 // If the fileID map contains an entry for this fileID then just return the relative filename of that entry; we don't want to write another unique filename for this one!
2405 if ($this->fileIDMap[$fileID]) {
2406 return \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($this->fileIDMap[$fileID]);
2407 }
2408 // Verify FileMount access to dir-prefix. Returns the best alternative relative path if any
2409 $dirPrefix = $this->verifyFolderAccess($origDirPrefix);
2410 if ($dirPrefix && (!$this->update || $origDirPrefix === $dirPrefix) && $this->checkOrCreateDir($dirPrefix)) {
2411 $fileHeaderInfo = $this->dat['header']['files'][$fileID];
2412 $updMode = $this->update && $this->import_mapId[$table][$uid] === $uid && $this->import_mode[$table . ':' . $uid] !== 'as_new';
2413 // Create new name for file:
2414 // Must have same ID in map array (just for security, is not really needed) and NOT be set "as_new".
2415 if ($updMode) {
2416 $newName = PATH_site . $dirPrefix . $fileName;
2417 } else {
2418 // Create unique filename:
2419 $fileProcObj = $this->getFileProcObj();
2420 $newName = $fileProcObj->getUniqueName($fileName, PATH_site . $dirPrefix);
2421 }
2422 // Write main file:
2423 if ($this->writeFileVerify($newName, $fileID)) {
2424 // If the resource was an HTML/CSS file with resources attached, we will write those as well!
2425 if (is_array($fileHeaderInfo['EXT_RES_ID'])) {
2426 $tokenizedContent = $this->dat['files'][$fileID]['tokenizedContent'];
2427 $tokenSubstituted = FALSE;
2428 $fileProcObj = $this->getFileProcObj();
2429 if ($updMode) {
2430 foreach ($fileHeaderInfo['EXT_RES_ID'] as $res_fileID) {
2431 if ($this->dat['files'][$res_fileID]['filename']) {
2432 // Resolve original filename:
2433 $relResourceFileName = $this->dat['files'][$res_fileID]['parentRelFileName'];
2434 $absResourceFileName = GeneralUtility::resolveBackPath(PATH_site . $origDirPrefix . $relResourceFileName);
2435 $absResourceFileName = GeneralUtility::getFileAbsFileName($absResourceFileName);
2436 if ($absResourceFileName && GeneralUtility::isFirstPartOfStr($absResourceFileName, PATH_site . $this->fileadminFolderName . '/')) {
2437 $destDir = \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix(dirname($absResourceFileName) . '/');
2438 if ($this->verifyFolderAccess($destDir, TRUE) && $this->checkOrCreateDir($destDir)) {
2439 $this->writeFileVerify($absResourceFileName, $res_fileID);
2440 } else {
2441 $this->error('ERROR: Could not create file in directory "' . $destDir . '"');
2442 }
2443 } else {
2444 $this->error('ERROR: Could not resolve path for "' . $relResourceFileName . '"');
2445 }
2446 $tokenizedContent = str_replace('{EXT_RES_ID:' . $res_fileID . '}', $relResourceFileName, $tokenizedContent);
2447 $tokenSubstituted = TRUE;
2448 }
2449 }
2450 } else {
2451 // Create the resouces directory name (filename without extension, suffixed "_FILES")
2452 $resourceDir = dirname($newName) . '/' . preg_replace('/\\.[^.]*$/', '', basename($newName)) . '_FILES';
2453 if (GeneralUtility::mkdir($resourceDir)) {
2454 foreach ($fileHeaderInfo['EXT_RES_ID'] as $res_fileID) {
2455 if ($this->dat['files'][$res_fileID]['filename']) {
2456 $absResourceFileName = $fileProcObj->getUniqueName($this->dat['files'][$res_fileID]['filename'], $resourceDir);
2457 $relResourceFileName = substr($absResourceFileName, strlen(dirname($resourceDir)) + 1);
2458 $this->writeFileVerify($absResourceFileName, $res_fileID);
2459 $tokenizedContent = str_replace('{EXT_RES_ID:' . $res_fileID . '}', $relResourceFileName, $tokenizedContent);
2460 $tokenSubstituted = TRUE;
2461 }
2462 }
2463 }
2464 }
2465 // If substitutions has been made, write the content to the file again:
2466 if ($tokenSubstituted) {
2467 GeneralUtility::writeFile($newName, $tokenizedContent);
2468 }
2469 }
2470 return \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($newName);
2471 }
2472 }
2473 }
2474
2475 /**
2476 * Writes a file from the import memory having $fileID to file name $fileName which must be an absolute path inside PATH_site
2477 *
2478 * @param string $fileName Absolute filename inside PATH_site to write to
2479 * @param string $fileID File ID from import memory
2480 * @param boolean $bypassMountCheck Bypasses the checking against filemounts - only for RTE files!
2481 * @return boolean Returns TRUE if it went well. Notice that the content of the file is read again, and md5 from import memory is validated.
2482 * @todo Define visibility
2483 */
2484 public function writeFileVerify($fileName, $fileID, $bypassMountCheck = FALSE) {
2485 $fileProcObj = $this->getFileProcObj();
2486 if ($fileProcObj->actionPerms['addFile']) {
2487 // Just for security, check again. Should actually not be necessary.
2488 if ($fileProcObj->checkPathAgainstMounts($fileName) || $bypassMountCheck) {
2489 $fI = GeneralUtility::split_fileref($fileName);
2490 if ($fileProcObj->checkIfAllowed($fI['fileext'], $fI['path'], $fI['file']) || $this->allowPHPScripts && $GLOBALS['BE_USER']->isAdmin()) {
2491 if (GeneralUtility::getFileAbsFileName($fileName)) {
2492 if ($this->dat['files'][$fileID]) {
2493 GeneralUtility::writeFile($fileName, $this->dat['files'][$fileID]['content']);
2494 $this->fileIDMap[$fileID] = $fileName;
2495 if (md5(GeneralUtility::getUrl($fileName)) == $this->dat['files'][$fileID]['content_md5']) {
2496 return TRUE;
2497 } else {
2498 $this->error('ERROR: File content "' . $fileName . '" was corrupted');
2499 }
2500 } else {
2501 $this->error('ERROR: File ID "' . $fileID . '" could not be found');
2502 }
2503 } else {
2504 $this->error('ERROR: Filename "' . $fileName . '" was not a valid relative file path!');
2505 }
2506 } else {
2507 $this->error('ERROR: Filename "' . $fileName . '" failed against extension check or deny-pattern!');
2508 }
2509 } else {
2510 $this->error('ERROR: Filename "' . $fileName . '" was not allowed in destination path!');
2511 }
2512 } else {
2513 $this->error('ERROR: You did not have sufficient permissions to write the file "' . $fileName . '"');
2514 }
2515 }
2516
2517 /**
2518 * Returns TRUE if directory exists and if it doesn't it will create directory and return TRUE if that succeeded.
2519 *
2520 * @param string $dirPrefix Directory to create. Having a trailing slash. Must be in fileadmin/. Relative to PATH_site
2521 * @return boolean TRUE, if directory exists (was created)
2522 * @todo Define visibility
2523 */
2524 public function checkOrCreateDir($dirPrefix) {
2525 // Split dir path and remove first directory (which should be "fileadmin")
2526 $filePathParts = explode('/', $dirPrefix);
2527 $firstDir = array_shift($filePathParts);
2528 if ($firstDir === $this->fileadminFolderName && GeneralUtility::getFileAbsFileName($dirPrefix)) {
2529 $pathAcc = '';
2530 foreach ($filePathParts as $dirname) {
2531 $pathAcc .= '/' . $dirname;
2532 if (strlen($dirname)) {
2533 if (!@is_dir((PATH_site . $this->fileadminFolderName . $pathAcc))) {
2534 if (!GeneralUtility::mkdir((PATH_site . $this->fileadminFolderName . $pathAcc))) {
2535 $this->error('ERROR: Directory could not be created....B');
2536 return FALSE;
2537 }
2538 }
2539 } elseif ($dirPrefix === $this->fileadminFolderName . $pathAcc) {
2540 return TRUE;
2541 } else {
2542 $this->error('ERROR: Directory could not be created....A');
2543 }
2544 }
2545 }
2546 }
2547
2548 /**
2549 * Verifies that the input path (relative to PATH_site) is found in the backend users filemounts.
2550 * If it doesn't it will try to find another relative filemount for the user and return an alternative path prefix for the file.
2551 *
2552 * @param string $dirPrefix Path relative to PATH_site
2553 * @param boolean $noAlternative If set, Do not look for alternative path! Just return FALSE
2554 * @return string If a path is available that will be returned, otherwise FALSE.
2555 * @todo Define visibility
2556 */
2557 public function verifyFolderAccess($dirPrefix, $noAlternative = FALSE) {
2558 $fileProcObj = $this->getFileProcObj();
2559 // Check, if dirPrefix is inside a valid Filemount for user:
2560 $result = $fileProcObj->checkPathAgainstMounts(PATH_site . $dirPrefix);
2561 // If not, try to find another relative filemount and use that instead:
2562 if (!$result) {
2563 if ($noAlternative) {
2564 return FALSE;
2565 }
2566 // Find first web folder:
2567 $result = $fileProcObj->findFirstWebFolder();
2568 // If that succeeded, return the path to it:
2569 if ($result) {
2570 // Remove the "fileadmin/" prefix of input path - and append the rest to the return value:
2571 if (GeneralUtility::isFirstPartOfStr($dirPrefix, $this->fileadminFolderName . '/')) {
2572 $dirPrefix = substr($dirPrefix, strlen($this->fileadminFolderName . '/'));
2573 }
2574 return \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($fileProcObj->mounts[$result]['path'] . $dirPrefix);
2575 }
2576 } else {
2577 return $dirPrefix;
2578 }
2579 }
2580
2581 /**************************
2582 * File Input
2583 *************************/
2584
2585 /**
2586 * Loads the header section/all of the $filename into memory
2587 *
2588 * @param string $filename Filename, absolute
2589 * @param boolean $all If set, all information is loaded (header, records and files). Otherwise the default is to read only the header information
2590 * @return boolean TRUE if the operation went well
2591 * @todo Define visibility
2592 */
2593 public function loadFile($filename, $all = 0) {
2594 if (@is_file($filename)) {
2595 $fI = pathinfo($filename);
2596 if (strtolower($fI['extension']) == 'xml') {
2597 // XML:
2598 $xmlContent = GeneralUtility::getUrl($filename);
2599 if (strlen($xmlContent)) {
2600 $this->dat = GeneralUtility::xml2array($xmlContent, '', TRUE);
2601 if (is_array($this->dat)) {
2602 if ($this->dat['_DOCUMENT_TAG'] === 'T3RecordDocument' && is_array($this->dat['header']) && is_array($this->dat['records'])) {
2603 $this->loadInit();
2604 return TRUE;
2605 } else {
2606 $this->error('XML file did not contain proper XML for TYPO3 Import');
2607 }
2608 } else {
2609 $this->error('XML could not be parsed: ' . $this->dat);
2610 }
2611 } else {
2612 $this->error('Error opening file: ' . $filename);
2613 }
2614 } else {
2615 // T3D
2616 if ($fd = fopen($filename, 'rb')) {
2617 $this->dat['header'] = $this->getNextFilePart($fd, 1, 'header');
2618 if ($all) {
2619 $this->dat['records'] = $this->getNextFilePart($fd, 1, 'records');
2620 $this->dat['files'] = $this->getNextFilePart($fd, 1, 'files');
2621 $this->dat['files_fal'] = $this->getNextFilePart($fd, 1, 'files_fal');
2622 }
2623 $this->loadInit();
2624 return TRUE;
2625 } else {
2626 $this->error('Error opening file: ' . $filename);
2627 }
2628 fclose($fd);
2629 }
2630 } else {
2631 $this->error('Filename not found: ' . $filename);
2632 }
2633 return FALSE;
2634 }
2635
2636 /**
2637 * Returns the next content part form the fileresource (t3d), $fd
2638 *
2639 * @param pointer $fd File pointer
2640 * @param boolean $unserialize If set, the returned content is unserialized into an array, otherwise you get the raw string
2641 * @param string $name For error messages this indicates the section of the problem.
2642 * @return string Data string
2643 * @access private
2644 * @see loadFile()
2645 * @todo Define visibility
2646 */
2647 public function getNextFilePart($fd, $unserialize = 0, $name = '') {
2648 $initStrLen = 32 + 1 + 1 + 1 + 10 + 1;
2649 // Getting header data
2650 $initStr = fread($fd, $initStrLen);
2651 $initStrDat = explode(':', $initStr);
2652 if (strstr($initStrDat[0], 'Warning') == FALSE) {
2653 if ((string)$initStrDat[3] === '') {
2654 $datString = fread($fd, (int)$initStrDat[2]);
2655 fread($fd, 1);
2656 if (md5($datString) === $initStrDat[0]) {
2657 if ($initStrDat[1]) {
2658 if ($this->compress) {
2659 $datString = gzuncompress($datString);
2660 } else {
2661 $this->error('Content read error: This file requires decompression, but this server does not offer gzcompress()/gzuncompress() functions.', 1);
2662 }
2663 }
2664 return $unserialize ? unserialize($datString) : $datString;
2665 } else {
2666 $this->error('MD5 check failed (' . $name . ')');
2667 }
2668 } else {
2669 $this->error('File read error: InitString had a wrong length. (' . $name . ')');
2670 }
2671 } else {
2672 $this->error('File read error: Warning message in file. (' . $initStr . fgets($fd) . ')');
2673 }
2674 }
2675
2676 /**
2677 * Loads T3D file content into the $this->dat array
2678 * (This function can be used to test the output strings from ->compileMemoryToFileContent())
2679 *
2680 * @param string $filecontent File content
2681 * @return void
2682 * @todo Define visibility
2683 */
2684 public function loadContent($filecontent) {
2685 $pointer = 0;
2686 $this->dat['header'] = $this->getNextContentPart($filecontent, $pointer, 1, 'header');
2687 $this->dat['records'] = $this->getNextContentPart($filecontent, $pointer, 1, 'records');
2688 $this->dat['files'] = $this->getNextContentPart($filecontent, $pointer, 1, 'files');
2689 $this->loadInit();
2690 }
2691
2692 /**
2693 * Returns the next content part from the $filecontent
2694 *
2695 * @param string $filecontent File content string
2696 * @param integer $pointer File pointer (where to read from)
2697 * @param boolean $unserialize If set, the returned content is unserialized into an array, otherwise you get the raw string
2698 * @param string $name For error messages this indicates the section of the problem.
2699 * @return string Data string
2700 * @todo Define visibility
2701 */
2702 public function getNextContentPart($filecontent, &$pointer, $unserialize = 0, $name = '') {
2703 $initStrLen = 32 + 1 + 1 + 1 + 10 + 1;
2704 // getting header data
2705 $initStr = substr($filecontent, $pointer, $initStrLen);
2706 $pointer += $initStrLen;
2707 $initStrDat = explode(':', $initStr);
2708 if ((string)$initStrDat[3] === '') {
2709 $datString = substr($filecontent, $pointer, (int)$initStrDat[2]);
2710 $pointer += (int)$initStrDat[2] + 1;
2711 if (md5($datString) === $initStrDat[0]) {
2712 if ($initStrDat[1]) {
2713 if ($this->compress) {
2714 $datString = gzuncompress($datString);
2715 } else {
2716 $this->error('Content read error: This file requires decompression, but this server does not offer gzcompress()/gzuncompress() functions.', 1);
2717 }
2718 }
2719 return $unserialize ? unserialize($datString) : $datString;
2720 } else {
2721 $this->error('MD5 check failed (' . $name . ')');
2722 }
2723 } else {
2724 $this->error('Content read error: InitString had a wrong length. (' . $name . ')');
2725 }
2726 }
2727
2728 /**
2729 * Setting up the object based on the recently loaded ->dat array
2730 *
2731 * @return void
2732 * @todo Define visibility
2733 */
2734 public function loadInit() {
2735 $this->relStaticTables = (array) $this->dat['header']['relStaticTables'];
2736 $this->excludeMap = (array) $this->dat['header']['excludeMap'];
2737 $this->softrefCfg = (array) $this->dat['header']['softrefCfg'];
2738 $this->extensionDependencies = (array) $this->dat['header']['extensionDependencies'];
2739 $this->fixCharsets();
2740 }
2741
2742 /**
2743 * Fix charset of import memory if different from system charset
2744 *
2745 * @return void
2746 * @see loadInit()
2747 * @todo Define visibility
2748 */
2749 public function fixCharsets() {
2750 global $LANG;
2751 $importCharset = $this->dat['header']['charset'];
2752 if ($importCharset) {
2753 if ($importCharset !== $LANG->charSet) {
2754 $this->error('CHARSET: Converting charset of input file (' . $importCharset . ') to the system charset (' . $LANG->charSet . ')');
2755 // Convert meta data:
2756 if (is_array($this->dat['header']['meta'])) {
2757 $LANG->csConvObj->convArray($this->dat['header']['meta'], $importCharset, $LANG->charSet);
2758 }
2759 // Convert record headers:
2760 if (is_array($this->dat['header']['records'])) {
2761 $LANG->csConvObj->convArray($this->dat['header']['records'], $importCharset, $LANG->charSet);
2762 }
2763 // Convert records themselves:
2764 if (is_array($this->dat['records'])) {
2765 $LANG->csConvObj->convArray($this->dat['records'], $importCharset, $LANG->charSet);
2766 }
2767 }
2768 } else {
2769 $this->error('CHARSET: No charset found in import file!');
2770 }
2771 }
2772
2773 /********************************************************
2774 * Visual rendering of import/export memory, $this->dat
2775 ********************************************************/
2776
2777 /**
2778 * Displays an overview of the header-content.
2779 *
2780 * @return string HTML content
2781 * @todo Define visibility
2782 */
2783 public function displayContentOverview() {