56de201903f984bca9e7266784158f80b4882390
[Packages/TYPO3.CMS.git] / typo3 / sysext / impexp / Classes / Export.php
1 <?php
2 namespace TYPO3\CMS\Impexp;
3
4 /*
5 * This file is part of the TYPO3 CMS project.
6 *
7 * It is free software; you can redistribute it and/or modify it under
8 * the terms of the GNU General Public License, either version 2
9 * of the License, or any later version.
10 *
11 * For the full copyright and license information, please read the
12 * LICENSE.txt file that was distributed with this source code.
13 *
14 * The TYPO3 project - inspiring people to share!
15 */
16
17 use TYPO3\CMS\Backend\Utility\BackendUtility;
18 use TYPO3\CMS\Core\Database\ReferenceIndex;
19 use TYPO3\CMS\Core\Exception;
20 use TYPO3\CMS\Core\Html\HtmlParser;
21 use TYPO3\CMS\Core\Resource\File;
22 use TYPO3\CMS\Core\Resource\ResourceFactory;
23 use TYPO3\CMS\Core\Utility\GeneralUtility;
24 use TYPO3\CMS\Core\Utility\PathUtility;
25
26 /**
27 * EXAMPLE for using the impexp-class for exporting stuff:
28 *
29 * Create and initialize:
30 * $this->export = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Impexp\ImportExport::class);
31 * $this->export->init();
32 * Set which tables relations we will allow:
33 * $this->export->relOnlyTables[]="tt_news"; // exclusively includes. See comment in the class
34 *
35 * Adding records:
36 * $this->export->export_addRecord("pages", $this->pageinfo);
37 * $this->export->export_addRecord("pages", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("pages", 38));
38 * $this->export->export_addRecord("pages", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("pages", 39));
39 * $this->export->export_addRecord("tt_content", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("tt_content", 12));
40 * $this->export->export_addRecord("tt_content", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("tt_content", 74));
41 * $this->export->export_addRecord("sys_template", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("sys_template", 20));
42 *
43 * Adding all the relations (recursively in 5 levels so relations has THEIR relations registered as well)
44 * for($a=0;$a<5;$a++) {
45 * $addR = $this->export->export_addDBRelations($a);
46 * if (empty($addR)) break;
47 * }
48 *
49 * Finally load all the files.
50 * $this->export->export_addFilesFromRelations(); // MUST be after the DBrelations are set so that file from ALL added records are included!
51 *
52 * Write export
53 * $out = $this->export->compileMemoryToFileContent();
54 */
55
56 /**
57 * T3D file Export library (TYPO3 Record Document)
58 */
59 class Export extends ImportExport
60 {
61 /**
62 * @deprecated since TYPO3 v9, will be removed in TYPO3 v10. In v10, just remove property, it is not used any longer.
63 * @var int
64 */
65 public $maxFileSize = 1000000;
66
67 /**
68 * @deprecated since TYPO3 v9, will be removed in TYPO3 v10. In v10, just remove property, it is not used any longer.
69 * @var int
70 */
71 public $maxRecordSize = 1000000;
72
73 /**
74 * @deprecated since TYPO3 v9, will be removed in TYPO3 v10. In v10, just remove property, it is not used any longer.
75 * @var int
76 */
77 public $maxExportSize = 10000000;
78
79 /**
80 * Set by user: If set, compression in t3d files is disabled
81 *
82 * @var bool
83 */
84 public $dontCompress = false;
85
86 /**
87 * If set, HTML file resources are included.
88 *
89 * @var bool
90 */
91 public $includeExtFileResources = false;
92
93 /**
94 * Files with external media (HTML/css style references inside)
95 *
96 * @var string
97 */
98 public $extFileResourceExtensions = 'html,htm,css';
99
100 /**
101 * Keys are [recordname], values are an array of fields to be included
102 * in the export
103 *
104 * @var array
105 */
106 protected $recordTypesIncludeFields = [];
107
108 /**
109 * Default array of fields to be included in the export
110 *
111 * @var array
112 */
113 protected $defaultRecordIncludeFields = ['uid', 'pid'];
114
115 /**
116 * @var bool
117 */
118 protected $saveFilesOutsideExportFile = false;
119
120 /**
121 * @var string|null
122 */
123 protected $temporaryFilesPathForExport = null;
124
125 /**************************
126 * Initialize
127 *************************/
128
129 /**
130 * Init the object
131 *
132 * @param bool $dontCompress If set, compression of t3d files is disabled
133 */
134 public function init($dontCompress = false)
135 {
136 parent::init();
137 $this->dontCompress = $dontCompress;
138 $this->mode = 'export';
139 }
140
141 /**************************
142 * Export / Init + Meta Data
143 *************************/
144
145 /**
146 * Set header basics
147 */
148 public function setHeaderBasics()
149 {
150 // Initializing:
151 if (is_array($this->softrefCfg)) {
152 foreach ($this->softrefCfg as $key => $value) {
153 if (!strlen($value['mode'])) {
154 unset($this->softrefCfg[$key]);
155 }
156 }
157 }
158 // Setting in header memory:
159 // Version of file format
160 $this->dat['header']['XMLversion'] = '1.0';
161 // Initialize meta data array (to put it in top of file)
162 $this->dat['header']['meta'] = [];
163 // Add list of tables to consider static
164 $this->dat['header']['relStaticTables'] = $this->relStaticTables;
165 // The list of excluded records
166 $this->dat['header']['excludeMap'] = $this->excludeMap;
167 // Soft Reference mode for elements
168 $this->dat['header']['softrefCfg'] = $this->softrefCfg;
169 // List of extensions the import depends on.
170 $this->dat['header']['extensionDependencies'] = $this->extensionDependencies;
171 $this->dat['header']['charset'] = 'utf-8';
172 }
173
174 /**
175 * Set charset
176 *
177 * @param string $charset Charset for the content in the export. During import the character set will be converted if the target system uses another charset.
178 */
179 public function setCharset($charset)
180 {
181 $this->dat['header']['charset'] = $charset;
182 }
183
184 /**
185 * Sets meta data
186 *
187 * @param string $title Title of the export
188 * @param string $description Description of the export
189 * @param string $notes Notes about the contents
190 * @param string $packager_username Backend Username of the packager (the guy making the export)
191 * @param string $packager_name Real name of the packager
192 * @param string $packager_email Email of the packager
193 */
194 public function setMetaData($title, $description, $notes, $packager_username, $packager_name, $packager_email)
195 {
196 $this->dat['header']['meta'] = [
197 'title' => $title,
198 'description' => $description,
199 'notes' => $notes,
200 'packager_username' => $packager_username,
201 'packager_name' => $packager_name,
202 'packager_email' => $packager_email,
203 'TYPO3_version' => TYPO3_version,
204 'created' => strftime('%A %e. %B %Y', $GLOBALS['EXEC_TIME'])
205 ];
206 }
207
208 /**
209 * Option to enable having the files not included in the export file.
210 * The files are saved to a temporary folder instead.
211 *
212 * @param bool $saveFilesOutsideExportFile
213 * @see getTemporaryFilesPathForExport()
214 */
215 public function setSaveFilesOutsideExportFile($saveFilesOutsideExportFile)
216 {
217 $this->saveFilesOutsideExportFile = $saveFilesOutsideExportFile;
218 }
219
220 /**************************
221 * Export / Init Page tree
222 *************************/
223
224 /**
225 * Sets the page-tree array in the export header and returns the array in a flattened version
226 *
227 * @param array $idH Hierarchy of ids, the page tree: array([uid] => array("uid" => [uid], "subrow" => array(.....)), [uid] => ....)
228 * @return array The hierarchical page tree converted to a one-dimensional list of pages
229 */
230 public function setPageTree($idH)
231 {
232 $this->dat['header']['pagetree'] = $this->unsetExcludedSections($idH);
233 return $this->flatInversePageTree($this->dat['header']['pagetree']);
234 }
235
236 /**
237 * Removes entries in the page tree which are found in ->excludeMap[]
238 *
239 * @param array $idH Page uid hierarchy
240 * @return array Modified input array
241 * @access private
242 * @see setPageTree()
243 */
244 public function unsetExcludedSections($idH)
245 {
246 if (is_array($idH)) {
247 foreach ($idH as $k => $v) {
248 if ($this->excludeMap['pages:' . $idH[$k]['uid']]) {
249 unset($idH[$k]);
250 } elseif (is_array($idH[$k]['subrow'])) {
251 $idH[$k]['subrow'] = $this->unsetExcludedSections($idH[$k]['subrow']);
252 }
253 }
254 }
255 return $idH;
256 }
257
258 /**************************
259 * Export
260 *************************/
261
262 /**
263 * Sets the fields of record types to be included in the export
264 *
265 * @param array $recordTypesIncludeFields Keys are [recordname], values are an array of fields to be included in the export
266 * @throws Exception if an array value is not type of array
267 */
268 public function setRecordTypesIncludeFields(array $recordTypesIncludeFields)
269 {
270 foreach ($recordTypesIncludeFields as $table => $fields) {
271 if (!is_array($fields)) {
272 throw new Exception('The include fields for record type ' . htmlspecialchars($table) . ' are not defined by an array.', 1391440658);
273 }
274 $this->setRecordTypeIncludeFields($table, $fields);
275 }
276 }
277
278 /**
279 * Sets the fields of a record type to be included in the export
280 *
281 * @param string $table The record type
282 * @param array $fields The fields to be included
283 */
284 public function setRecordTypeIncludeFields($table, array $fields)
285 {
286 $this->recordTypesIncludeFields[$table] = $fields;
287 }
288
289 /**
290 * Adds the record $row from $table.
291 * No checking for relations done here. Pure data.
292 *
293 * @param string $table Table name
294 * @param array $row Record row.
295 * @param int $relationLevel (Internal) if the record is added as a relation, this is set to the "level" it was on.
296 */
297 public function export_addRecord($table, $row, $relationLevel = 0)
298 {
299 BackendUtility::workspaceOL($table, $row);
300 if ($this->excludeDisabledRecords && !$this->isActive($table, $row['uid'])) {
301 return;
302 }
303 if ((string)$table !== '' && is_array($row) && $row['uid'] > 0 && !$this->excludeMap[$table . ':' . $row['uid']]) {
304 if ($this->checkPID($table === 'pages' ? $row['uid'] : $row['pid'])) {
305 if (!isset($this->dat['records'][$table . ':' . $row['uid']])) {
306 // Prepare header info:
307 $row = $this->filterRecordFields($table, $row);
308 $headerInfo = [];
309 $headerInfo['uid'] = $row['uid'];
310 $headerInfo['pid'] = $row['pid'];
311 $headerInfo['title'] = GeneralUtility::fixed_lgd_cs(BackendUtility::getRecordTitle($table, $row), 40);
312 if ($relationLevel) {
313 $headerInfo['relationLevel'] = $relationLevel;
314 }
315 // Set the header summary:
316 $this->dat['header']['records'][$table][$row['uid']] = $headerInfo;
317 // Create entry in the PID lookup:
318 $this->dat['header']['pid_lookup'][$row['pid']][$table][$row['uid']] = 1;
319 // Initialize reference index object:
320 $refIndexObj = GeneralUtility::makeInstance(ReferenceIndex::class);
321 $refIndexObj->enableRuntimeCache();
322 // Yes to workspace overlays for exporting....
323 $refIndexObj->WSOL = true;
324 $relations = $refIndexObj->getRelations($table, $row);
325 $relations = $this->fixFileIDsInRelations($relations);
326 $relations = $this->removeSoftrefsHavingTheSameDatabaseRelation($relations);
327 // Data:
328 $this->dat['records'][$table . ':' . $row['uid']] = [];
329 $this->dat['records'][$table . ':' . $row['uid']]['data'] = $row;
330 $this->dat['records'][$table . ':' . $row['uid']]['rels'] = $relations;
331 // Add information about the relations in the record in the header:
332 $this->dat['header']['records'][$table][$row['uid']]['rels'] = $this->flatDBrels($this->dat['records'][$table . ':' . $row['uid']]['rels']);
333 // Add information about the softrefs to header:
334 $this->dat['header']['records'][$table][$row['uid']]['softrefs'] = $this->flatSoftRefs($this->dat['records'][$table . ':' . $row['uid']]['rels']);
335 } else {
336 $this->error('Record ' . $table . ':' . $row['uid'] . ' already added.');
337 }
338 } else {
339 $this->error('Record ' . $table . ':' . $row['uid'] . ' was outside your DB mounts!');
340 }
341 }
342 }
343
344 /**
345 * This changes the file reference ID from a hash based on the absolute file path
346 * (coming from ReferenceIndex) to a hash based on the relative file path.
347 *
348 * @param array $relations
349 * @return array
350 */
351 protected function fixFileIDsInRelations(array $relations)
352 {
353 foreach ($relations as $field => $relation) {
354 if (isset($relation['type']) && $relation['type'] === 'file') {
355 foreach ($relation['newValueFiles'] as $key => $fileRelationData) {
356 $absoluteFilePath = $fileRelationData['ID_absFile'];
357 if (GeneralUtility::isFirstPartOfStr($absoluteFilePath, PATH_site)) {
358 $relatedFilePath = PathUtility::stripPathSitePrefix($absoluteFilePath);
359 $relations[$field]['newValueFiles'][$key]['ID'] = md5($relatedFilePath);
360 }
361 }
362 }
363 if ($relation['type'] === 'flex') {
364 if (is_array($relation['flexFormRels']['file'])) {
365 foreach ($relation['flexFormRels']['file'] as $key => $subList) {
366 foreach ($subList as $subKey => $fileRelationData) {
367 $absoluteFilePath = $fileRelationData['ID_absFile'];
368 if (GeneralUtility::isFirstPartOfStr($absoluteFilePath, PATH_site)) {
369 $relatedFilePath = PathUtility::stripPathSitePrefix($absoluteFilePath);
370 $relations[$field]['flexFormRels']['file'][$key][$subKey]['ID'] = md5($relatedFilePath);
371 }
372 }
373 }
374 }
375 }
376 }
377 return $relations;
378 }
379
380 /**
381 * Relations could contain db relations to sys_file records. Some configuration combinations of TCA and
382 * SoftReferenceIndex create also softref relation entries for the identical file. This results
383 * in double included files, one in array "files" and one in array "file_fal".
384 * This function checks the relations for this double inclusions and removes the redundant softref relation.
385 *
386 * @param array $relations
387 * @return array
388 */
389 protected function removeSoftrefsHavingTheSameDatabaseRelation($relations)
390 {
391 $fixedRelations = [];
392 foreach ($relations as $field => $relation) {
393 $newRelation = $relation;
394 if (isset($newRelation['type']) && $newRelation['type'] === 'db') {
395 foreach ($newRelation['itemArray'] as $key => $dbRelationData) {
396 if ($dbRelationData['table'] === 'sys_file') {
397 if (isset($newRelation['softrefs']['keys']['typolink'])) {
398 foreach ($newRelation['softrefs']['keys']['typolink'] as $softrefKey => $softRefData) {
399 if ($softRefData['subst']['type'] === 'file') {
400 $file = ResourceFactory::getInstance()->retrieveFileOrFolderObject($softRefData['subst']['relFileName']);
401 if ($file instanceof File) {
402 if ($file->getUid() == $dbRelationData['id']) {
403 unset($newRelation['softrefs']['keys']['typolink'][$softrefKey]);
404 }
405 }
406 }
407 }
408 if (empty($newRelation['softrefs']['keys']['typolink'])) {
409 unset($newRelation['softrefs']);
410 }
411 }
412 }
413 }
414 }
415 $fixedRelations[$field] = $newRelation;
416 }
417 return $fixedRelations;
418 }
419
420 /**
421 * This analyses the existing added records, finds all database relations to records and adds these records to the export file.
422 * This function can be called repeatedly until it returns an empty array.
423 * In principle it should not allow to infinite recursivity, but you better set a limit...
424 * Call this BEFORE the ext_addFilesFromRelations (so files from added relations are also included of course)
425 *
426 * @param int $relationLevel Recursion level
427 * @return array overview of relations found and added: Keys [table]:[uid], values array with table and id
428 * @see export_addFilesFromRelations()
429 */
430 public function export_addDBRelations($relationLevel = 0)
431 {
432 // Traverse all "rels" registered for "records"
433 if (!is_array($this->dat['records'])) {
434 $this->error('There were no records available.');
435 return [];
436 }
437 $addR = [];
438 foreach ($this->dat['records'] as $k => $value) {
439 if (!is_array($this->dat['records'][$k])) {
440 continue;
441 }
442 foreach ($this->dat['records'][$k]['rels'] as $fieldname => $vR) {
443 // For all DB types of relations:
444 if ($vR['type'] === 'db') {
445 foreach ($vR['itemArray'] as $fI) {
446 $this->export_addDBRelations_registerRelation($fI, $addR);
447 }
448 }
449 // For all flex/db types of relations:
450 if ($vR['type'] === 'flex') {
451 // DB relations in flex form fields:
452 if (is_array($vR['flexFormRels']['db'])) {
453 foreach ($vR['flexFormRels']['db'] as $subList) {
454 foreach ($subList as $fI) {
455 $this->export_addDBRelations_registerRelation($fI, $addR);
456 }
457 }
458 }
459 // DB oriented soft references in flex form fields:
460 if (is_array($vR['flexFormRels']['softrefs'])) {
461 foreach ($vR['flexFormRels']['softrefs'] as $subList) {
462 foreach ($subList['keys'] as $spKey => $elements) {
463 foreach ($elements as $el) {
464 if ($el['subst']['type'] === 'db' && $this->includeSoftref($el['subst']['tokenID'])) {
465 list($tempTable, $tempUid) = explode(':', $el['subst']['recordRef']);
466 $fI = [
467 'table' => $tempTable,
468 'id' => $tempUid
469 ];
470 $this->export_addDBRelations_registerRelation($fI, $addR, $el['subst']['tokenID']);
471 }
472 }
473 }
474 }
475 }
476 }
477 // In any case, if there are soft refs:
478 if (is_array($vR['softrefs']['keys'])) {
479 foreach ($vR['softrefs']['keys'] as $spKey => $elements) {
480 foreach ($elements as $el) {
481 if ($el['subst']['type'] === 'db' && $this->includeSoftref($el['subst']['tokenID'])) {
482 list($tempTable, $tempUid) = explode(':', $el['subst']['recordRef']);
483 $fI = [
484 'table' => $tempTable,
485 'id' => $tempUid
486 ];
487 $this->export_addDBRelations_registerRelation($fI, $addR, $el['subst']['tokenID']);
488 }
489 }
490 }
491 }
492 }
493 }
494
495 // Now, if there were new records to add, do so:
496 if (!empty($addR)) {
497 foreach ($addR as $fI) {
498 // Get and set record:
499 $row = BackendUtility::getRecord($fI['table'], $fI['id']);
500 // Depending on db driver, int fields may or may not be returned as integer or as string. The
501 // loop aligns that detail and forces strings for everything to have exports more db agnostic.
502 foreach ($row as $fieldName => $value) {
503 // Keep null but force everything else to string
504 $row[$fieldName] = $value === null ? $value : (string)$value;
505 }
506
507 if (is_array($row)) {
508 $this->export_addRecord($fI['table'], $row, $relationLevel + 1);
509 }
510 // Set status message
511 // Relation pointers always larger than zero except certain "select" types with
512 // negative values pointing to uids - but that is not supported here.
513 if ($fI['id'] > 0) {
514 $rId = $fI['table'] . ':' . $fI['id'];
515 if (!isset($this->dat['records'][$rId])) {
516 $this->dat['records'][$rId] = 'NOT_FOUND';
517 $this->error('Relation record ' . $rId . ' was not found!');
518 }
519 }
520 }
521 }
522 // Return overview of relations found and added
523 return $addR;
524 }
525
526 /**
527 * Helper function for export_addDBRelations()
528 *
529 * @param array $fI Array with table/id keys to add
530 * @param array $addR Add array, passed by reference to be modified
531 * @param string $tokenID Softref Token ID, if applicable.
532 * @see export_addDBRelations()
533 */
534 public function export_addDBRelations_registerRelation($fI, &$addR, $tokenID = '')
535 {
536 $rId = $fI['table'] . ':' . $fI['id'];
537 if (
538 isset($GLOBALS['TCA'][$fI['table']]) && !$this->isTableStatic($fI['table']) && !$this->isExcluded($fI['table'], $fI['id'])
539 && (!$tokenID || $this->includeSoftref($tokenID)) && $this->inclRelation($fI['table'])
540 ) {
541 if (!isset($this->dat['records'][$rId])) {
542 // Set this record to be included since it is not already.
543 $addR[$rId] = $fI;
544 }
545 }
546 }
547
548 /**
549 * This adds all files in relations.
550 * Call this method AFTER adding all records including relations.
551 *
552 * @see export_addDBRelations()
553 */
554 public function export_addFilesFromRelations()
555 {
556 // Traverse all "rels" registered for "records"
557 if (!is_array($this->dat['records'])) {
558 $this->error('There were no records available.');
559 return;
560 }
561 foreach ($this->dat['records'] as $k => $value) {
562 if (!isset($this->dat['records'][$k]['rels']) || !is_array($this->dat['records'][$k]['rels'])) {
563 continue;
564 }
565 foreach ($this->dat['records'][$k]['rels'] as $fieldname => $vR) {
566 // For all file type relations:
567 if ($vR['type'] === 'file') {
568 foreach ($vR['newValueFiles'] as $key => $fI) {
569 $this->export_addFile($fI, $k, $fieldname);
570 // Remove the absolute reference to the file so it doesn't expose absolute paths from source server:
571 unset($this->dat['records'][$k]['rels'][$fieldname]['newValueFiles'][$key]['ID_absFile']);
572 }
573 }
574 // For all flex type relations:
575 if ($vR['type'] === 'flex') {
576 if (is_array($vR['flexFormRels']['file'])) {
577 foreach ($vR['flexFormRels']['file'] as $key => $subList) {
578 foreach ($subList as $subKey => $fI) {
579 $this->export_addFile($fI, $k, $fieldname);
580 // Remove the absolute reference to the file so it doesn't expose absolute paths from source server:
581 unset($this->dat['records'][$k]['rels'][$fieldname]['flexFormRels']['file'][$key][$subKey]['ID_absFile']);
582 }
583 }
584 }
585 // DB oriented soft references in flex form fields:
586 if (is_array($vR['flexFormRels']['softrefs'])) {
587 foreach ($vR['flexFormRels']['softrefs'] as $key => $subList) {
588 foreach ($subList['keys'] as $spKey => $elements) {
589 foreach ($elements as $subKey => $el) {
590 if ($el['subst']['type'] === 'file' && $this->includeSoftref($el['subst']['tokenID'])) {
591 // Create abs path and ID for file:
592 $ID_absFile = GeneralUtility::getFileAbsFileName(PATH_site . $el['subst']['relFileName']);
593 $ID = md5($el['subst']['relFileName']);
594 if ($ID_absFile) {
595 if (!$this->dat['files'][$ID]) {
596 $fI = [
597 'filename' => PathUtility::basename($ID_absFile),
598 'ID_absFile' => $ID_absFile,
599 'ID' => $ID,
600 'relFileName' => $el['subst']['relFileName']
601 ];
602 $this->export_addFile($fI, '_SOFTREF_');
603 }
604 $this->dat['records'][$k]['rels'][$fieldname]['flexFormRels']['softrefs'][$key]['keys'][$spKey][$subKey]['file_ID'] = $ID;
605 }
606 }
607 }
608 }
609 }
610 }
611 }
612 // In any case, if there are soft refs:
613 if (is_array($vR['softrefs']['keys'])) {
614 foreach ($vR['softrefs']['keys'] as $spKey => $elements) {
615 foreach ($elements as $subKey => $el) {
616 if ($el['subst']['type'] === 'file' && $this->includeSoftref($el['subst']['tokenID'])) {
617 // Create abs path and ID for file:
618 $ID_absFile = GeneralUtility::getFileAbsFileName(PATH_site . $el['subst']['relFileName']);
619 $ID = md5($el['subst']['relFileName']);
620 if ($ID_absFile) {
621 if (!$this->dat['files'][$ID]) {
622 $fI = [
623 'filename' => PathUtility::basename($ID_absFile),
624 'ID_absFile' => $ID_absFile,
625 'ID' => $ID,
626 'relFileName' => $el['subst']['relFileName']
627 ];
628 $this->export_addFile($fI, '_SOFTREF_');
629 }
630 $this->dat['records'][$k]['rels'][$fieldname]['softrefs']['keys'][$spKey][$subKey]['file_ID'] = $ID;
631 }
632 }
633 }
634 }
635 }
636 }
637 }
638 }
639
640 /**
641 * This adds all files from sys_file records
642 */
643 public function export_addFilesFromSysFilesRecords()
644 {
645 if (!isset($this->dat['header']['records']['sys_file']) || !is_array($this->dat['header']['records']['sys_file'])) {
646 return;
647 }
648 foreach ($this->dat['header']['records']['sys_file'] as $sysFileUid => $_) {
649 $recordData = $this->dat['records']['sys_file:' . $sysFileUid]['data'];
650 $file = ResourceFactory::getInstance()->createFileObject($recordData);
651 $this->export_addSysFile($file);
652 }
653 }
654
655 /**
656 * Adds a files content from a sys file record to the export memory
657 *
658 * @param File $file
659 */
660 public function export_addSysFile(File $file)
661 {
662 $fileContent = '';
663 try {
664 if (!$this->saveFilesOutsideExportFile) {
665 $fileContent = $file->getContents();
666 } else {
667 $file->checkActionPermission('read');
668 }
669 } catch (\Exception $e) {
670 $this->error('Error when trying to add file ' . $file->getCombinedIdentifier() . ': ' . $e->getMessage());
671 return;
672 }
673 $fileUid = $file->getUid();
674 $fileSha1 = $file->getStorage()->hashFile($file, 'sha1');
675 if ($fileSha1 !== $file->getProperty('sha1')) {
676 $this->error('File sha1 hash of ' . $file->getCombinedIdentifier() . ' is not up-to-date in index! File added on current sha1.');
677 $this->dat['records']['sys_file:' . $fileUid]['data']['sha1'] = $fileSha1;
678 }
679
680 $fileRec = [];
681 $fileRec['filename'] = $file->getProperty('name');
682 $fileRec['filemtime'] = $file->getProperty('modification_date');
683
684 // build unique id based on the storage and the file identifier
685 $fileId = md5($file->getStorage()->getUid() . ':' . $file->getProperty('identifier_hash'));
686
687 // Setting this data in the header
688 $this->dat['header']['files_fal'][$fileId] = $fileRec;
689
690 if (!$this->saveFilesOutsideExportFile) {
691 // ... and finally add the heavy stuff:
692 $fileRec['content'] = $fileContent;
693 } else {
694 GeneralUtility::upload_copy_move($file->getForLocalProcessing(false), $this->getTemporaryFilesPathForExport() . $file->getProperty('sha1'));
695 }
696 $fileRec['content_sha1'] = $fileSha1;
697
698 $this->dat['files_fal'][$fileId] = $fileRec;
699 }
700
701 /**
702 * Adds a files content to the export memory
703 *
704 * @param array $fI File information with three keys: "filename" = filename without path, "ID_absFile" = absolute filepath to the file (including the filename), "ID" = md5 hash of "ID_absFile". "relFileName" is optional for files attached to records, but mandatory for soft referenced files (since the relFileName determines where such a file should be stored!)
705 * @param string $recordRef If the file is related to a record, this is the id on the form [table]:[id]. Information purposes only.
706 * @param string $fieldname If the file is related to a record, this is the field name it was related to. Information purposes only.
707 */
708 public function export_addFile($fI, $recordRef = '', $fieldname = '')
709 {
710 if (!@is_file($fI['ID_absFile'])) {
711 $this->error($fI['ID_absFile'] . ' was not a file! Skipping.');
712 return;
713 }
714 $fileInfo = stat($fI['ID_absFile']);
715 $fileRec = [];
716 $fileRec['filename'] = PathUtility::basename($fI['ID_absFile']);
717 $fileRec['filemtime'] = $fileInfo['mtime'];
718 //for internal type file_reference
719 $fileRec['relFileRef'] = PathUtility::stripPathSitePrefix($fI['ID_absFile']);
720 if ($recordRef) {
721 $fileRec['record_ref'] = $recordRef . '/' . $fieldname;
722 }
723 if ($fI['relFileName']) {
724 $fileRec['relFileName'] = $fI['relFileName'];
725 }
726 // Setting this data in the header
727 $this->dat['header']['files'][$fI['ID']] = $fileRec;
728 // ... and for the recordlisting, why not let us know WHICH relations there was...
729 if ($recordRef && $recordRef !== '_SOFTREF_') {
730 $refParts = explode(':', $recordRef, 2);
731 if (!is_array($this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'])) {
732 $this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'] = [];
733 }
734 $this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'][] = $fI['ID'];
735 }
736 $fileMd5 = md5_file($fI['ID_absFile']);
737 if (!$this->saveFilesOutsideExportFile) {
738 // ... and finally add the heavy stuff:
739 $fileRec['content'] = file_get_contents($fI['ID_absFile']);
740 } else {
741 GeneralUtility::upload_copy_move($fI['ID_absFile'], $this->getTemporaryFilesPathForExport() . $fileMd5);
742 }
743 $fileRec['content_md5'] = $fileMd5;
744 $this->dat['files'][$fI['ID']] = $fileRec;
745 // For soft references, do further processing:
746 if ($recordRef === '_SOFTREF_') {
747 // RTE files?
748 if ($RTEoriginal = $this->getRTEoriginalFilename(PathUtility::basename($fI['ID_absFile']))) {
749 $RTEoriginal_absPath = PathUtility::dirname($fI['ID_absFile']) . '/' . $RTEoriginal;
750 if (@is_file($RTEoriginal_absPath)) {
751 $RTEoriginal_ID = md5($RTEoriginal_absPath);
752 $fileInfo = stat($RTEoriginal_absPath);
753 $fileRec = [];
754 $fileRec['filename'] = PathUtility::basename($RTEoriginal_absPath);
755 $fileRec['filemtime'] = $fileInfo['mtime'];
756 $fileRec['record_ref'] = '_RTE_COPY_ID:' . $fI['ID'];
757 $this->dat['header']['files'][$fI['ID']]['RTE_ORIG_ID'] = $RTEoriginal_ID;
758 // Setting this data in the header
759 $this->dat['header']['files'][$RTEoriginal_ID] = $fileRec;
760 $fileMd5 = md5_file($RTEoriginal_absPath);
761 if (!$this->saveFilesOutsideExportFile) {
762 // ... and finally add the heavy stuff:
763 $fileRec['content'] = file_get_contents($RTEoriginal_absPath);
764 } else {
765 GeneralUtility::upload_copy_move($RTEoriginal_absPath, $this->getTemporaryFilesPathForExport() . $fileMd5);
766 }
767 $fileRec['content_md5'] = $fileMd5;
768 $this->dat['files'][$RTEoriginal_ID] = $fileRec;
769 } else {
770 $this->error('RTE original file "' . PathUtility::stripPathSitePrefix($RTEoriginal_absPath) . '" was not found!');
771 }
772 }
773 // Files with external media?
774 // This is only done with files grabbed by a softreference parser since it is deemed improbable that hard-referenced files should undergo this treatment.
775 $html_fI = pathinfo(PathUtility::basename($fI['ID_absFile']));
776 if ($this->includeExtFileResources && GeneralUtility::inList($this->extFileResourceExtensions, strtolower($html_fI['extension']))) {
777 $uniquePrefix = '###' . md5($GLOBALS['EXEC_TIME']) . '###';
778 if (strtolower($html_fI['extension']) === 'css') {
779 $prefixedMedias = explode($uniquePrefix, preg_replace('/(url[[:space:]]*\\([[:space:]]*["\']?)([^"\')]*)(["\']?[[:space:]]*\\))/i', '\\1' . $uniquePrefix . '\\2' . $uniquePrefix . '\\3', $fileRec['content']));
780 } else {
781 // html, htm:
782 $htmlParser = GeneralUtility::makeInstance(HtmlParser::class);
783 $prefixedMedias = explode($uniquePrefix, $htmlParser->prefixResourcePath($uniquePrefix, $fileRec['content'], [], $uniquePrefix));
784 }
785 $htmlResourceCaptured = false;
786 foreach ($prefixedMedias as $k => $v) {
787 if ($k % 2) {
788 $EXTres_absPath = GeneralUtility::resolveBackPath(PathUtility::dirname($fI['ID_absFile']) . '/' . $v);
789 $EXTres_absPath = GeneralUtility::getFileAbsFileName($EXTres_absPath);
790 if ($EXTres_absPath && GeneralUtility::isFirstPartOfStr($EXTres_absPath, PATH_site . $this->fileadminFolderName . '/') && @is_file($EXTres_absPath)) {
791 $htmlResourceCaptured = true;
792 $EXTres_ID = md5($EXTres_absPath);
793 $this->dat['header']['files'][$fI['ID']]['EXT_RES_ID'][] = $EXTres_ID;
794 $prefixedMedias[$k] = '{EXT_RES_ID:' . $EXTres_ID . '}';
795 // Add file to memory if it is not set already:
796 if (!isset($this->dat['header']['files'][$EXTres_ID])) {
797 $fileInfo = stat($EXTres_absPath);
798 $fileRec = [];
799 $fileRec['filename'] = PathUtility::basename($EXTres_absPath);
800 $fileRec['filemtime'] = $fileInfo['mtime'];
801 $fileRec['record_ref'] = '_EXT_PARENT_:' . $fI['ID'];
802 // Media relative to the HTML file.
803 $fileRec['parentRelFileName'] = $v;
804 // Setting this data in the header
805 $this->dat['header']['files'][$EXTres_ID] = $fileRec;
806 // ... and finally add the heavy stuff:
807 $fileRec['content'] = file_get_contents($EXTres_absPath);
808 $fileRec['content_md5'] = md5($fileRec['content']);
809 $this->dat['files'][$EXTres_ID] = $fileRec;
810 }
811 }
812 }
813 }
814 if ($htmlResourceCaptured) {
815 $this->dat['files'][$fI['ID']]['tokenizedContent'] = implode('', $prefixedMedias);
816 }
817 }
818 }
819 }
820
821 /**
822 * If saveFilesOutsideExportFile is enabled, this function returns the path
823 * where the files referenced in the export are copied to.
824 *
825 * @return string
826 * @throws \RuntimeException
827 * @see setSaveFilesOutsideExportFile()
828 */
829 public function getTemporaryFilesPathForExport()
830 {
831 if (!$this->saveFilesOutsideExportFile) {
832 throw new \RuntimeException('You need to set saveFilesOutsideExportFile to TRUE before you want to get the temporary files path for export.', 1401205213);
833 }
834 if ($this->temporaryFilesPathForExport === null) {
835 $temporaryFolderName = $this->getTemporaryFolderName();
836 $this->temporaryFilesPathForExport = $temporaryFolderName . '/';
837 }
838 return $this->temporaryFilesPathForExport;
839 }
840
841 /**
842 * DB relations flattend to 1-dim array.
843 * The list will be unique, no table/uid combination will appear twice.
844 *
845 * @param array $dbrels 2-dim Array of database relations organized by table key
846 * @return array 1-dim array where entries are table:uid and keys are array with table/id
847 */
848 public function flatDBrels($dbrels)
849 {
850 $list = [];
851 foreach ($dbrels as $dat) {
852 if ($dat['type'] === 'db') {
853 foreach ($dat['itemArray'] as $i) {
854 $list[$i['table'] . ':' . $i['id']] = $i;
855 }
856 }
857 if ($dat['type'] === 'flex' && is_array($dat['flexFormRels']['db'])) {
858 foreach ($dat['flexFormRels']['db'] as $subList) {
859 foreach ($subList as $i) {
860 $list[$i['table'] . ':' . $i['id']] = $i;
861 }
862 }
863 }
864 }
865 return $list;
866 }
867
868 /**
869 * Soft References flattend to 1-dim array.
870 *
871 * @param array $dbrels 2-dim Array of database relations organized by table key
872 * @return array 1-dim array where entries are arrays with properties of the soft link found and keys are a unique combination of field, spKey, structure path if applicable and token ID
873 */
874 public function flatSoftRefs($dbrels)
875 {
876 $list = [];
877 foreach ($dbrels as $field => $dat) {
878 if (is_array($dat['softrefs']['keys'])) {
879 foreach ($dat['softrefs']['keys'] as $spKey => $elements) {
880 if (is_array($elements)) {
881 foreach ($elements as $subKey => $el) {
882 $lKey = $field . ':' . $spKey . ':' . $subKey;
883 $list[$lKey] = array_merge(['field' => $field, 'spKey' => $spKey], $el);
884 // Add file_ID key to header - slightly "risky" way of doing this because if the calculation
885 // changes for the same value in $this->records[...] this will not work anymore!
886 if ($el['subst'] && $el['subst']['relFileName']) {
887 $list[$lKey]['file_ID'] = md5(PATH_site . $el['subst']['relFileName']);
888 }
889 }
890 }
891 }
892 }
893 if ($dat['type'] === 'flex' && is_array($dat['flexFormRels']['softrefs'])) {
894 foreach ($dat['flexFormRels']['softrefs'] as $structurePath => $subSoftrefs) {
895 if (is_array($subSoftrefs['keys'])) {
896 foreach ($subSoftrefs['keys'] as $spKey => $elements) {
897 foreach ($elements as $subKey => $el) {
898 $lKey = $field . ':' . $structurePath . ':' . $spKey . ':' . $subKey;
899 $list[$lKey] = array_merge(['field' => $field, 'spKey' => $spKey, 'structurePath' => $structurePath], $el);
900 // Add file_ID key to header - slightly "risky" way of doing this because if the calculation
901 // changes for the same value in $this->records[...] this will not work anymore!
902 if ($el['subst'] && $el['subst']['relFileName']) {
903 $list[$lKey]['file_ID'] = md5(PATH_site . $el['subst']['relFileName']);
904 }
905 }
906 }
907 }
908 }
909 }
910 }
911 return $list;
912 }
913
914 /**
915 * If include fields for a specific record type are set, the data
916 * are filtered out with fields are not included in the fields.
917 *
918 * @param string $table The record type to be filtered
919 * @param array $row The data to be filtered
920 * @return array The filtered record row
921 */
922 protected function filterRecordFields($table, array $row)
923 {
924 if (isset($this->recordTypesIncludeFields[$table])) {
925 $includeFields = array_unique(array_merge(
926 $this->recordTypesIncludeFields[$table],
927 $this->defaultRecordIncludeFields
928 ));
929 $newRow = [];
930 foreach ($row as $key => $value) {
931 if (in_array($key, $includeFields)) {
932 $newRow[$key] = $value;
933 }
934 }
935 } else {
936 $newRow = $row;
937 }
938 return $newRow;
939 }
940
941 /**************************
942 * File Output
943 *************************/
944
945 /**
946 * This compiles and returns the data content for an exported file
947 *
948 * @param string $type Type of output; "xml" gives xml, otherwise serialized array, possibly compressed.
949 * @return string The output file stream
950 */
951 public function compileMemoryToFileContent($type = '')
952 {
953 if ($type === 'xml') {
954 $out = $this->createXML();
955 } else {
956 $compress = $this->doOutputCompress();
957 $out = '';
958 // adding header:
959 $out .= $this->addFilePart(serialize($this->dat['header']), $compress);
960 // adding records:
961 $out .= $this->addFilePart(serialize($this->dat['records']), $compress);
962 // adding files:
963 $out .= $this->addFilePart(serialize($this->dat['files']), $compress);
964 // adding files_fal:
965 $out .= $this->addFilePart(serialize($this->dat['files_fal']), $compress);
966 }
967 return $out;
968 }
969
970 /**
971 * Creates XML string from input array
972 *
973 * @return string XML content
974 */
975 public function createXML()
976 {
977 // Options:
978 $options = [
979 'alt_options' => [
980 '/header' => [
981 'disableTypeAttrib' => true,
982 'clearStackPath' => true,
983 'parentTagMap' => [
984 'files' => 'file',
985 'files_fal' => 'file',
986 'records' => 'table',
987 'table' => 'rec',
988 'rec:rels' => 'relations',
989 'relations' => 'element',
990 'filerefs' => 'file',
991 'pid_lookup' => 'page_contents',
992 'header:relStaticTables' => 'static_tables',
993 'static_tables' => 'tablename',
994 'excludeMap' => 'item',
995 'softrefCfg' => 'softrefExportMode',
996 'extensionDependencies' => 'extkey',
997 'softrefs' => 'softref_element'
998 ],
999 'alt_options' => [
1000 '/pagetree' => [
1001 'disableTypeAttrib' => true,
1002 'useIndexTagForNum' => 'node',
1003 'parentTagMap' => [
1004 'node:subrow' => 'node'
1005 ]
1006 ],
1007 '/pid_lookup/page_contents' => [
1008 'disableTypeAttrib' => true,
1009 'parentTagMap' => [
1010 'page_contents' => 'table'
1011 ],
1012 'grandParentTagMap' => [
1013 'page_contents/table' => 'item'
1014 ]
1015 ]
1016 ]
1017 ],
1018 '/records' => [
1019 'disableTypeAttrib' => true,
1020 'parentTagMap' => [
1021 'records' => 'tablerow',
1022 'tablerow:data' => 'fieldlist',
1023 'tablerow:rels' => 'related',
1024 'related' => 'field',
1025 'field:itemArray' => 'relations',
1026 'field:newValueFiles' => 'filerefs',
1027 'field:flexFormRels' => 'flexform',
1028 'relations' => 'element',
1029 'filerefs' => 'file',
1030 'flexform:db' => 'db_relations',
1031 'flexform:file' => 'file_relations',
1032 'flexform:softrefs' => 'softref_relations',
1033 'softref_relations' => 'structurePath',
1034 'db_relations' => 'path',
1035 'file_relations' => 'path',
1036 'path' => 'element',
1037 'keys' => 'softref_key',
1038 'softref_key' => 'softref_element'
1039 ],
1040 'alt_options' => [
1041 '/records/tablerow/fieldlist' => [
1042 'useIndexTagForAssoc' => 'field'
1043 ]
1044 ]
1045 ],
1046 '/files' => [
1047 'disableTypeAttrib' => true,
1048 'parentTagMap' => [
1049 'files' => 'file'
1050 ]
1051 ],
1052 '/files_fal' => [
1053 'disableTypeAttrib' => true,
1054 'parentTagMap' => [
1055 'files_fal' => 'file'
1056 ]
1057 ]
1058 ]
1059 ];
1060 // Creating XML file from $outputArray:
1061 $charset = $this->dat['header']['charset'] ?: 'utf-8';
1062 $XML = '<?xml version="1.0" encoding="' . $charset . '" standalone="yes" ?>' . LF;
1063 $XML .= GeneralUtility::array2xml($this->dat, '', 0, 'T3RecordDocument', 0, $options);
1064 return $XML;
1065 }
1066
1067 /**
1068 * Returns TRUE if the output should be compressed.
1069 *
1070 * @return bool TRUE if compression is possible AND requested.
1071 */
1072 public function doOutputCompress()
1073 {
1074 return $this->compress && !$this->dontCompress;
1075 }
1076
1077 /**
1078 * Returns a content part for a filename being build.
1079 *
1080 * @param array $data Data to store in part
1081 * @param bool $compress Compress file?
1082 * @return string Content stream.
1083 */
1084 public function addFilePart($data, $compress = false)
1085 {
1086 if ($compress) {
1087 $data = gzcompress($data);
1088 }
1089 return md5($data) . ':' . ($compress ? '1' : '0') . ':' . str_pad(strlen($data), 10, '0', STR_PAD_LEFT) . ':' . $data . ':';
1090 }
1091 }