01ca52c56095c1115e28c249eec7dfb313efc220
[Packages/TYPO3.CMS.git] / typo3 / sysext / impexp / Classes / Export.php
1 <?php
2 namespace TYPO3\CMS\Impexp;
3
4 /*
5 * This file is part of the TYPO3 CMS project.
6 *
7 * It is free software; you can redistribute it and/or modify it under
8 * the terms of the GNU General Public License, either version 2
9 * of the License, or any later version.
10 *
11 * For the full copyright and license information, please read the
12 * LICENSE.txt file that was distributed with this source code.
13 *
14 * The TYPO3 project - inspiring people to share!
15 */
16
17 use TYPO3\CMS\Backend\Utility\BackendUtility;
18 use TYPO3\CMS\Core\Database\ReferenceIndex;
19 use TYPO3\CMS\Core\Exception;
20 use TYPO3\CMS\Core\Html\HtmlParser;
21 use TYPO3\CMS\Core\Resource\File;
22 use TYPO3\CMS\Core\Resource\ResourceFactory;
23 use TYPO3\CMS\Core\Utility\GeneralUtility;
24 use TYPO3\CMS\Core\Utility\PathUtility;
25
26 /**
27 * EXAMPLE for using the impexp-class for exporting stuff:
28 *
29 * Create and initialize:
30 * $this->export = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Impexp\ImportExport::class);
31 * $this->export->init();
32 * Set which tables relations we will allow:
33 * $this->export->relOnlyTables[]="tt_news"; // exclusively includes. See comment in the class
34 *
35 * Adding records:
36 * $this->export->export_addRecord("pages", $this->pageinfo);
37 * $this->export->export_addRecord("pages", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("pages", 38));
38 * $this->export->export_addRecord("pages", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("pages", 39));
39 * $this->export->export_addRecord("tt_content", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("tt_content", 12));
40 * $this->export->export_addRecord("tt_content", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("tt_content", 74));
41 * $this->export->export_addRecord("sys_template", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("sys_template", 20));
42 *
43 * Adding all the relations (recursively in 5 levels so relations has THEIR relations registered as well)
44 * for($a=0;$a<5;$a++) {
45 * $addR = $this->export->export_addDBRelations($a);
46 * if (empty($addR)) break;
47 * }
48 *
49 * Finally load all the files.
50 * $this->export->export_addFilesFromRelations(); // MUST be after the DBrelations are set so that file from ALL added records are included!
51 *
52 * Write export
53 * $out = $this->export->compileMemoryToFileContent();
54 */
55
56 /**
57 * T3D file Export library (TYPO3 Record Document)
58 */
59 class Export extends ImportExport
60 {
61 /**
62 * 1MB max file size
63 *
64 * @var int
65 */
66 public $maxFileSize = 1000000;
67
68 /**
69 * 1MB max record size
70 *
71 * @var int
72 */
73 public $maxRecordSize = 1000000;
74
75 /**
76 * 10MB max export size
77 *
78 * @var int
79 */
80 public $maxExportSize = 10000000;
81
82 /**
83 * Set by user: If set, compression in t3d files is disabled
84 *
85 * @var bool
86 */
87 public $dontCompress = false;
88
89 /**
90 * If set, HTML file resources are included.
91 *
92 * @var bool
93 */
94 public $includeExtFileResources = false;
95
96 /**
97 * Files with external media (HTML/css style references inside)
98 *
99 * @var string
100 */
101 public $extFileResourceExtensions = 'html,htm,css';
102
103 /**
104 * Keys are [recordname], values are an array of fields to be included
105 * in the export
106 *
107 * @var array
108 */
109 protected $recordTypesIncludeFields = [];
110
111 /**
112 * Default array of fields to be included in the export
113 *
114 * @var array
115 */
116 protected $defaultRecordIncludeFields = ['uid', 'pid'];
117
118 /**
119 * @var bool
120 */
121 protected $saveFilesOutsideExportFile = false;
122
123 /**
124 * @var string|null
125 */
126 protected $temporaryFilesPathForExport = null;
127
128 /**************************
129 * Initialize
130 *************************/
131
132 /**
133 * Init the object
134 *
135 * @param bool $dontCompress If set, compression of t3d files is disabled
136 */
137 public function init($dontCompress = false)
138 {
139 parent::init();
140 $this->dontCompress = $dontCompress;
141 $this->mode = 'export';
142 }
143
144 /**************************
145 * Export / Init + Meta Data
146 *************************/
147
148 /**
149 * Set header basics
150 */
151 public function setHeaderBasics()
152 {
153 // Initializing:
154 if (is_array($this->softrefCfg)) {
155 foreach ($this->softrefCfg as $key => $value) {
156 if (!strlen($value['mode'])) {
157 unset($this->softrefCfg[$key]);
158 }
159 }
160 }
161 // Setting in header memory:
162 // Version of file format
163 $this->dat['header']['XMLversion'] = '1.0';
164 // Initialize meta data array (to put it in top of file)
165 $this->dat['header']['meta'] = [];
166 // Add list of tables to consider static
167 $this->dat['header']['relStaticTables'] = $this->relStaticTables;
168 // The list of excluded records
169 $this->dat['header']['excludeMap'] = $this->excludeMap;
170 // Soft Reference mode for elements
171 $this->dat['header']['softrefCfg'] = $this->softrefCfg;
172 // List of extensions the import depends on.
173 $this->dat['header']['extensionDependencies'] = $this->extensionDependencies;
174 $this->dat['header']['charset'] = 'utf-8';
175 }
176
177 /**
178 * Set charset
179 *
180 * @param string $charset Charset for the content in the export. During import the character set will be converted if the target system uses another charset.
181 */
182 public function setCharset($charset)
183 {
184 $this->dat['header']['charset'] = $charset;
185 }
186
187 /**
188 * Sets meta data
189 *
190 * @param string $title Title of the export
191 * @param string $description Description of the export
192 * @param string $notes Notes about the contents
193 * @param string $packager_username Backend Username of the packager (the guy making the export)
194 * @param string $packager_name Real name of the packager
195 * @param string $packager_email Email of the packager
196 */
197 public function setMetaData($title, $description, $notes, $packager_username, $packager_name, $packager_email)
198 {
199 $this->dat['header']['meta'] = [
200 'title' => $title,
201 'description' => $description,
202 'notes' => $notes,
203 'packager_username' => $packager_username,
204 'packager_name' => $packager_name,
205 'packager_email' => $packager_email,
206 'TYPO3_version' => TYPO3_version,
207 'created' => strftime('%A %e. %B %Y', $GLOBALS['EXEC_TIME'])
208 ];
209 }
210
211 /**
212 * Option to enable having the files not included in the export file.
213 * The files are saved to a temporary folder instead.
214 *
215 * @param bool $saveFilesOutsideExportFile
216 * @see getTemporaryFilesPathForExport()
217 */
218 public function setSaveFilesOutsideExportFile($saveFilesOutsideExportFile)
219 {
220 $this->saveFilesOutsideExportFile = $saveFilesOutsideExportFile;
221 }
222
223 /**************************
224 * Export / Init Page tree
225 *************************/
226
227 /**
228 * Sets the page-tree array in the export header and returns the array in a flattened version
229 *
230 * @param array $idH Hierarchy of ids, the page tree: array([uid] => array("uid" => [uid], "subrow" => array(.....)), [uid] => ....)
231 * @return array The hierarchical page tree converted to a one-dimensional list of pages
232 */
233 public function setPageTree($idH)
234 {
235 $this->dat['header']['pagetree'] = $this->unsetExcludedSections($idH);
236 return $this->flatInversePageTree($this->dat['header']['pagetree']);
237 }
238
239 /**
240 * Removes entries in the page tree which are found in ->excludeMap[]
241 *
242 * @param array $idH Page uid hierarchy
243 * @return array Modified input array
244 * @access private
245 * @see setPageTree()
246 */
247 public function unsetExcludedSections($idH)
248 {
249 if (is_array($idH)) {
250 foreach ($idH as $k => $v) {
251 if ($this->excludeMap['pages:' . $idH[$k]['uid']]) {
252 unset($idH[$k]);
253 } elseif (is_array($idH[$k]['subrow'])) {
254 $idH[$k]['subrow'] = $this->unsetExcludedSections($idH[$k]['subrow']);
255 }
256 }
257 }
258 return $idH;
259 }
260
261 /**************************
262 * Export
263 *************************/
264
265 /**
266 * Sets the fields of record types to be included in the export
267 *
268 * @param array $recordTypesIncludeFields Keys are [recordname], values are an array of fields to be included in the export
269 * @throws Exception if an array value is not type of array
270 */
271 public function setRecordTypesIncludeFields(array $recordTypesIncludeFields)
272 {
273 foreach ($recordTypesIncludeFields as $table => $fields) {
274 if (!is_array($fields)) {
275 throw new Exception('The include fields for record type ' . htmlspecialchars($table) . ' are not defined by an array.', 1391440658);
276 }
277 $this->setRecordTypeIncludeFields($table, $fields);
278 }
279 }
280
281 /**
282 * Sets the fields of a record type to be included in the export
283 *
284 * @param string $table The record type
285 * @param array $fields The fields to be included
286 */
287 public function setRecordTypeIncludeFields($table, array $fields)
288 {
289 $this->recordTypesIncludeFields[$table] = $fields;
290 }
291
292 /**
293 * Adds the record $row from $table.
294 * No checking for relations done here. Pure data.
295 *
296 * @param string $table Table name
297 * @param array $row Record row.
298 * @param int $relationLevel (Internal) if the record is added as a relation, this is set to the "level" it was on.
299 */
300 public function export_addRecord($table, $row, $relationLevel = 0)
301 {
302 BackendUtility::workspaceOL($table, $row);
303 if ($this->excludeDisabledRecords && !$this->isActive($table, $row['uid'])) {
304 return;
305 }
306 if ((string)$table !== '' && is_array($row) && $row['uid'] > 0 && !$this->excludeMap[$table . ':' . $row['uid']]) {
307 if ($this->checkPID($table === 'pages' ? $row['uid'] : $row['pid'])) {
308 if (!isset($this->dat['records'][$table . ':' . $row['uid']])) {
309 // Prepare header info:
310 $row = $this->filterRecordFields($table, $row);
311 $headerInfo = [];
312 $headerInfo['uid'] = $row['uid'];
313 $headerInfo['pid'] = $row['pid'];
314 $headerInfo['title'] = GeneralUtility::fixed_lgd_cs(BackendUtility::getRecordTitle($table, $row), 40);
315 $headerInfo['size'] = strlen(serialize($row));
316 if ($relationLevel) {
317 $headerInfo['relationLevel'] = $relationLevel;
318 }
319 // If record content is not too large in size, set the header content and add the rest:
320 if ($headerInfo['size'] < $this->maxRecordSize) {
321 // Set the header summary:
322 $this->dat['header']['records'][$table][$row['uid']] = $headerInfo;
323 // Create entry in the PID lookup:
324 $this->dat['header']['pid_lookup'][$row['pid']][$table][$row['uid']] = 1;
325 // Initialize reference index object:
326 $refIndexObj = GeneralUtility::makeInstance(ReferenceIndex::class);
327 $refIndexObj->enableRuntimeCache();
328 // Yes to workspace overlays for exporting....
329 $refIndexObj->WSOL = true;
330 $relations = $refIndexObj->getRelations($table, $row);
331 $relations = $this->fixFileIDsInRelations($relations);
332 $relations = $this->removeSoftrefsHavingTheSameDatabaseRelation($relations);
333 // Data:
334 $this->dat['records'][$table . ':' . $row['uid']] = [];
335 $this->dat['records'][$table . ':' . $row['uid']]['data'] = $row;
336 $this->dat['records'][$table . ':' . $row['uid']]['rels'] = $relations;
337 // Add information about the relations in the record in the header:
338 $this->dat['header']['records'][$table][$row['uid']]['rels'] = $this->flatDBrels($this->dat['records'][$table . ':' . $row['uid']]['rels']);
339 // Add information about the softrefs to header:
340 $this->dat['header']['records'][$table][$row['uid']]['softrefs'] = $this->flatSoftRefs($this->dat['records'][$table . ':' . $row['uid']]['rels']);
341 } else {
342 $this->error('Record ' . $table . ':' . $row['uid'] . ' was larger than maxRecordSize (' . GeneralUtility::formatSize($this->maxRecordSize) . ')');
343 }
344 } else {
345 $this->error('Record ' . $table . ':' . $row['uid'] . ' already added.');
346 }
347 } else {
348 $this->error('Record ' . $table . ':' . $row['uid'] . ' was outside your DB mounts!');
349 }
350 }
351 }
352
353 /**
354 * This changes the file reference ID from a hash based on the absolute file path
355 * (coming from ReferenceIndex) to a hash based on the relative file path.
356 *
357 * @param array $relations
358 * @return array
359 */
360 protected function fixFileIDsInRelations(array $relations)
361 {
362 foreach ($relations as $field => $relation) {
363 if (isset($relation['type']) && $relation['type'] === 'file') {
364 foreach ($relation['newValueFiles'] as $key => $fileRelationData) {
365 $absoluteFilePath = $fileRelationData['ID_absFile'];
366 if (GeneralUtility::isFirstPartOfStr($absoluteFilePath, PATH_site)) {
367 $relatedFilePath = PathUtility::stripPathSitePrefix($absoluteFilePath);
368 $relations[$field]['newValueFiles'][$key]['ID'] = md5($relatedFilePath);
369 }
370 }
371 }
372 if ($relation['type'] === 'flex') {
373 if (is_array($relation['flexFormRels']['file'])) {
374 foreach ($relation['flexFormRels']['file'] as $key => $subList) {
375 foreach ($subList as $subKey => $fileRelationData) {
376 $absoluteFilePath = $fileRelationData['ID_absFile'];
377 if (GeneralUtility::isFirstPartOfStr($absoluteFilePath, PATH_site)) {
378 $relatedFilePath = PathUtility::stripPathSitePrefix($absoluteFilePath);
379 $relations[$field]['flexFormRels']['file'][$key][$subKey]['ID'] = md5($relatedFilePath);
380 }
381 }
382 }
383 }
384 }
385 }
386 return $relations;
387 }
388
389 /**
390 * Relations could contain db relations to sys_file records. Some configuration combinations of TCA and
391 * SoftReferenceIndex create also softref relation entries for the identical file. This results
392 * in double included files, one in array "files" and one in array "file_fal".
393 * This function checks the relations for this double inclusions and removes the redundant softref relation.
394 *
395 * @param array $relations
396 * @return array
397 */
398 protected function removeSoftrefsHavingTheSameDatabaseRelation($relations)
399 {
400 $fixedRelations = [];
401 foreach ($relations as $field => $relation) {
402 $newRelation = $relation;
403 if (isset($newRelation['type']) && $newRelation['type'] === 'db') {
404 foreach ($newRelation['itemArray'] as $key => $dbRelationData) {
405 if ($dbRelationData['table'] === 'sys_file') {
406 if (isset($newRelation['softrefs']['keys']['typolink'])) {
407 foreach ($newRelation['softrefs']['keys']['typolink'] as $softrefKey => $softRefData) {
408 if ($softRefData['subst']['type'] === 'file') {
409 $file = ResourceFactory::getInstance()->retrieveFileOrFolderObject($softRefData['subst']['relFileName']);
410 if ($file instanceof File) {
411 if ($file->getUid() == $dbRelationData['id']) {
412 unset($newRelation['softrefs']['keys']['typolink'][$softrefKey]);
413 }
414 }
415 }
416 }
417 if (empty($newRelation['softrefs']['keys']['typolink'])) {
418 unset($newRelation['softrefs']);
419 }
420 }
421 }
422 }
423 }
424 $fixedRelations[$field] = $newRelation;
425 }
426 return $fixedRelations;
427 }
428
429 /**
430 * This analyses the existing added records, finds all database relations to records and adds these records to the export file.
431 * This function can be called repeatedly until it returns an empty array.
432 * In principle it should not allow to infinite recursivity, but you better set a limit...
433 * Call this BEFORE the ext_addFilesFromRelations (so files from added relations are also included of course)
434 *
435 * @param int $relationLevel Recursion level
436 * @return array overview of relations found and added: Keys [table]:[uid], values array with table and id
437 * @see export_addFilesFromRelations()
438 */
439 public function export_addDBRelations($relationLevel = 0)
440 {
441 // Traverse all "rels" registered for "records"
442 if (!is_array($this->dat['records'])) {
443 $this->error('There were no records available.');
444 return [];
445 }
446 $addR = [];
447 foreach ($this->dat['records'] as $k => $value) {
448 if (!is_array($this->dat['records'][$k])) {
449 continue;
450 }
451 foreach ($this->dat['records'][$k]['rels'] as $fieldname => $vR) {
452 // For all DB types of relations:
453 if ($vR['type'] === 'db') {
454 foreach ($vR['itemArray'] as $fI) {
455 $this->export_addDBRelations_registerRelation($fI, $addR);
456 }
457 }
458 // For all flex/db types of relations:
459 if ($vR['type'] === 'flex') {
460 // DB relations in flex form fields:
461 if (is_array($vR['flexFormRels']['db'])) {
462 foreach ($vR['flexFormRels']['db'] as $subList) {
463 foreach ($subList as $fI) {
464 $this->export_addDBRelations_registerRelation($fI, $addR);
465 }
466 }
467 }
468 // DB oriented soft references in flex form fields:
469 if (is_array($vR['flexFormRels']['softrefs'])) {
470 foreach ($vR['flexFormRels']['softrefs'] as $subList) {
471 foreach ($subList['keys'] as $spKey => $elements) {
472 foreach ($elements as $el) {
473 if ($el['subst']['type'] === 'db' && $this->includeSoftref($el['subst']['tokenID'])) {
474 list($tempTable, $tempUid) = explode(':', $el['subst']['recordRef']);
475 $fI = [
476 'table' => $tempTable,
477 'id' => $tempUid
478 ];
479 $this->export_addDBRelations_registerRelation($fI, $addR, $el['subst']['tokenID']);
480 }
481 }
482 }
483 }
484 }
485 }
486 // In any case, if there are soft refs:
487 if (is_array($vR['softrefs']['keys'])) {
488 foreach ($vR['softrefs']['keys'] as $spKey => $elements) {
489 foreach ($elements as $el) {
490 if ($el['subst']['type'] === 'db' && $this->includeSoftref($el['subst']['tokenID'])) {
491 list($tempTable, $tempUid) = explode(':', $el['subst']['recordRef']);
492 $fI = [
493 'table' => $tempTable,
494 'id' => $tempUid
495 ];
496 $this->export_addDBRelations_registerRelation($fI, $addR, $el['subst']['tokenID']);
497 }
498 }
499 }
500 }
501 }
502 }
503
504 // Now, if there were new records to add, do so:
505 if (!empty($addR)) {
506 foreach ($addR as $fI) {
507 // Get and set record:
508 $row = BackendUtility::getRecord($fI['table'], $fI['id']);
509 if (is_array($row)) {
510 $this->export_addRecord($fI['table'], $row, $relationLevel + 1);
511 }
512 // Set status message
513 // Relation pointers always larger than zero except certain "select" types with
514 // negative values pointing to uids - but that is not supported here.
515 if ($fI['id'] > 0) {
516 $rId = $fI['table'] . ':' . $fI['id'];
517 if (!isset($this->dat['records'][$rId])) {
518 $this->dat['records'][$rId] = 'NOT_FOUND';
519 $this->error('Relation record ' . $rId . ' was not found!');
520 }
521 }
522 }
523 }
524 // Return overview of relations found and added
525 return $addR;
526 }
527
528 /**
529 * Helper function for export_addDBRelations()
530 *
531 * @param array $fI Array with table/id keys to add
532 * @param array $addR Add array, passed by reference to be modified
533 * @param string $tokenID Softref Token ID, if applicable.
534 * @see export_addDBRelations()
535 */
536 public function export_addDBRelations_registerRelation($fI, &$addR, $tokenID = '')
537 {
538 $rId = $fI['table'] . ':' . $fI['id'];
539 if (
540 isset($GLOBALS['TCA'][$fI['table']]) && !$this->isTableStatic($fI['table']) && !$this->isExcluded($fI['table'], $fI['id'])
541 && (!$tokenID || $this->includeSoftref($tokenID)) && $this->inclRelation($fI['table'])
542 ) {
543 if (!isset($this->dat['records'][$rId])) {
544 // Set this record to be included since it is not already.
545 $addR[$rId] = $fI;
546 }
547 }
548 }
549
550 /**
551 * This adds all files in relations.
552 * Call this method AFTER adding all records including relations.
553 *
554 * @see export_addDBRelations()
555 */
556 public function export_addFilesFromRelations()
557 {
558 // Traverse all "rels" registered for "records"
559 if (!is_array($this->dat['records'])) {
560 $this->error('There were no records available.');
561 return;
562 }
563 foreach ($this->dat['records'] as $k => $value) {
564 if (!isset($this->dat['records'][$k]['rels']) || !is_array($this->dat['records'][$k]['rels'])) {
565 continue;
566 }
567 foreach ($this->dat['records'][$k]['rels'] as $fieldname => $vR) {
568 // For all file type relations:
569 if ($vR['type'] === 'file') {
570 foreach ($vR['newValueFiles'] as $key => $fI) {
571 $this->export_addFile($fI, $k, $fieldname);
572 // Remove the absolute reference to the file so it doesn't expose absolute paths from source server:
573 unset($this->dat['records'][$k]['rels'][$fieldname]['newValueFiles'][$key]['ID_absFile']);
574 }
575 }
576 // For all flex type relations:
577 if ($vR['type'] === 'flex') {
578 if (is_array($vR['flexFormRels']['file'])) {
579 foreach ($vR['flexFormRels']['file'] as $key => $subList) {
580 foreach ($subList as $subKey => $fI) {
581 $this->export_addFile($fI, $k, $fieldname);
582 // Remove the absolute reference to the file so it doesn't expose absolute paths from source server:
583 unset($this->dat['records'][$k]['rels'][$fieldname]['flexFormRels']['file'][$key][$subKey]['ID_absFile']);
584 }
585 }
586 }
587 // DB oriented soft references in flex form fields:
588 if (is_array($vR['flexFormRels']['softrefs'])) {
589 foreach ($vR['flexFormRels']['softrefs'] as $key => $subList) {
590 foreach ($subList['keys'] as $spKey => $elements) {
591 foreach ($elements as $subKey => $el) {
592 if ($el['subst']['type'] === 'file' && $this->includeSoftref($el['subst']['tokenID'])) {
593 // Create abs path and ID for file:
594 $ID_absFile = GeneralUtility::getFileAbsFileName(PATH_site . $el['subst']['relFileName']);
595 $ID = md5($el['subst']['relFileName']);
596 if ($ID_absFile) {
597 if (!$this->dat['files'][$ID]) {
598 $fI = [
599 'filename' => PathUtility::basename($ID_absFile),
600 'ID_absFile' => $ID_absFile,
601 'ID' => $ID,
602 'relFileName' => $el['subst']['relFileName']
603 ];
604 $this->export_addFile($fI, '_SOFTREF_');
605 }
606 $this->dat['records'][$k]['rels'][$fieldname]['flexFormRels']['softrefs'][$key]['keys'][$spKey][$subKey]['file_ID'] = $ID;
607 }
608 }
609 }
610 }
611 }
612 }
613 }
614 // In any case, if there are soft refs:
615 if (is_array($vR['softrefs']['keys'])) {
616 foreach ($vR['softrefs']['keys'] as $spKey => $elements) {
617 foreach ($elements as $subKey => $el) {
618 if ($el['subst']['type'] === 'file' && $this->includeSoftref($el['subst']['tokenID'])) {
619 // Create abs path and ID for file:
620 $ID_absFile = GeneralUtility::getFileAbsFileName(PATH_site . $el['subst']['relFileName']);
621 $ID = md5($el['subst']['relFileName']);
622 if ($ID_absFile) {
623 if (!$this->dat['files'][$ID]) {
624 $fI = [
625 'filename' => PathUtility::basename($ID_absFile),
626 'ID_absFile' => $ID_absFile,
627 'ID' => $ID,
628 'relFileName' => $el['subst']['relFileName']
629 ];
630 $this->export_addFile($fI, '_SOFTREF_');
631 }
632 $this->dat['records'][$k]['rels'][$fieldname]['softrefs']['keys'][$spKey][$subKey]['file_ID'] = $ID;
633 }
634 }
635 }
636 }
637 }
638 }
639 }
640 }
641
642 /**
643 * This adds all files from sys_file records
644 */
645 public function export_addFilesFromSysFilesRecords()
646 {
647 if (!isset($this->dat['header']['records']['sys_file']) || !is_array($this->dat['header']['records']['sys_file'])) {
648 return;
649 }
650 foreach ($this->dat['header']['records']['sys_file'] as $sysFileUid => $_) {
651 $recordData = $this->dat['records']['sys_file:' . $sysFileUid]['data'];
652 $file = ResourceFactory::getInstance()->createFileObject($recordData);
653 $this->export_addSysFile($file);
654 }
655 }
656
657 /**
658 * Adds a files content from a sys file record to the export memory
659 *
660 * @param File $file
661 */
662 public function export_addSysFile(File $file)
663 {
664 if ($file->getProperty('size') >= $this->maxFileSize) {
665 $this->error('File ' . $file->getPublicUrl() . ' was larger (' . GeneralUtility::formatSize($file->getProperty('size')) . ') than the maxFileSize (' . GeneralUtility::formatSize($this->maxFileSize) . ')! Skipping.');
666 return;
667 }
668 $fileContent = '';
669 try {
670 if (!$this->saveFilesOutsideExportFile) {
671 $fileContent = $file->getContents();
672 } else {
673 $file->checkActionPermission('read');
674 }
675 } catch (\Exception $e) {
676 $this->error('Error when trying to add file ' . $file->getCombinedIdentifier() . ': ' . $e->getMessage());
677 return;
678 }
679 $fileUid = $file->getUid();
680 $fileInfo = $file->getStorage()->getFileInfo($file);
681 $fileSize = (int)$fileInfo['size'];
682 if ($fileSize !== (int)$file->getProperty('size')) {
683 $this->error('File size of ' . $file->getCombinedIdentifier() . ' is not up-to-date in index! File added with current size.');
684 $this->dat['records']['sys_file:' . $fileUid]['data']['size'] = $fileSize;
685 }
686 $fileSha1 = $file->getStorage()->hashFile($file, 'sha1');
687 if ($fileSha1 !== $file->getProperty('sha1')) {
688 $this->error('File sha1 hash of ' . $file->getCombinedIdentifier() . ' is not up-to-date in index! File added on current sha1.');
689 $this->dat['records']['sys_file:' . $fileUid]['data']['sha1'] = $fileSha1;
690 }
691
692 $fileRec = [];
693 $fileRec['filesize'] = $fileSize;
694 $fileRec['filename'] = $file->getProperty('name');
695 $fileRec['filemtime'] = $file->getProperty('modification_date');
696
697 // build unique id based on the storage and the file identifier
698 $fileId = md5($file->getStorage()->getUid() . ':' . $file->getProperty('identifier_hash'));
699
700 // Setting this data in the header
701 $this->dat['header']['files_fal'][$fileId] = $fileRec;
702
703 if (!$this->saveFilesOutsideExportFile) {
704 // ... and finally add the heavy stuff:
705 $fileRec['content'] = $fileContent;
706 } else {
707 GeneralUtility::upload_copy_move($file->getForLocalProcessing(false), $this->getTemporaryFilesPathForExport() . $file->getProperty('sha1'));
708 }
709 $fileRec['content_sha1'] = $fileSha1;
710
711 $this->dat['files_fal'][$fileId] = $fileRec;
712 }
713
714 /**
715 * Adds a files content to the export memory
716 *
717 * @param array $fI File information with three keys: "filename" = filename without path, "ID_absFile" = absolute filepath to the file (including the filename), "ID" = md5 hash of "ID_absFile". "relFileName" is optional for files attached to records, but mandatory for soft referenced files (since the relFileName determines where such a file should be stored!)
718 * @param string $recordRef If the file is related to a record, this is the id on the form [table]:[id]. Information purposes only.
719 * @param string $fieldname If the file is related to a record, this is the field name it was related to. Information purposes only.
720 */
721 public function export_addFile($fI, $recordRef = '', $fieldname = '')
722 {
723 if (!@is_file($fI['ID_absFile'])) {
724 $this->error($fI['ID_absFile'] . ' was not a file! Skipping.');
725 return;
726 }
727 if (filesize($fI['ID_absFile']) >= $this->maxFileSize) {
728 $this->error($fI['ID_absFile'] . ' was larger (' . GeneralUtility::formatSize(filesize($fI['ID_absFile'])) . ') than the maxFileSize (' . GeneralUtility::formatSize($this->maxFileSize) . ')! Skipping.');
729 return;
730 }
731 $fileInfo = stat($fI['ID_absFile']);
732 $fileRec = [];
733 $fileRec['filesize'] = $fileInfo['size'];
734 $fileRec['filename'] = PathUtility::basename($fI['ID_absFile']);
735 $fileRec['filemtime'] = $fileInfo['mtime'];
736 //for internal type file_reference
737 $fileRec['relFileRef'] = PathUtility::stripPathSitePrefix($fI['ID_absFile']);
738 if ($recordRef) {
739 $fileRec['record_ref'] = $recordRef . '/' . $fieldname;
740 }
741 if ($fI['relFileName']) {
742 $fileRec['relFileName'] = $fI['relFileName'];
743 }
744 // Setting this data in the header
745 $this->dat['header']['files'][$fI['ID']] = $fileRec;
746 // ... and for the recordlisting, why not let us know WHICH relations there was...
747 if ($recordRef && $recordRef !== '_SOFTREF_') {
748 $refParts = explode(':', $recordRef, 2);
749 if (!is_array($this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'])) {
750 $this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'] = [];
751 }
752 $this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'][] = $fI['ID'];
753 }
754 $fileMd5 = md5_file($fI['ID_absFile']);
755 if (!$this->saveFilesOutsideExportFile) {
756 // ... and finally add the heavy stuff:
757 $fileRec['content'] = file_get_contents($fI['ID_absFile']);
758 } else {
759 GeneralUtility::upload_copy_move($fI['ID_absFile'], $this->getTemporaryFilesPathForExport() . $fileMd5);
760 }
761 $fileRec['content_md5'] = $fileMd5;
762 $this->dat['files'][$fI['ID']] = $fileRec;
763 // For soft references, do further processing:
764 if ($recordRef === '_SOFTREF_') {
765 // RTE files?
766 if ($RTEoriginal = $this->getRTEoriginalFilename(PathUtility::basename($fI['ID_absFile']))) {
767 $RTEoriginal_absPath = PathUtility::dirname($fI['ID_absFile']) . '/' . $RTEoriginal;
768 if (@is_file($RTEoriginal_absPath)) {
769 $RTEoriginal_ID = md5($RTEoriginal_absPath);
770 $fileInfo = stat($RTEoriginal_absPath);
771 $fileRec = [];
772 $fileRec['filesize'] = $fileInfo['size'];
773 $fileRec['filename'] = PathUtility::basename($RTEoriginal_absPath);
774 $fileRec['filemtime'] = $fileInfo['mtime'];
775 $fileRec['record_ref'] = '_RTE_COPY_ID:' . $fI['ID'];
776 $this->dat['header']['files'][$fI['ID']]['RTE_ORIG_ID'] = $RTEoriginal_ID;
777 // Setting this data in the header
778 $this->dat['header']['files'][$RTEoriginal_ID] = $fileRec;
779 $fileMd5 = md5_file($RTEoriginal_absPath);
780 if (!$this->saveFilesOutsideExportFile) {
781 // ... and finally add the heavy stuff:
782 $fileRec['content'] = file_get_contents($RTEoriginal_absPath);
783 } else {
784 GeneralUtility::upload_copy_move($RTEoriginal_absPath, $this->getTemporaryFilesPathForExport() . $fileMd5);
785 }
786 $fileRec['content_md5'] = $fileMd5;
787 $this->dat['files'][$RTEoriginal_ID] = $fileRec;
788 } else {
789 $this->error('RTE original file "' . PathUtility::stripPathSitePrefix($RTEoriginal_absPath) . '" was not found!');
790 }
791 }
792 // Files with external media?
793 // This is only done with files grabbed by a softreference parser since it is deemed improbable that hard-referenced files should undergo this treatment.
794 $html_fI = pathinfo(PathUtility::basename($fI['ID_absFile']));
795 if ($this->includeExtFileResources && GeneralUtility::inList($this->extFileResourceExtensions, strtolower($html_fI['extension']))) {
796 $uniquePrefix = '###' . md5($GLOBALS['EXEC_TIME']) . '###';
797 if (strtolower($html_fI['extension']) === 'css') {
798 $prefixedMedias = explode($uniquePrefix, preg_replace('/(url[[:space:]]*\\([[:space:]]*["\']?)([^"\')]*)(["\']?[[:space:]]*\\))/i', '\\1' . $uniquePrefix . '\\2' . $uniquePrefix . '\\3', $fileRec['content']));
799 } else {
800 // html, htm:
801 $htmlParser = GeneralUtility::makeInstance(HtmlParser::class);
802 $prefixedMedias = explode($uniquePrefix, $htmlParser->prefixResourcePath($uniquePrefix, $fileRec['content'], [], $uniquePrefix));
803 }
804 $htmlResourceCaptured = false;
805 foreach ($prefixedMedias as $k => $v) {
806 if ($k % 2) {
807 $EXTres_absPath = GeneralUtility::resolveBackPath(PathUtility::dirname($fI['ID_absFile']) . '/' . $v);
808 $EXTres_absPath = GeneralUtility::getFileAbsFileName($EXTres_absPath);
809 if ($EXTres_absPath && GeneralUtility::isFirstPartOfStr($EXTres_absPath, PATH_site . $this->fileadminFolderName . '/') && @is_file($EXTres_absPath)) {
810 $htmlResourceCaptured = true;
811 $EXTres_ID = md5($EXTres_absPath);
812 $this->dat['header']['files'][$fI['ID']]['EXT_RES_ID'][] = $EXTres_ID;
813 $prefixedMedias[$k] = '{EXT_RES_ID:' . $EXTres_ID . '}';
814 // Add file to memory if it is not set already:
815 if (!isset($this->dat['header']['files'][$EXTres_ID])) {
816 $fileInfo = stat($EXTres_absPath);
817 $fileRec = [];
818 $fileRec['filesize'] = $fileInfo['size'];
819 $fileRec['filename'] = PathUtility::basename($EXTres_absPath);
820 $fileRec['filemtime'] = $fileInfo['mtime'];
821 $fileRec['record_ref'] = '_EXT_PARENT_:' . $fI['ID'];
822 // Media relative to the HTML file.
823 $fileRec['parentRelFileName'] = $v;
824 // Setting this data in the header
825 $this->dat['header']['files'][$EXTres_ID] = $fileRec;
826 // ... and finally add the heavy stuff:
827 $fileRec['content'] = file_get_contents($EXTres_absPath);
828 $fileRec['content_md5'] = md5($fileRec['content']);
829 $this->dat['files'][$EXTres_ID] = $fileRec;
830 }
831 }
832 }
833 }
834 if ($htmlResourceCaptured) {
835 $this->dat['files'][$fI['ID']]['tokenizedContent'] = implode('', $prefixedMedias);
836 }
837 }
838 }
839 }
840
841 /**
842 * If saveFilesOutsideExportFile is enabled, this function returns the path
843 * where the files referenced in the export are copied to.
844 *
845 * @return string
846 * @throws \RuntimeException
847 * @see setSaveFilesOutsideExportFile()
848 */
849 public function getTemporaryFilesPathForExport()
850 {
851 if (!$this->saveFilesOutsideExportFile) {
852 throw new \RuntimeException('You need to set saveFilesOutsideExportFile to TRUE before you want to get the temporary files path for export.', 1401205213);
853 }
854 if ($this->temporaryFilesPathForExport === null) {
855 $temporaryFolderName = $this->getTemporaryFolderName();
856 $this->temporaryFilesPathForExport = $temporaryFolderName . '/';
857 }
858 return $this->temporaryFilesPathForExport;
859 }
860
861 /**
862 * DB relations flattend to 1-dim array.
863 * The list will be unique, no table/uid combination will appear twice.
864 *
865 * @param array $dbrels 2-dim Array of database relations organized by table key
866 * @return array 1-dim array where entries are table:uid and keys are array with table/id
867 */
868 public function flatDBrels($dbrels)
869 {
870 $list = [];
871 foreach ($dbrels as $dat) {
872 if ($dat['type'] === 'db') {
873 foreach ($dat['itemArray'] as $i) {
874 $list[$i['table'] . ':' . $i['id']] = $i;
875 }
876 }
877 if ($dat['type'] === 'flex' && is_array($dat['flexFormRels']['db'])) {
878 foreach ($dat['flexFormRels']['db'] as $subList) {
879 foreach ($subList as $i) {
880 $list[$i['table'] . ':' . $i['id']] = $i;
881 }
882 }
883 }
884 }
885 return $list;
886 }
887
888 /**
889 * Soft References flattend to 1-dim array.
890 *
891 * @param array $dbrels 2-dim Array of database relations organized by table key
892 * @return array 1-dim array where entries are arrays with properties of the soft link found and keys are a unique combination of field, spKey, structure path if applicable and token ID
893 */
894 public function flatSoftRefs($dbrels)
895 {
896 $list = [];
897 foreach ($dbrels as $field => $dat) {
898 if (is_array($dat['softrefs']['keys'])) {
899 foreach ($dat['softrefs']['keys'] as $spKey => $elements) {
900 if (is_array($elements)) {
901 foreach ($elements as $subKey => $el) {
902 $lKey = $field . ':' . $spKey . ':' . $subKey;
903 $list[$lKey] = array_merge(['field' => $field, 'spKey' => $spKey], $el);
904 // Add file_ID key to header - slightly "risky" way of doing this because if the calculation
905 // changes for the same value in $this->records[...] this will not work anymore!
906 if ($el['subst'] && $el['subst']['relFileName']) {
907 $list[$lKey]['file_ID'] = md5(PATH_site . $el['subst']['relFileName']);
908 }
909 }
910 }
911 }
912 }
913 if ($dat['type'] === 'flex' && is_array($dat['flexFormRels']['softrefs'])) {
914 foreach ($dat['flexFormRels']['softrefs'] as $structurePath => $subSoftrefs) {
915 if (is_array($subSoftrefs['keys'])) {
916 foreach ($subSoftrefs['keys'] as $spKey => $elements) {
917 foreach ($elements as $subKey => $el) {
918 $lKey = $field . ':' . $structurePath . ':' . $spKey . ':' . $subKey;
919 $list[$lKey] = array_merge(['field' => $field, 'spKey' => $spKey, 'structurePath' => $structurePath], $el);
920 // Add file_ID key to header - slightly "risky" way of doing this because if the calculation
921 // changes for the same value in $this->records[...] this will not work anymore!
922 if ($el['subst'] && $el['subst']['relFileName']) {
923 $list[$lKey]['file_ID'] = md5(PATH_site . $el['subst']['relFileName']);
924 }
925 }
926 }
927 }
928 }
929 }
930 }
931 return $list;
932 }
933
934 /**
935 * If include fields for a specific record type are set, the data
936 * are filtered out with fields are not included in the fields.
937 *
938 * @param string $table The record type to be filtered
939 * @param array $row The data to be filtered
940 * @return array The filtered record row
941 */
942 protected function filterRecordFields($table, array $row)
943 {
944 if (isset($this->recordTypesIncludeFields[$table])) {
945 $includeFields = array_unique(array_merge(
946 $this->recordTypesIncludeFields[$table],
947 $this->defaultRecordIncludeFields
948 ));
949 $newRow = [];
950 foreach ($row as $key => $value) {
951 if (in_array($key, $includeFields)) {
952 $newRow[$key] = $value;
953 }
954 }
955 } else {
956 $newRow = $row;
957 }
958 return $newRow;
959 }
960
961 /**************************
962 * File Output
963 *************************/
964
965 /**
966 * This compiles and returns the data content for an exported file
967 *
968 * @param string $type Type of output; "xml" gives xml, otherwise serialized array, possibly compressed.
969 * @return string The output file stream
970 */
971 public function compileMemoryToFileContent($type = '')
972 {
973 if ($type === 'xml') {
974 $out = $this->createXML();
975 } else {
976 $compress = $this->doOutputCompress();
977 $out = '';
978 // adding header:
979 $out .= $this->addFilePart(serialize($this->dat['header']), $compress);
980 // adding records:
981 $out .= $this->addFilePart(serialize($this->dat['records']), $compress);
982 // adding files:
983 $out .= $this->addFilePart(serialize($this->dat['files']), $compress);
984 // adding files_fal:
985 $out .= $this->addFilePart(serialize($this->dat['files_fal']), $compress);
986 }
987 return $out;
988 }
989
990 /**
991 * Creates XML string from input array
992 *
993 * @return string XML content
994 */
995 public function createXML()
996 {
997 // Options:
998 $options = [
999 'alt_options' => [
1000 '/header' => [
1001 'disableTypeAttrib' => true,
1002 'clearStackPath' => true,
1003 'parentTagMap' => [
1004 'files' => 'file',
1005 'files_fal' => 'file',
1006 'records' => 'table',
1007 'table' => 'rec',
1008 'rec:rels' => 'relations',
1009 'relations' => 'element',
1010 'filerefs' => 'file',
1011 'pid_lookup' => 'page_contents',
1012 'header:relStaticTables' => 'static_tables',
1013 'static_tables' => 'tablename',
1014 'excludeMap' => 'item',
1015 'softrefCfg' => 'softrefExportMode',
1016 'extensionDependencies' => 'extkey',
1017 'softrefs' => 'softref_element'
1018 ],
1019 'alt_options' => [
1020 '/pagetree' => [
1021 'disableTypeAttrib' => true,
1022 'useIndexTagForNum' => 'node',
1023 'parentTagMap' => [
1024 'node:subrow' => 'node'
1025 ]
1026 ],
1027 '/pid_lookup/page_contents' => [
1028 'disableTypeAttrib' => true,
1029 'parentTagMap' => [
1030 'page_contents' => 'table'
1031 ],
1032 'grandParentTagMap' => [
1033 'page_contents/table' => 'item'
1034 ]
1035 ]
1036 ]
1037 ],
1038 '/records' => [
1039 'disableTypeAttrib' => true,
1040 'parentTagMap' => [
1041 'records' => 'tablerow',
1042 'tablerow:data' => 'fieldlist',
1043 'tablerow:rels' => 'related',
1044 'related' => 'field',
1045 'field:itemArray' => 'relations',
1046 'field:newValueFiles' => 'filerefs',
1047 'field:flexFormRels' => 'flexform',
1048 'relations' => 'element',
1049 'filerefs' => 'file',
1050 'flexform:db' => 'db_relations',
1051 'flexform:file' => 'file_relations',
1052 'flexform:softrefs' => 'softref_relations',
1053 'softref_relations' => 'structurePath',
1054 'db_relations' => 'path',
1055 'file_relations' => 'path',
1056 'path' => 'element',
1057 'keys' => 'softref_key',
1058 'softref_key' => 'softref_element'
1059 ],
1060 'alt_options' => [
1061 '/records/tablerow/fieldlist' => [
1062 'useIndexTagForAssoc' => 'field'
1063 ]
1064 ]
1065 ],
1066 '/files' => [
1067 'disableTypeAttrib' => true,
1068 'parentTagMap' => [
1069 'files' => 'file'
1070 ]
1071 ],
1072 '/files_fal' => [
1073 'disableTypeAttrib' => true,
1074 'parentTagMap' => [
1075 'files_fal' => 'file'
1076 ]
1077 ]
1078 ]
1079 ];
1080 // Creating XML file from $outputArray:
1081 $charset = $this->dat['header']['charset'] ?: 'utf-8';
1082 $XML = '<?xml version="1.0" encoding="' . $charset . '" standalone="yes" ?>' . LF;
1083 $XML .= GeneralUtility::array2xml($this->dat, '', 0, 'T3RecordDocument', 0, $options);
1084 return $XML;
1085 }
1086
1087 /**
1088 * Returns TRUE if the output should be compressed.
1089 *
1090 * @return bool TRUE if compression is possible AND requested.
1091 */
1092 public function doOutputCompress()
1093 {
1094 return $this->compress && !$this->dontCompress;
1095 }
1096
1097 /**
1098 * Returns a content part for a filename being build.
1099 *
1100 * @param array $data Data to store in part
1101 * @param bool $compress Compress file?
1102 * @return string Content stream.
1103 */
1104 public function addFilePart($data, $compress = false)
1105 {
1106 if ($compress) {
1107 $data = gzcompress($data);
1108 }
1109 return md5($data) . ':' . ($compress ? '1' : '0') . ':' . str_pad(strlen($data), 10, '0', STR_PAD_LEFT) . ':' . $data . ':';
1110 }
1111 }