[TASK] impexp: Remove "Max file size" restriction
[Packages/TYPO3.CMS.git] / typo3 / sysext / impexp / Classes / Export.php
1 <?php
2 namespace TYPO3\CMS\Impexp;
3
4 /*
5 * This file is part of the TYPO3 CMS project.
6 *
7 * It is free software; you can redistribute it and/or modify it under
8 * the terms of the GNU General Public License, either version 2
9 * of the License, or any later version.
10 *
11 * For the full copyright and license information, please read the
12 * LICENSE.txt file that was distributed with this source code.
13 *
14 * The TYPO3 project - inspiring people to share!
15 */
16
17 use TYPO3\CMS\Backend\Utility\BackendUtility;
18 use TYPO3\CMS\Core\Database\ReferenceIndex;
19 use TYPO3\CMS\Core\Exception;
20 use TYPO3\CMS\Core\Html\HtmlParser;
21 use TYPO3\CMS\Core\Resource\File;
22 use TYPO3\CMS\Core\Resource\ResourceFactory;
23 use TYPO3\CMS\Core\Utility\GeneralUtility;
24 use TYPO3\CMS\Core\Utility\PathUtility;
25
26 /**
27 * EXAMPLE for using the impexp-class for exporting stuff:
28 *
29 * Create and initialize:
30 * $this->export = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Impexp\ImportExport::class);
31 * $this->export->init();
32 * Set which tables relations we will allow:
33 * $this->export->relOnlyTables[]="tt_news"; // exclusively includes. See comment in the class
34 *
35 * Adding records:
36 * $this->export->export_addRecord("pages", $this->pageinfo);
37 * $this->export->export_addRecord("pages", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("pages", 38));
38 * $this->export->export_addRecord("pages", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("pages", 39));
39 * $this->export->export_addRecord("tt_content", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("tt_content", 12));
40 * $this->export->export_addRecord("tt_content", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("tt_content", 74));
41 * $this->export->export_addRecord("sys_template", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("sys_template", 20));
42 *
43 * Adding all the relations (recursively in 5 levels so relations has THEIR relations registered as well)
44 * for($a=0;$a<5;$a++) {
45 * $addR = $this->export->export_addDBRelations($a);
46 * if (empty($addR)) break;
47 * }
48 *
49 * Finally load all the files.
50 * $this->export->export_addFilesFromRelations(); // MUST be after the DBrelations are set so that file from ALL added records are included!
51 *
52 * Write export
53 * $out = $this->export->compileMemoryToFileContent();
54 */
55
56 /**
57 * T3D file Export library (TYPO3 Record Document)
58 */
59 class Export extends ImportExport
60 {
61 /**
62 * @deprecated since TYPO3 v9, will be removed in TYPO3 v10. In v10, just remove property, it is not used any longer.
63 * @var int
64 */
65 public $maxFileSize = 1000000;
66
67 /**
68 * 1MB max record size
69 *
70 * @var int
71 */
72 public $maxRecordSize = 1000000;
73
74 /**
75 * 10MB max export size
76 *
77 * @var int
78 */
79 public $maxExportSize = 10000000;
80
81 /**
82 * Set by user: If set, compression in t3d files is disabled
83 *
84 * @var bool
85 */
86 public $dontCompress = false;
87
88 /**
89 * If set, HTML file resources are included.
90 *
91 * @var bool
92 */
93 public $includeExtFileResources = false;
94
95 /**
96 * Files with external media (HTML/css style references inside)
97 *
98 * @var string
99 */
100 public $extFileResourceExtensions = 'html,htm,css';
101
102 /**
103 * Keys are [recordname], values are an array of fields to be included
104 * in the export
105 *
106 * @var array
107 */
108 protected $recordTypesIncludeFields = [];
109
110 /**
111 * Default array of fields to be included in the export
112 *
113 * @var array
114 */
115 protected $defaultRecordIncludeFields = ['uid', 'pid'];
116
117 /**
118 * @var bool
119 */
120 protected $saveFilesOutsideExportFile = false;
121
122 /**
123 * @var string|null
124 */
125 protected $temporaryFilesPathForExport = null;
126
127 /**************************
128 * Initialize
129 *************************/
130
131 /**
132 * Init the object
133 *
134 * @param bool $dontCompress If set, compression of t3d files is disabled
135 */
136 public function init($dontCompress = false)
137 {
138 parent::init();
139 $this->dontCompress = $dontCompress;
140 $this->mode = 'export';
141 }
142
143 /**************************
144 * Export / Init + Meta Data
145 *************************/
146
147 /**
148 * Set header basics
149 */
150 public function setHeaderBasics()
151 {
152 // Initializing:
153 if (is_array($this->softrefCfg)) {
154 foreach ($this->softrefCfg as $key => $value) {
155 if (!strlen($value['mode'])) {
156 unset($this->softrefCfg[$key]);
157 }
158 }
159 }
160 // Setting in header memory:
161 // Version of file format
162 $this->dat['header']['XMLversion'] = '1.0';
163 // Initialize meta data array (to put it in top of file)
164 $this->dat['header']['meta'] = [];
165 // Add list of tables to consider static
166 $this->dat['header']['relStaticTables'] = $this->relStaticTables;
167 // The list of excluded records
168 $this->dat['header']['excludeMap'] = $this->excludeMap;
169 // Soft Reference mode for elements
170 $this->dat['header']['softrefCfg'] = $this->softrefCfg;
171 // List of extensions the import depends on.
172 $this->dat['header']['extensionDependencies'] = $this->extensionDependencies;
173 $this->dat['header']['charset'] = 'utf-8';
174 }
175
176 /**
177 * Set charset
178 *
179 * @param string $charset Charset for the content in the export. During import the character set will be converted if the target system uses another charset.
180 */
181 public function setCharset($charset)
182 {
183 $this->dat['header']['charset'] = $charset;
184 }
185
186 /**
187 * Sets meta data
188 *
189 * @param string $title Title of the export
190 * @param string $description Description of the export
191 * @param string $notes Notes about the contents
192 * @param string $packager_username Backend Username of the packager (the guy making the export)
193 * @param string $packager_name Real name of the packager
194 * @param string $packager_email Email of the packager
195 */
196 public function setMetaData($title, $description, $notes, $packager_username, $packager_name, $packager_email)
197 {
198 $this->dat['header']['meta'] = [
199 'title' => $title,
200 'description' => $description,
201 'notes' => $notes,
202 'packager_username' => $packager_username,
203 'packager_name' => $packager_name,
204 'packager_email' => $packager_email,
205 'TYPO3_version' => TYPO3_version,
206 'created' => strftime('%A %e. %B %Y', $GLOBALS['EXEC_TIME'])
207 ];
208 }
209
210 /**
211 * Option to enable having the files not included in the export file.
212 * The files are saved to a temporary folder instead.
213 *
214 * @param bool $saveFilesOutsideExportFile
215 * @see getTemporaryFilesPathForExport()
216 */
217 public function setSaveFilesOutsideExportFile($saveFilesOutsideExportFile)
218 {
219 $this->saveFilesOutsideExportFile = $saveFilesOutsideExportFile;
220 }
221
222 /**************************
223 * Export / Init Page tree
224 *************************/
225
226 /**
227 * Sets the page-tree array in the export header and returns the array in a flattened version
228 *
229 * @param array $idH Hierarchy of ids, the page tree: array([uid] => array("uid" => [uid], "subrow" => array(.....)), [uid] => ....)
230 * @return array The hierarchical page tree converted to a one-dimensional list of pages
231 */
232 public function setPageTree($idH)
233 {
234 $this->dat['header']['pagetree'] = $this->unsetExcludedSections($idH);
235 return $this->flatInversePageTree($this->dat['header']['pagetree']);
236 }
237
238 /**
239 * Removes entries in the page tree which are found in ->excludeMap[]
240 *
241 * @param array $idH Page uid hierarchy
242 * @return array Modified input array
243 * @access private
244 * @see setPageTree()
245 */
246 public function unsetExcludedSections($idH)
247 {
248 if (is_array($idH)) {
249 foreach ($idH as $k => $v) {
250 if ($this->excludeMap['pages:' . $idH[$k]['uid']]) {
251 unset($idH[$k]);
252 } elseif (is_array($idH[$k]['subrow'])) {
253 $idH[$k]['subrow'] = $this->unsetExcludedSections($idH[$k]['subrow']);
254 }
255 }
256 }
257 return $idH;
258 }
259
260 /**************************
261 * Export
262 *************************/
263
264 /**
265 * Sets the fields of record types to be included in the export
266 *
267 * @param array $recordTypesIncludeFields Keys are [recordname], values are an array of fields to be included in the export
268 * @throws Exception if an array value is not type of array
269 */
270 public function setRecordTypesIncludeFields(array $recordTypesIncludeFields)
271 {
272 foreach ($recordTypesIncludeFields as $table => $fields) {
273 if (!is_array($fields)) {
274 throw new Exception('The include fields for record type ' . htmlspecialchars($table) . ' are not defined by an array.', 1391440658);
275 }
276 $this->setRecordTypeIncludeFields($table, $fields);
277 }
278 }
279
280 /**
281 * Sets the fields of a record type to be included in the export
282 *
283 * @param string $table The record type
284 * @param array $fields The fields to be included
285 */
286 public function setRecordTypeIncludeFields($table, array $fields)
287 {
288 $this->recordTypesIncludeFields[$table] = $fields;
289 }
290
291 /**
292 * Adds the record $row from $table.
293 * No checking for relations done here. Pure data.
294 *
295 * @param string $table Table name
296 * @param array $row Record row.
297 * @param int $relationLevel (Internal) if the record is added as a relation, this is set to the "level" it was on.
298 */
299 public function export_addRecord($table, $row, $relationLevel = 0)
300 {
301 BackendUtility::workspaceOL($table, $row);
302 if ($this->excludeDisabledRecords && !$this->isActive($table, $row['uid'])) {
303 return;
304 }
305 if ((string)$table !== '' && is_array($row) && $row['uid'] > 0 && !$this->excludeMap[$table . ':' . $row['uid']]) {
306 if ($this->checkPID($table === 'pages' ? $row['uid'] : $row['pid'])) {
307 if (!isset($this->dat['records'][$table . ':' . $row['uid']])) {
308 // Prepare header info:
309 $row = $this->filterRecordFields($table, $row);
310 $headerInfo = [];
311 $headerInfo['uid'] = $row['uid'];
312 $headerInfo['pid'] = $row['pid'];
313 $headerInfo['title'] = GeneralUtility::fixed_lgd_cs(BackendUtility::getRecordTitle($table, $row), 40);
314 $headerInfo['size'] = strlen(serialize($row));
315 if ($relationLevel) {
316 $headerInfo['relationLevel'] = $relationLevel;
317 }
318 // If record content is not too large in size, set the header content and add the rest:
319 if ($headerInfo['size'] < $this->maxRecordSize) {
320 // Set the header summary:
321 $this->dat['header']['records'][$table][$row['uid']] = $headerInfo;
322 // Create entry in the PID lookup:
323 $this->dat['header']['pid_lookup'][$row['pid']][$table][$row['uid']] = 1;
324 // Initialize reference index object:
325 $refIndexObj = GeneralUtility::makeInstance(ReferenceIndex::class);
326 $refIndexObj->enableRuntimeCache();
327 // Yes to workspace overlays for exporting....
328 $refIndexObj->WSOL = true;
329 $relations = $refIndexObj->getRelations($table, $row);
330 $relations = $this->fixFileIDsInRelations($relations);
331 $relations = $this->removeSoftrefsHavingTheSameDatabaseRelation($relations);
332 // Data:
333 $this->dat['records'][$table . ':' . $row['uid']] = [];
334 $this->dat['records'][$table . ':' . $row['uid']]['data'] = $row;
335 $this->dat['records'][$table . ':' . $row['uid']]['rels'] = $relations;
336 // Add information about the relations in the record in the header:
337 $this->dat['header']['records'][$table][$row['uid']]['rels'] = $this->flatDBrels($this->dat['records'][$table . ':' . $row['uid']]['rels']);
338 // Add information about the softrefs to header:
339 $this->dat['header']['records'][$table][$row['uid']]['softrefs'] = $this->flatSoftRefs($this->dat['records'][$table . ':' . $row['uid']]['rels']);
340 } else {
341 $this->error('Record ' . $table . ':' . $row['uid'] . ' was larger than maxRecordSize (' . GeneralUtility::formatSize($this->maxRecordSize) . ')');
342 }
343 } else {
344 $this->error('Record ' . $table . ':' . $row['uid'] . ' already added.');
345 }
346 } else {
347 $this->error('Record ' . $table . ':' . $row['uid'] . ' was outside your DB mounts!');
348 }
349 }
350 }
351
352 /**
353 * This changes the file reference ID from a hash based on the absolute file path
354 * (coming from ReferenceIndex) to a hash based on the relative file path.
355 *
356 * @param array $relations
357 * @return array
358 */
359 protected function fixFileIDsInRelations(array $relations)
360 {
361 foreach ($relations as $field => $relation) {
362 if (isset($relation['type']) && $relation['type'] === 'file') {
363 foreach ($relation['newValueFiles'] as $key => $fileRelationData) {
364 $absoluteFilePath = $fileRelationData['ID_absFile'];
365 if (GeneralUtility::isFirstPartOfStr($absoluteFilePath, PATH_site)) {
366 $relatedFilePath = PathUtility::stripPathSitePrefix($absoluteFilePath);
367 $relations[$field]['newValueFiles'][$key]['ID'] = md5($relatedFilePath);
368 }
369 }
370 }
371 if ($relation['type'] === 'flex') {
372 if (is_array($relation['flexFormRels']['file'])) {
373 foreach ($relation['flexFormRels']['file'] as $key => $subList) {
374 foreach ($subList as $subKey => $fileRelationData) {
375 $absoluteFilePath = $fileRelationData['ID_absFile'];
376 if (GeneralUtility::isFirstPartOfStr($absoluteFilePath, PATH_site)) {
377 $relatedFilePath = PathUtility::stripPathSitePrefix($absoluteFilePath);
378 $relations[$field]['flexFormRels']['file'][$key][$subKey]['ID'] = md5($relatedFilePath);
379 }
380 }
381 }
382 }
383 }
384 }
385 return $relations;
386 }
387
388 /**
389 * Relations could contain db relations to sys_file records. Some configuration combinations of TCA and
390 * SoftReferenceIndex create also softref relation entries for the identical file. This results
391 * in double included files, one in array "files" and one in array "file_fal".
392 * This function checks the relations for this double inclusions and removes the redundant softref relation.
393 *
394 * @param array $relations
395 * @return array
396 */
397 protected function removeSoftrefsHavingTheSameDatabaseRelation($relations)
398 {
399 $fixedRelations = [];
400 foreach ($relations as $field => $relation) {
401 $newRelation = $relation;
402 if (isset($newRelation['type']) && $newRelation['type'] === 'db') {
403 foreach ($newRelation['itemArray'] as $key => $dbRelationData) {
404 if ($dbRelationData['table'] === 'sys_file') {
405 if (isset($newRelation['softrefs']['keys']['typolink'])) {
406 foreach ($newRelation['softrefs']['keys']['typolink'] as $softrefKey => $softRefData) {
407 if ($softRefData['subst']['type'] === 'file') {
408 $file = ResourceFactory::getInstance()->retrieveFileOrFolderObject($softRefData['subst']['relFileName']);
409 if ($file instanceof File) {
410 if ($file->getUid() == $dbRelationData['id']) {
411 unset($newRelation['softrefs']['keys']['typolink'][$softrefKey]);
412 }
413 }
414 }
415 }
416 if (empty($newRelation['softrefs']['keys']['typolink'])) {
417 unset($newRelation['softrefs']);
418 }
419 }
420 }
421 }
422 }
423 $fixedRelations[$field] = $newRelation;
424 }
425 return $fixedRelations;
426 }
427
428 /**
429 * This analyses the existing added records, finds all database relations to records and adds these records to the export file.
430 * This function can be called repeatedly until it returns an empty array.
431 * In principle it should not allow to infinite recursivity, but you better set a limit...
432 * Call this BEFORE the ext_addFilesFromRelations (so files from added relations are also included of course)
433 *
434 * @param int $relationLevel Recursion level
435 * @return array overview of relations found and added: Keys [table]:[uid], values array with table and id
436 * @see export_addFilesFromRelations()
437 */
438 public function export_addDBRelations($relationLevel = 0)
439 {
440 // Traverse all "rels" registered for "records"
441 if (!is_array($this->dat['records'])) {
442 $this->error('There were no records available.');
443 return [];
444 }
445 $addR = [];
446 foreach ($this->dat['records'] as $k => $value) {
447 if (!is_array($this->dat['records'][$k])) {
448 continue;
449 }
450 foreach ($this->dat['records'][$k]['rels'] as $fieldname => $vR) {
451 // For all DB types of relations:
452 if ($vR['type'] === 'db') {
453 foreach ($vR['itemArray'] as $fI) {
454 $this->export_addDBRelations_registerRelation($fI, $addR);
455 }
456 }
457 // For all flex/db types of relations:
458 if ($vR['type'] === 'flex') {
459 // DB relations in flex form fields:
460 if (is_array($vR['flexFormRels']['db'])) {
461 foreach ($vR['flexFormRels']['db'] as $subList) {
462 foreach ($subList as $fI) {
463 $this->export_addDBRelations_registerRelation($fI, $addR);
464 }
465 }
466 }
467 // DB oriented soft references in flex form fields:
468 if (is_array($vR['flexFormRels']['softrefs'])) {
469 foreach ($vR['flexFormRels']['softrefs'] as $subList) {
470 foreach ($subList['keys'] as $spKey => $elements) {
471 foreach ($elements as $el) {
472 if ($el['subst']['type'] === 'db' && $this->includeSoftref($el['subst']['tokenID'])) {
473 list($tempTable, $tempUid) = explode(':', $el['subst']['recordRef']);
474 $fI = [
475 'table' => $tempTable,
476 'id' => $tempUid
477 ];
478 $this->export_addDBRelations_registerRelation($fI, $addR, $el['subst']['tokenID']);
479 }
480 }
481 }
482 }
483 }
484 }
485 // In any case, if there are soft refs:
486 if (is_array($vR['softrefs']['keys'])) {
487 foreach ($vR['softrefs']['keys'] as $spKey => $elements) {
488 foreach ($elements as $el) {
489 if ($el['subst']['type'] === 'db' && $this->includeSoftref($el['subst']['tokenID'])) {
490 list($tempTable, $tempUid) = explode(':', $el['subst']['recordRef']);
491 $fI = [
492 'table' => $tempTable,
493 'id' => $tempUid
494 ];
495 $this->export_addDBRelations_registerRelation($fI, $addR, $el['subst']['tokenID']);
496 }
497 }
498 }
499 }
500 }
501 }
502
503 // Now, if there were new records to add, do so:
504 if (!empty($addR)) {
505 foreach ($addR as $fI) {
506 // Get and set record:
507 $row = BackendUtility::getRecord($fI['table'], $fI['id']);
508 if (is_array($row)) {
509 $this->export_addRecord($fI['table'], $row, $relationLevel + 1);
510 }
511 // Set status message
512 // Relation pointers always larger than zero except certain "select" types with
513 // negative values pointing to uids - but that is not supported here.
514 if ($fI['id'] > 0) {
515 $rId = $fI['table'] . ':' . $fI['id'];
516 if (!isset($this->dat['records'][$rId])) {
517 $this->dat['records'][$rId] = 'NOT_FOUND';
518 $this->error('Relation record ' . $rId . ' was not found!');
519 }
520 }
521 }
522 }
523 // Return overview of relations found and added
524 return $addR;
525 }
526
527 /**
528 * Helper function for export_addDBRelations()
529 *
530 * @param array $fI Array with table/id keys to add
531 * @param array $addR Add array, passed by reference to be modified
532 * @param string $tokenID Softref Token ID, if applicable.
533 * @see export_addDBRelations()
534 */
535 public function export_addDBRelations_registerRelation($fI, &$addR, $tokenID = '')
536 {
537 $rId = $fI['table'] . ':' . $fI['id'];
538 if (
539 isset($GLOBALS['TCA'][$fI['table']]) && !$this->isTableStatic($fI['table']) && !$this->isExcluded($fI['table'], $fI['id'])
540 && (!$tokenID || $this->includeSoftref($tokenID)) && $this->inclRelation($fI['table'])
541 ) {
542 if (!isset($this->dat['records'][$rId])) {
543 // Set this record to be included since it is not already.
544 $addR[$rId] = $fI;
545 }
546 }
547 }
548
549 /**
550 * This adds all files in relations.
551 * Call this method AFTER adding all records including relations.
552 *
553 * @see export_addDBRelations()
554 */
555 public function export_addFilesFromRelations()
556 {
557 // Traverse all "rels" registered for "records"
558 if (!is_array($this->dat['records'])) {
559 $this->error('There were no records available.');
560 return;
561 }
562 foreach ($this->dat['records'] as $k => $value) {
563 if (!isset($this->dat['records'][$k]['rels']) || !is_array($this->dat['records'][$k]['rels'])) {
564 continue;
565 }
566 foreach ($this->dat['records'][$k]['rels'] as $fieldname => $vR) {
567 // For all file type relations:
568 if ($vR['type'] === 'file') {
569 foreach ($vR['newValueFiles'] as $key => $fI) {
570 $this->export_addFile($fI, $k, $fieldname);
571 // Remove the absolute reference to the file so it doesn't expose absolute paths from source server:
572 unset($this->dat['records'][$k]['rels'][$fieldname]['newValueFiles'][$key]['ID_absFile']);
573 }
574 }
575 // For all flex type relations:
576 if ($vR['type'] === 'flex') {
577 if (is_array($vR['flexFormRels']['file'])) {
578 foreach ($vR['flexFormRels']['file'] as $key => $subList) {
579 foreach ($subList as $subKey => $fI) {
580 $this->export_addFile($fI, $k, $fieldname);
581 // Remove the absolute reference to the file so it doesn't expose absolute paths from source server:
582 unset($this->dat['records'][$k]['rels'][$fieldname]['flexFormRels']['file'][$key][$subKey]['ID_absFile']);
583 }
584 }
585 }
586 // DB oriented soft references in flex form fields:
587 if (is_array($vR['flexFormRels']['softrefs'])) {
588 foreach ($vR['flexFormRels']['softrefs'] as $key => $subList) {
589 foreach ($subList['keys'] as $spKey => $elements) {
590 foreach ($elements as $subKey => $el) {
591 if ($el['subst']['type'] === 'file' && $this->includeSoftref($el['subst']['tokenID'])) {
592 // Create abs path and ID for file:
593 $ID_absFile = GeneralUtility::getFileAbsFileName(PATH_site . $el['subst']['relFileName']);
594 $ID = md5($el['subst']['relFileName']);
595 if ($ID_absFile) {
596 if (!$this->dat['files'][$ID]) {
597 $fI = [
598 'filename' => PathUtility::basename($ID_absFile),
599 'ID_absFile' => $ID_absFile,
600 'ID' => $ID,
601 'relFileName' => $el['subst']['relFileName']
602 ];
603 $this->export_addFile($fI, '_SOFTREF_');
604 }
605 $this->dat['records'][$k]['rels'][$fieldname]['flexFormRels']['softrefs'][$key]['keys'][$spKey][$subKey]['file_ID'] = $ID;
606 }
607 }
608 }
609 }
610 }
611 }
612 }
613 // In any case, if there are soft refs:
614 if (is_array($vR['softrefs']['keys'])) {
615 foreach ($vR['softrefs']['keys'] as $spKey => $elements) {
616 foreach ($elements as $subKey => $el) {
617 if ($el['subst']['type'] === 'file' && $this->includeSoftref($el['subst']['tokenID'])) {
618 // Create abs path and ID for file:
619 $ID_absFile = GeneralUtility::getFileAbsFileName(PATH_site . $el['subst']['relFileName']);
620 $ID = md5($el['subst']['relFileName']);
621 if ($ID_absFile) {
622 if (!$this->dat['files'][$ID]) {
623 $fI = [
624 'filename' => PathUtility::basename($ID_absFile),
625 'ID_absFile' => $ID_absFile,
626 'ID' => $ID,
627 'relFileName' => $el['subst']['relFileName']
628 ];
629 $this->export_addFile($fI, '_SOFTREF_');
630 }
631 $this->dat['records'][$k]['rels'][$fieldname]['softrefs']['keys'][$spKey][$subKey]['file_ID'] = $ID;
632 }
633 }
634 }
635 }
636 }
637 }
638 }
639 }
640
641 /**
642 * This adds all files from sys_file records
643 */
644 public function export_addFilesFromSysFilesRecords()
645 {
646 if (!isset($this->dat['header']['records']['sys_file']) || !is_array($this->dat['header']['records']['sys_file'])) {
647 return;
648 }
649 foreach ($this->dat['header']['records']['sys_file'] as $sysFileUid => $_) {
650 $recordData = $this->dat['records']['sys_file:' . $sysFileUid]['data'];
651 $file = ResourceFactory::getInstance()->createFileObject($recordData);
652 $this->export_addSysFile($file);
653 }
654 }
655
656 /**
657 * Adds a files content from a sys file record to the export memory
658 *
659 * @param File $file
660 */
661 public function export_addSysFile(File $file)
662 {
663 $fileContent = '';
664 try {
665 if (!$this->saveFilesOutsideExportFile) {
666 $fileContent = $file->getContents();
667 } else {
668 $file->checkActionPermission('read');
669 }
670 } catch (\Exception $e) {
671 $this->error('Error when trying to add file ' . $file->getCombinedIdentifier() . ': ' . $e->getMessage());
672 return;
673 }
674 $fileUid = $file->getUid();
675 $fileInfo = $file->getStorage()->getFileInfo($file);
676 $fileSize = (int)$fileInfo['size'];
677 if ($fileSize !== (int)$file->getProperty('size')) {
678 $this->error('File size of ' . $file->getCombinedIdentifier() . ' is not up-to-date in index! File added with current size.');
679 $this->dat['records']['sys_file:' . $fileUid]['data']['size'] = $fileSize;
680 }
681 $fileSha1 = $file->getStorage()->hashFile($file, 'sha1');
682 if ($fileSha1 !== $file->getProperty('sha1')) {
683 $this->error('File sha1 hash of ' . $file->getCombinedIdentifier() . ' is not up-to-date in index! File added on current sha1.');
684 $this->dat['records']['sys_file:' . $fileUid]['data']['sha1'] = $fileSha1;
685 }
686
687 $fileRec = [];
688 $fileRec['filesize'] = $fileSize;
689 $fileRec['filename'] = $file->getProperty('name');
690 $fileRec['filemtime'] = $file->getProperty('modification_date');
691
692 // build unique id based on the storage and the file identifier
693 $fileId = md5($file->getStorage()->getUid() . ':' . $file->getProperty('identifier_hash'));
694
695 // Setting this data in the header
696 $this->dat['header']['files_fal'][$fileId] = $fileRec;
697
698 if (!$this->saveFilesOutsideExportFile) {
699 // ... and finally add the heavy stuff:
700 $fileRec['content'] = $fileContent;
701 } else {
702 GeneralUtility::upload_copy_move($file->getForLocalProcessing(false), $this->getTemporaryFilesPathForExport() . $file->getProperty('sha1'));
703 }
704 $fileRec['content_sha1'] = $fileSha1;
705
706 $this->dat['files_fal'][$fileId] = $fileRec;
707 }
708
709 /**
710 * Adds a files content to the export memory
711 *
712 * @param array $fI File information with three keys: "filename" = filename without path, "ID_absFile" = absolute filepath to the file (including the filename), "ID" = md5 hash of "ID_absFile". "relFileName" is optional for files attached to records, but mandatory for soft referenced files (since the relFileName determines where such a file should be stored!)
713 * @param string $recordRef If the file is related to a record, this is the id on the form [table]:[id]. Information purposes only.
714 * @param string $fieldname If the file is related to a record, this is the field name it was related to. Information purposes only.
715 */
716 public function export_addFile($fI, $recordRef = '', $fieldname = '')
717 {
718 if (!@is_file($fI['ID_absFile'])) {
719 $this->error($fI['ID_absFile'] . ' was not a file! Skipping.');
720 return;
721 }
722 $fileInfo = stat($fI['ID_absFile']);
723 $fileRec = [];
724 $fileRec['filesize'] = $fileInfo['size'];
725 $fileRec['filename'] = PathUtility::basename($fI['ID_absFile']);
726 $fileRec['filemtime'] = $fileInfo['mtime'];
727 //for internal type file_reference
728 $fileRec['relFileRef'] = PathUtility::stripPathSitePrefix($fI['ID_absFile']);
729 if ($recordRef) {
730 $fileRec['record_ref'] = $recordRef . '/' . $fieldname;
731 }
732 if ($fI['relFileName']) {
733 $fileRec['relFileName'] = $fI['relFileName'];
734 }
735 // Setting this data in the header
736 $this->dat['header']['files'][$fI['ID']] = $fileRec;
737 // ... and for the recordlisting, why not let us know WHICH relations there was...
738 if ($recordRef && $recordRef !== '_SOFTREF_') {
739 $refParts = explode(':', $recordRef, 2);
740 if (!is_array($this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'])) {
741 $this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'] = [];
742 }
743 $this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'][] = $fI['ID'];
744 }
745 $fileMd5 = md5_file($fI['ID_absFile']);
746 if (!$this->saveFilesOutsideExportFile) {
747 // ... and finally add the heavy stuff:
748 $fileRec['content'] = file_get_contents($fI['ID_absFile']);
749 } else {
750 GeneralUtility::upload_copy_move($fI['ID_absFile'], $this->getTemporaryFilesPathForExport() . $fileMd5);
751 }
752 $fileRec['content_md5'] = $fileMd5;
753 $this->dat['files'][$fI['ID']] = $fileRec;
754 // For soft references, do further processing:
755 if ($recordRef === '_SOFTREF_') {
756 // RTE files?
757 if ($RTEoriginal = $this->getRTEoriginalFilename(PathUtility::basename($fI['ID_absFile']))) {
758 $RTEoriginal_absPath = PathUtility::dirname($fI['ID_absFile']) . '/' . $RTEoriginal;
759 if (@is_file($RTEoriginal_absPath)) {
760 $RTEoriginal_ID = md5($RTEoriginal_absPath);
761 $fileInfo = stat($RTEoriginal_absPath);
762 $fileRec = [];
763 $fileRec['filesize'] = $fileInfo['size'];
764 $fileRec['filename'] = PathUtility::basename($RTEoriginal_absPath);
765 $fileRec['filemtime'] = $fileInfo['mtime'];
766 $fileRec['record_ref'] = '_RTE_COPY_ID:' . $fI['ID'];
767 $this->dat['header']['files'][$fI['ID']]['RTE_ORIG_ID'] = $RTEoriginal_ID;
768 // Setting this data in the header
769 $this->dat['header']['files'][$RTEoriginal_ID] = $fileRec;
770 $fileMd5 = md5_file($RTEoriginal_absPath);
771 if (!$this->saveFilesOutsideExportFile) {
772 // ... and finally add the heavy stuff:
773 $fileRec['content'] = file_get_contents($RTEoriginal_absPath);
774 } else {
775 GeneralUtility::upload_copy_move($RTEoriginal_absPath, $this->getTemporaryFilesPathForExport() . $fileMd5);
776 }
777 $fileRec['content_md5'] = $fileMd5;
778 $this->dat['files'][$RTEoriginal_ID] = $fileRec;
779 } else {
780 $this->error('RTE original file "' . PathUtility::stripPathSitePrefix($RTEoriginal_absPath) . '" was not found!');
781 }
782 }
783 // Files with external media?
784 // This is only done with files grabbed by a softreference parser since it is deemed improbable that hard-referenced files should undergo this treatment.
785 $html_fI = pathinfo(PathUtility::basename($fI['ID_absFile']));
786 if ($this->includeExtFileResources && GeneralUtility::inList($this->extFileResourceExtensions, strtolower($html_fI['extension']))) {
787 $uniquePrefix = '###' . md5($GLOBALS['EXEC_TIME']) . '###';
788 if (strtolower($html_fI['extension']) === 'css') {
789 $prefixedMedias = explode($uniquePrefix, preg_replace('/(url[[:space:]]*\\([[:space:]]*["\']?)([^"\')]*)(["\']?[[:space:]]*\\))/i', '\\1' . $uniquePrefix . '\\2' . $uniquePrefix . '\\3', $fileRec['content']));
790 } else {
791 // html, htm:
792 $htmlParser = GeneralUtility::makeInstance(HtmlParser::class);
793 $prefixedMedias = explode($uniquePrefix, $htmlParser->prefixResourcePath($uniquePrefix, $fileRec['content'], [], $uniquePrefix));
794 }
795 $htmlResourceCaptured = false;
796 foreach ($prefixedMedias as $k => $v) {
797 if ($k % 2) {
798 $EXTres_absPath = GeneralUtility::resolveBackPath(PathUtility::dirname($fI['ID_absFile']) . '/' . $v);
799 $EXTres_absPath = GeneralUtility::getFileAbsFileName($EXTres_absPath);
800 if ($EXTres_absPath && GeneralUtility::isFirstPartOfStr($EXTres_absPath, PATH_site . $this->fileadminFolderName . '/') && @is_file($EXTres_absPath)) {
801 $htmlResourceCaptured = true;
802 $EXTres_ID = md5($EXTres_absPath);
803 $this->dat['header']['files'][$fI['ID']]['EXT_RES_ID'][] = $EXTres_ID;
804 $prefixedMedias[$k] = '{EXT_RES_ID:' . $EXTres_ID . '}';
805 // Add file to memory if it is not set already:
806 if (!isset($this->dat['header']['files'][$EXTres_ID])) {
807 $fileInfo = stat($EXTres_absPath);
808 $fileRec = [];
809 $fileRec['filesize'] = $fileInfo['size'];
810 $fileRec['filename'] = PathUtility::basename($EXTres_absPath);
811 $fileRec['filemtime'] = $fileInfo['mtime'];
812 $fileRec['record_ref'] = '_EXT_PARENT_:' . $fI['ID'];
813 // Media relative to the HTML file.
814 $fileRec['parentRelFileName'] = $v;
815 // Setting this data in the header
816 $this->dat['header']['files'][$EXTres_ID] = $fileRec;
817 // ... and finally add the heavy stuff:
818 $fileRec['content'] = file_get_contents($EXTres_absPath);
819 $fileRec['content_md5'] = md5($fileRec['content']);
820 $this->dat['files'][$EXTres_ID] = $fileRec;
821 }
822 }
823 }
824 }
825 if ($htmlResourceCaptured) {
826 $this->dat['files'][$fI['ID']]['tokenizedContent'] = implode('', $prefixedMedias);
827 }
828 }
829 }
830 }
831
832 /**
833 * If saveFilesOutsideExportFile is enabled, this function returns the path
834 * where the files referenced in the export are copied to.
835 *
836 * @return string
837 * @throws \RuntimeException
838 * @see setSaveFilesOutsideExportFile()
839 */
840 public function getTemporaryFilesPathForExport()
841 {
842 if (!$this->saveFilesOutsideExportFile) {
843 throw new \RuntimeException('You need to set saveFilesOutsideExportFile to TRUE before you want to get the temporary files path for export.', 1401205213);
844 }
845 if ($this->temporaryFilesPathForExport === null) {
846 $temporaryFolderName = $this->getTemporaryFolderName();
847 $this->temporaryFilesPathForExport = $temporaryFolderName . '/';
848 }
849 return $this->temporaryFilesPathForExport;
850 }
851
852 /**
853 * DB relations flattend to 1-dim array.
854 * The list will be unique, no table/uid combination will appear twice.
855 *
856 * @param array $dbrels 2-dim Array of database relations organized by table key
857 * @return array 1-dim array where entries are table:uid and keys are array with table/id
858 */
859 public function flatDBrels($dbrels)
860 {
861 $list = [];
862 foreach ($dbrels as $dat) {
863 if ($dat['type'] === 'db') {
864 foreach ($dat['itemArray'] as $i) {
865 $list[$i['table'] . ':' . $i['id']] = $i;
866 }
867 }
868 if ($dat['type'] === 'flex' && is_array($dat['flexFormRels']['db'])) {
869 foreach ($dat['flexFormRels']['db'] as $subList) {
870 foreach ($subList as $i) {
871 $list[$i['table'] . ':' . $i['id']] = $i;
872 }
873 }
874 }
875 }
876 return $list;
877 }
878
879 /**
880 * Soft References flattend to 1-dim array.
881 *
882 * @param array $dbrels 2-dim Array of database relations organized by table key
883 * @return array 1-dim array where entries are arrays with properties of the soft link found and keys are a unique combination of field, spKey, structure path if applicable and token ID
884 */
885 public function flatSoftRefs($dbrels)
886 {
887 $list = [];
888 foreach ($dbrels as $field => $dat) {
889 if (is_array($dat['softrefs']['keys'])) {
890 foreach ($dat['softrefs']['keys'] as $spKey => $elements) {
891 if (is_array($elements)) {
892 foreach ($elements as $subKey => $el) {
893 $lKey = $field . ':' . $spKey . ':' . $subKey;
894 $list[$lKey] = array_merge(['field' => $field, 'spKey' => $spKey], $el);
895 // Add file_ID key to header - slightly "risky" way of doing this because if the calculation
896 // changes for the same value in $this->records[...] this will not work anymore!
897 if ($el['subst'] && $el['subst']['relFileName']) {
898 $list[$lKey]['file_ID'] = md5(PATH_site . $el['subst']['relFileName']);
899 }
900 }
901 }
902 }
903 }
904 if ($dat['type'] === 'flex' && is_array($dat['flexFormRels']['softrefs'])) {
905 foreach ($dat['flexFormRels']['softrefs'] as $structurePath => $subSoftrefs) {
906 if (is_array($subSoftrefs['keys'])) {
907 foreach ($subSoftrefs['keys'] as $spKey => $elements) {
908 foreach ($elements as $subKey => $el) {
909 $lKey = $field . ':' . $structurePath . ':' . $spKey . ':' . $subKey;
910 $list[$lKey] = array_merge(['field' => $field, 'spKey' => $spKey, 'structurePath' => $structurePath], $el);
911 // Add file_ID key to header - slightly "risky" way of doing this because if the calculation
912 // changes for the same value in $this->records[...] this will not work anymore!
913 if ($el['subst'] && $el['subst']['relFileName']) {
914 $list[$lKey]['file_ID'] = md5(PATH_site . $el['subst']['relFileName']);
915 }
916 }
917 }
918 }
919 }
920 }
921 }
922 return $list;
923 }
924
925 /**
926 * If include fields for a specific record type are set, the data
927 * are filtered out with fields are not included in the fields.
928 *
929 * @param string $table The record type to be filtered
930 * @param array $row The data to be filtered
931 * @return array The filtered record row
932 */
933 protected function filterRecordFields($table, array $row)
934 {
935 if (isset($this->recordTypesIncludeFields[$table])) {
936 $includeFields = array_unique(array_merge(
937 $this->recordTypesIncludeFields[$table],
938 $this->defaultRecordIncludeFields
939 ));
940 $newRow = [];
941 foreach ($row as $key => $value) {
942 if (in_array($key, $includeFields)) {
943 $newRow[$key] = $value;
944 }
945 }
946 } else {
947 $newRow = $row;
948 }
949 return $newRow;
950 }
951
952 /**************************
953 * File Output
954 *************************/
955
956 /**
957 * This compiles and returns the data content for an exported file
958 *
959 * @param string $type Type of output; "xml" gives xml, otherwise serialized array, possibly compressed.
960 * @return string The output file stream
961 */
962 public function compileMemoryToFileContent($type = '')
963 {
964 if ($type === 'xml') {
965 $out = $this->createXML();
966 } else {
967 $compress = $this->doOutputCompress();
968 $out = '';
969 // adding header:
970 $out .= $this->addFilePart(serialize($this->dat['header']), $compress);
971 // adding records:
972 $out .= $this->addFilePart(serialize($this->dat['records']), $compress);
973 // adding files:
974 $out .= $this->addFilePart(serialize($this->dat['files']), $compress);
975 // adding files_fal:
976 $out .= $this->addFilePart(serialize($this->dat['files_fal']), $compress);
977 }
978 return $out;
979 }
980
981 /**
982 * Creates XML string from input array
983 *
984 * @return string XML content
985 */
986 public function createXML()
987 {
988 // Options:
989 $options = [
990 'alt_options' => [
991 '/header' => [
992 'disableTypeAttrib' => true,
993 'clearStackPath' => true,
994 'parentTagMap' => [
995 'files' => 'file',
996 'files_fal' => 'file',
997 'records' => 'table',
998 'table' => 'rec',
999 'rec:rels' => 'relations',
1000 'relations' => 'element',
1001 'filerefs' => 'file',
1002 'pid_lookup' => 'page_contents',
1003 'header:relStaticTables' => 'static_tables',
1004 'static_tables' => 'tablename',
1005 'excludeMap' => 'item',
1006 'softrefCfg' => 'softrefExportMode',
1007 'extensionDependencies' => 'extkey',
1008 'softrefs' => 'softref_element'
1009 ],
1010 'alt_options' => [
1011 '/pagetree' => [
1012 'disableTypeAttrib' => true,
1013 'useIndexTagForNum' => 'node',
1014 'parentTagMap' => [
1015 'node:subrow' => 'node'
1016 ]
1017 ],
1018 '/pid_lookup/page_contents' => [
1019 'disableTypeAttrib' => true,
1020 'parentTagMap' => [
1021 'page_contents' => 'table'
1022 ],
1023 'grandParentTagMap' => [
1024 'page_contents/table' => 'item'
1025 ]
1026 ]
1027 ]
1028 ],
1029 '/records' => [
1030 'disableTypeAttrib' => true,
1031 'parentTagMap' => [
1032 'records' => 'tablerow',
1033 'tablerow:data' => 'fieldlist',
1034 'tablerow:rels' => 'related',
1035 'related' => 'field',
1036 'field:itemArray' => 'relations',
1037 'field:newValueFiles' => 'filerefs',
1038 'field:flexFormRels' => 'flexform',
1039 'relations' => 'element',
1040 'filerefs' => 'file',
1041 'flexform:db' => 'db_relations',
1042 'flexform:file' => 'file_relations',
1043 'flexform:softrefs' => 'softref_relations',
1044 'softref_relations' => 'structurePath',
1045 'db_relations' => 'path',
1046 'file_relations' => 'path',
1047 'path' => 'element',
1048 'keys' => 'softref_key',
1049 'softref_key' => 'softref_element'
1050 ],
1051 'alt_options' => [
1052 '/records/tablerow/fieldlist' => [
1053 'useIndexTagForAssoc' => 'field'
1054 ]
1055 ]
1056 ],
1057 '/files' => [
1058 'disableTypeAttrib' => true,
1059 'parentTagMap' => [
1060 'files' => 'file'
1061 ]
1062 ],
1063 '/files_fal' => [
1064 'disableTypeAttrib' => true,
1065 'parentTagMap' => [
1066 'files_fal' => 'file'
1067 ]
1068 ]
1069 ]
1070 ];
1071 // Creating XML file from $outputArray:
1072 $charset = $this->dat['header']['charset'] ?: 'utf-8';
1073 $XML = '<?xml version="1.0" encoding="' . $charset . '" standalone="yes" ?>' . LF;
1074 $XML .= GeneralUtility::array2xml($this->dat, '', 0, 'T3RecordDocument', 0, $options);
1075 return $XML;
1076 }
1077
1078 /**
1079 * Returns TRUE if the output should be compressed.
1080 *
1081 * @return bool TRUE if compression is possible AND requested.
1082 */
1083 public function doOutputCompress()
1084 {
1085 return $this->compress && !$this->dontCompress;
1086 }
1087
1088 /**
1089 * Returns a content part for a filename being build.
1090 *
1091 * @param array $data Data to store in part
1092 * @param bool $compress Compress file?
1093 * @return string Content stream.
1094 */
1095 public function addFilePart($data, $compress = false)
1096 {
1097 if ($compress) {
1098 $data = gzcompress($data);
1099 }
1100 return md5($data) . ':' . ($compress ? '1' : '0') . ':' . str_pad(strlen($data), 10, '0', STR_PAD_LEFT) . ':' . $data . ':';
1101 }
1102 }