[TASK] Use file_get_contents instead of getUrl()
[Packages/TYPO3.CMS.git] / typo3 / sysext / impexp / Classes / Export.php
1 <?php
2 namespace TYPO3\CMS\Impexp;
3
4 /*
5 * This file is part of the TYPO3 CMS project.
6 *
7 * It is free software; you can redistribute it and/or modify it under
8 * the terms of the GNU General Public License, either version 2
9 * of the License, or any later version.
10 *
11 * For the full copyright and license information, please read the
12 * LICENSE.txt file that was distributed with this source code.
13 *
14 * The TYPO3 project - inspiring people to share!
15 */
16
17 use TYPO3\CMS\Backend\Utility\BackendUtility;
18 use TYPO3\CMS\Core\Database\ReferenceIndex;
19 use TYPO3\CMS\Core\Exception;
20 use TYPO3\CMS\Core\Html\HtmlParser;
21 use TYPO3\CMS\Core\Resource\File;
22 use TYPO3\CMS\Core\Resource\ResourceFactory;
23 use TYPO3\CMS\Core\Utility\GeneralUtility;
24 use TYPO3\CMS\Core\Utility\PathUtility;
25
26 /**
27 * EXAMPLE for using the impexp-class for exporting stuff:
28 *
29 * Create and initialize:
30 * $this->export = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Impexp\ImportExport::class);
31 * $this->export->init();
32 * Set which tables relations we will allow:
33 * $this->export->relOnlyTables[]="tt_news"; // exclusively includes. See comment in the class
34 *
35 * Adding records:
36 * $this->export->export_addRecord("pages", $this->pageinfo);
37 * $this->export->export_addRecord("pages", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("pages", 38));
38 * $this->export->export_addRecord("pages", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("pages", 39));
39 * $this->export->export_addRecord("tt_content", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("tt_content", 12));
40 * $this->export->export_addRecord("tt_content", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("tt_content", 74));
41 * $this->export->export_addRecord("sys_template", \TYPO3\CMS\Backend\Utility\BackendUtility::getRecord("sys_template", 20));
42 *
43 * Adding all the relations (recursively in 5 levels so relations has THEIR relations registered as well)
44 * for($a=0;$a<5;$a++) {
45 * $addR = $this->export->export_addDBRelations($a);
46 * if (empty($addR)) break;
47 * }
48 *
49 * Finally load all the files.
50 * $this->export->export_addFilesFromRelations(); // MUST be after the DBrelations are set so that file from ALL added records are included!
51 *
52 * Write export
53 * $out = $this->export->compileMemoryToFileContent();
54 */
55
56 /**
57 * T3D file Export library (TYPO3 Record Document)
58 */
59 class Export extends ImportExport
60 {
61 /**
62 * 1MB max file size
63 *
64 * @var int
65 */
66 public $maxFileSize = 1000000;
67
68 /**
69 * 1MB max record size
70 *
71 * @var int
72 */
73 public $maxRecordSize = 1000000;
74
75 /**
76 * 10MB max export size
77 *
78 * @var int
79 */
80 public $maxExportSize = 10000000;
81
82 /**
83 * Set by user: If set, compression in t3d files is disabled
84 *
85 * @var bool
86 */
87 public $dontCompress = false;
88
89 /**
90 * If set, HTML file resources are included.
91 *
92 * @var bool
93 */
94 public $includeExtFileResources = false;
95
96 /**
97 * Files with external media (HTML/css style references inside)
98 *
99 * @var string
100 */
101 public $extFileResourceExtensions = 'html,htm,css';
102
103 /**
104 * Keys are [recordname], values are an array of fields to be included
105 * in the export
106 *
107 * @var array
108 */
109 protected $recordTypesIncludeFields = array();
110
111 /**
112 * Default array of fields to be included in the export
113 *
114 * @var array
115 */
116 protected $defaultRecordIncludeFields = array('uid', 'pid');
117
118 /**
119 * @var bool
120 */
121 protected $saveFilesOutsideExportFile = false;
122
123 /**
124 * @var NULL|string
125 */
126 protected $temporaryFilesPathForExport = null;
127
128 /**************************
129 * Initialize
130 *************************/
131
132 /**
133 * Init the object
134 *
135 * @param bool $dontCompress If set, compression of t3d files is disabled
136 * @return void
137 */
138 public function init($dontCompress = false)
139 {
140 parent::init();
141 $this->dontCompress = $dontCompress;
142 $this->mode = 'export';
143 }
144
145 /**************************
146 * Export / Init + Meta Data
147 *************************/
148
149 /**
150 * Set header basics
151 *
152 * @return void
153 */
154 public function setHeaderBasics()
155 {
156 // Initializing:
157 if (is_array($this->softrefCfg)) {
158 foreach ($this->softrefCfg as $key => $value) {
159 if (!strlen($value['mode'])) {
160 unset($this->softrefCfg[$key]);
161 }
162 }
163 }
164 // Setting in header memory:
165 // Version of file format
166 $this->dat['header']['XMLversion'] = '1.0';
167 // Initialize meta data array (to put it in top of file)
168 $this->dat['header']['meta'] = array();
169 // Add list of tables to consider static
170 $this->dat['header']['relStaticTables'] = $this->relStaticTables;
171 // The list of excluded records
172 $this->dat['header']['excludeMap'] = $this->excludeMap;
173 // Soft Reference mode for elements
174 $this->dat['header']['softrefCfg'] = $this->softrefCfg;
175 // List of extensions the import depends on.
176 $this->dat['header']['extensionDependencies'] = $this->extensionDependencies;
177 $this->dat['header']['charset'] = 'utf-8';
178 }
179
180 /**
181 * Set charset
182 *
183 * @param string $charset Charset for the content in the export. During import the character set will be converted if the target system uses another charset.
184 * @return void
185 */
186 public function setCharset($charset)
187 {
188 $this->dat['header']['charset'] = $charset;
189 }
190
191 /**
192 * Sets meta data
193 *
194 * @param string $title Title of the export
195 * @param string $description Description of the export
196 * @param string $notes Notes about the contents
197 * @param string $packager_username Backend Username of the packager (the guy making the export)
198 * @param string $packager_name Real name of the packager
199 * @param string $packager_email Email of the packager
200 * @return void
201 */
202 public function setMetaData($title, $description, $notes, $packager_username, $packager_name, $packager_email)
203 {
204 $this->dat['header']['meta'] = array(
205 'title' => $title,
206 'description' => $description,
207 'notes' => $notes,
208 'packager_username' => $packager_username,
209 'packager_name' => $packager_name,
210 'packager_email' => $packager_email,
211 'TYPO3_version' => TYPO3_version,
212 'created' => strftime('%A %e. %B %Y', $GLOBALS['EXEC_TIME'])
213 );
214 }
215
216 /**
217 * Option to enable having the files not included in the export file.
218 * The files are saved to a temporary folder instead.
219 *
220 * @param bool $saveFilesOutsideExportFile
221 * @see getTemporaryFilesPathForExport()
222 */
223 public function setSaveFilesOutsideExportFile($saveFilesOutsideExportFile)
224 {
225 $this->saveFilesOutsideExportFile = $saveFilesOutsideExportFile;
226 }
227
228 /**************************
229 * Export / Init Page tree
230 *************************/
231
232 /**
233 * Sets the page-tree array in the export header and returns the array in a flattened version
234 *
235 * @param array $idH Hierarchy of ids, the page tree: array([uid] => array("uid" => [uid], "subrow" => array(.....)), [uid] => ....)
236 * @return array The hierarchical page tree converted to a one-dimensional list of pages
237 */
238 public function setPageTree($idH)
239 {
240 $this->dat['header']['pagetree'] = $this->unsetExcludedSections($idH);
241 return $this->flatInversePageTree($this->dat['header']['pagetree']);
242 }
243
244 /**
245 * Removes entries in the page tree which are found in ->excludeMap[]
246 *
247 * @param array $idH Page uid hierarchy
248 * @return array Modified input array
249 * @access private
250 * @see setPageTree()
251 */
252 public function unsetExcludedSections($idH)
253 {
254 if (is_array($idH)) {
255 foreach ($idH as $k => $v) {
256 if ($this->excludeMap['pages:' . $idH[$k]['uid']]) {
257 unset($idH[$k]);
258 } elseif (is_array($idH[$k]['subrow'])) {
259 $idH[$k]['subrow'] = $this->unsetExcludedSections($idH[$k]['subrow']);
260 }
261 }
262 }
263 return $idH;
264 }
265
266 /**************************
267 * Export
268 *************************/
269
270 /**
271 * Sets the fields of record types to be included in the export
272 *
273 * @param array $recordTypesIncludeFields Keys are [recordname], values are an array of fields to be included in the export
274 * @throws Exception if an array value is not type of array
275 * @return void
276 */
277 public function setRecordTypesIncludeFields(array $recordTypesIncludeFields)
278 {
279 foreach ($recordTypesIncludeFields as $table => $fields) {
280 if (!is_array($fields)) {
281 throw new Exception('The include fields for record type ' . htmlspecialchars($table) . ' are not defined by an array.', 1391440658);
282 }
283 $this->setRecordTypeIncludeFields($table, $fields);
284 }
285 }
286
287 /**
288 * Sets the fields of a record type to be included in the export
289 *
290 * @param string $table The record type
291 * @param array $fields The fields to be included
292 * @return void
293 */
294 public function setRecordTypeIncludeFields($table, array $fields)
295 {
296 $this->recordTypesIncludeFields[$table] = $fields;
297 }
298
299 /**
300 * Adds the record $row from $table.
301 * No checking for relations done here. Pure data.
302 *
303 * @param string $table Table name
304 * @param array $row Record row.
305 * @param int $relationLevel (Internal) if the record is added as a relation, this is set to the "level" it was on.
306 * @return void
307 */
308 public function export_addRecord($table, $row, $relationLevel = 0)
309 {
310 BackendUtility::workspaceOL($table, $row);
311 if ($this->excludeDisabledRecords && !$this->isActive($table, $row['uid'])) {
312 return;
313 }
314 if ((string)$table !== '' && is_array($row) && $row['uid'] > 0 && !$this->excludeMap[$table . ':' . $row['uid']]) {
315 if ($this->checkPID($table === 'pages' ? $row['uid'] : $row['pid'])) {
316 if (!isset($this->dat['records'][$table . ':' . $row['uid']])) {
317 // Prepare header info:
318 $row = $this->filterRecordFields($table, $row);
319 $headerInfo = array();
320 $headerInfo['uid'] = $row['uid'];
321 $headerInfo['pid'] = $row['pid'];
322 $headerInfo['title'] = GeneralUtility::fixed_lgd_cs(BackendUtility::getRecordTitle($table, $row), 40);
323 $headerInfo['size'] = strlen(serialize($row));
324 if ($relationLevel) {
325 $headerInfo['relationLevel'] = $relationLevel;
326 }
327 // If record content is not too large in size, set the header content and add the rest:
328 if ($headerInfo['size'] < $this->maxRecordSize) {
329 // Set the header summary:
330 $this->dat['header']['records'][$table][$row['uid']] = $headerInfo;
331 // Create entry in the PID lookup:
332 $this->dat['header']['pid_lookup'][$row['pid']][$table][$row['uid']] = 1;
333 // Initialize reference index object:
334 $refIndexObj = GeneralUtility::makeInstance(ReferenceIndex::class);
335 // Yes to workspace overlays for exporting....
336 $refIndexObj->WSOL = true;
337 $relations = $refIndexObj->getRelations($table, $row);
338 $relations = $this->fixFileIDsInRelations($relations);
339 $relations = $this->removeSoftrefsHavingTheSameDatabaseRelation($relations);
340 // Data:
341 $this->dat['records'][$table . ':' . $row['uid']] = array();
342 $this->dat['records'][$table . ':' . $row['uid']]['data'] = $row;
343 $this->dat['records'][$table . ':' . $row['uid']]['rels'] = $relations;
344 // Add information about the relations in the record in the header:
345 $this->dat['header']['records'][$table][$row['uid']]['rels'] = $this->flatDBrels($this->dat['records'][$table . ':' . $row['uid']]['rels']);
346 // Add information about the softrefs to header:
347 $this->dat['header']['records'][$table][$row['uid']]['softrefs'] = $this->flatSoftRefs($this->dat['records'][$table . ':' . $row['uid']]['rels']);
348 } else {
349 $this->error('Record ' . $table . ':' . $row['uid'] . ' was larger than maxRecordSize (' . GeneralUtility::formatSize($this->maxRecordSize) . ')');
350 }
351 } else {
352 $this->error('Record ' . $table . ':' . $row['uid'] . ' already added.');
353 }
354 } else {
355 $this->error('Record ' . $table . ':' . $row['uid'] . ' was outside your DB mounts!');
356 }
357 }
358 }
359
360 /**
361 * This changes the file reference ID from a hash based on the absolute file path
362 * (coming from ReferenceIndex) to a hash based on the relative file path.
363 *
364 * @param array $relations
365 * @return array
366 */
367 protected function fixFileIDsInRelations(array $relations)
368 {
369 foreach ($relations as $field => $relation) {
370 if (isset($relation['type']) && $relation['type'] === 'file') {
371 foreach ($relation['newValueFiles'] as $key => $fileRelationData) {
372 $absoluteFilePath = $fileRelationData['ID_absFile'];
373 if (GeneralUtility::isFirstPartOfStr($absoluteFilePath, PATH_site)) {
374 $relatedFilePath = PathUtility::stripPathSitePrefix($absoluteFilePath);
375 $relations[$field]['newValueFiles'][$key]['ID'] = md5($relatedFilePath);
376 }
377 }
378 }
379 if ($relation['type'] === 'flex') {
380 if (is_array($relation['flexFormRels']['file'])) {
381 foreach ($relation['flexFormRels']['file'] as $key => $subList) {
382 foreach ($subList as $subKey => $fileRelationData) {
383 $absoluteFilePath = $fileRelationData['ID_absFile'];
384 if (GeneralUtility::isFirstPartOfStr($absoluteFilePath, PATH_site)) {
385 $relatedFilePath = PathUtility::stripPathSitePrefix($absoluteFilePath);
386 $relations[$field]['flexFormRels']['file'][$key][$subKey]['ID'] = md5($relatedFilePath);
387 }
388 }
389 }
390 }
391 }
392 }
393 return $relations;
394 }
395
396 /**
397 * Relations could contain db relations to sys_file records. Some configuration combinations of TCA and
398 * SoftReferenceIndex create also softref relation entries for the identical file. This results
399 * in double included files, one in array "files" and one in array "file_fal".
400 * This function checks the relations for this double inclusions and removes the redundant softref relation.
401 *
402 * @param array $relations
403 * @return array
404 */
405 protected function removeSoftrefsHavingTheSameDatabaseRelation($relations)
406 {
407 $fixedRelations = array();
408 foreach ($relations as $field => $relation) {
409 $newRelation = $relation;
410 if (isset($newRelation['type']) && $newRelation['type'] === 'db') {
411 foreach ($newRelation['itemArray'] as $key => $dbRelationData) {
412 if ($dbRelationData['table'] === 'sys_file') {
413 if (isset($newRelation['softrefs']['keys']['typolink'])) {
414 foreach ($newRelation['softrefs']['keys']['typolink'] as $softrefKey => $softRefData) {
415 if ($softRefData['subst']['type'] === 'file') {
416 $file = ResourceFactory::getInstance()->retrieveFileOrFolderObject($softRefData['subst']['relFileName']);
417 if ($file instanceof File) {
418 if ($file->getUid() == $dbRelationData['id']) {
419 unset($newRelation['softrefs']['keys']['typolink'][$softrefKey]);
420 }
421 }
422 }
423 }
424 if (empty($newRelation['softrefs']['keys']['typolink'])) {
425 unset($newRelation['softrefs']);
426 }
427 }
428 }
429 }
430 }
431 $fixedRelations[$field] = $newRelation;
432 }
433 return $fixedRelations;
434 }
435
436 /**
437 * This analyses the existing added records, finds all database relations to records and adds these records to the export file.
438 * This function can be called repeatedly until it returns an empty array.
439 * In principle it should not allow to infinite recursivity, but you better set a limit...
440 * Call this BEFORE the ext_addFilesFromRelations (so files from added relations are also included of course)
441 *
442 * @param int $relationLevel Recursion level
443 * @return array overview of relations found and added: Keys [table]:[uid], values array with table and id
444 * @see export_addFilesFromRelations()
445 */
446 public function export_addDBRelations($relationLevel = 0)
447 {
448 // Traverse all "rels" registered for "records"
449 if (!is_array($this->dat['records'])) {
450 $this->error('There were no records available.');
451 return array();
452 }
453 $addR = array();
454 foreach ($this->dat['records'] as $k => $value) {
455 if (!is_array($this->dat['records'][$k])) {
456 continue;
457 }
458 foreach ($this->dat['records'][$k]['rels'] as $fieldname => $vR) {
459 // For all DB types of relations:
460 if ($vR['type'] == 'db') {
461 foreach ($vR['itemArray'] as $fI) {
462 $this->export_addDBRelations_registerRelation($fI, $addR);
463 }
464 }
465 // For all flex/db types of relations:
466 if ($vR['type'] == 'flex') {
467 // DB relations in flex form fields:
468 if (is_array($vR['flexFormRels']['db'])) {
469 foreach ($vR['flexFormRels']['db'] as $subList) {
470 foreach ($subList as $fI) {
471 $this->export_addDBRelations_registerRelation($fI, $addR);
472 }
473 }
474 }
475 // DB oriented soft references in flex form fields:
476 if (is_array($vR['flexFormRels']['softrefs'])) {
477 foreach ($vR['flexFormRels']['softrefs'] as $subList) {
478 foreach ($subList['keys'] as $spKey => $elements) {
479 foreach ($elements as $el) {
480 if ($el['subst']['type'] === 'db' && $this->includeSoftref($el['subst']['tokenID'])) {
481 list($tempTable, $tempUid) = explode(':', $el['subst']['recordRef']);
482 $fI = array(
483 'table' => $tempTable,
484 'id' => $tempUid
485 );
486 $this->export_addDBRelations_registerRelation($fI, $addR, $el['subst']['tokenID']);
487 }
488 }
489 }
490 }
491 }
492 }
493 // In any case, if there are soft refs:
494 if (is_array($vR['softrefs']['keys'])) {
495 foreach ($vR['softrefs']['keys'] as $spKey => $elements) {
496 foreach ($elements as $el) {
497 if ($el['subst']['type'] === 'db' && $this->includeSoftref($el['subst']['tokenID'])) {
498 list($tempTable, $tempUid) = explode(':', $el['subst']['recordRef']);
499 $fI = array(
500 'table' => $tempTable,
501 'id' => $tempUid
502 );
503 $this->export_addDBRelations_registerRelation($fI, $addR, $el['subst']['tokenID']);
504 }
505 }
506 }
507 }
508 }
509 }
510
511 // Now, if there were new records to add, do so:
512 if (!empty($addR)) {
513 foreach ($addR as $fI) {
514 // Get and set record:
515 $row = BackendUtility::getRecord($fI['table'], $fI['id']);
516 if (is_array($row)) {
517 $this->export_addRecord($fI['table'], $row, $relationLevel + 1);
518 }
519 // Set status message
520 // Relation pointers always larger than zero except certain "select" types with
521 // negative values pointing to uids - but that is not supported here.
522 if ($fI['id'] > 0) {
523 $rId = $fI['table'] . ':' . $fI['id'];
524 if (!isset($this->dat['records'][$rId])) {
525 $this->dat['records'][$rId] = 'NOT_FOUND';
526 $this->error('Relation record ' . $rId . ' was not found!');
527 }
528 }
529 }
530 }
531 // Return overview of relations found and added
532 return $addR;
533 }
534
535 /**
536 * Helper function for export_addDBRelations()
537 *
538 * @param array $fI Array with table/id keys to add
539 * @param array $addR Add array, passed by reference to be modified
540 * @param string $tokenID Softref Token ID, if applicable.
541 * @return void
542 * @see export_addDBRelations()
543 */
544 public function export_addDBRelations_registerRelation($fI, &$addR, $tokenID = '')
545 {
546 $rId = $fI['table'] . ':' . $fI['id'];
547 if (
548 isset($GLOBALS['TCA'][$fI['table']]) && !$this->isTableStatic($fI['table']) && !$this->isExcluded($fI['table'], $fI['id'])
549 && (!$tokenID || $this->includeSoftref($tokenID)) && $this->inclRelation($fI['table'])
550 ) {
551 if (!isset($this->dat['records'][$rId])) {
552 // Set this record to be included since it is not already.
553 $addR[$rId] = $fI;
554 }
555 }
556 }
557
558 /**
559 * This adds all files in relations.
560 * Call this method AFTER adding all records including relations.
561 *
562 * @return void
563 * @see export_addDBRelations()
564 */
565 public function export_addFilesFromRelations()
566 {
567 // Traverse all "rels" registered for "records"
568 if (!is_array($this->dat['records'])) {
569 $this->error('There were no records available.');
570 return;
571 }
572 foreach ($this->dat['records'] as $k => $value) {
573 if (!isset($this->dat['records'][$k]['rels']) || !is_array($this->dat['records'][$k]['rels'])) {
574 continue;
575 }
576 foreach ($this->dat['records'][$k]['rels'] as $fieldname => $vR) {
577 // For all file type relations:
578 if ($vR['type'] == 'file') {
579 foreach ($vR['newValueFiles'] as $key => $fI) {
580 $this->export_addFile($fI, $k, $fieldname);
581 // Remove the absolute reference to the file so it doesn't expose absolute paths from source server:
582 unset($this->dat['records'][$k]['rels'][$fieldname]['newValueFiles'][$key]['ID_absFile']);
583 }
584 }
585 // For all flex type relations:
586 if ($vR['type'] == 'flex') {
587 if (is_array($vR['flexFormRels']['file'])) {
588 foreach ($vR['flexFormRels']['file'] as $key => $subList) {
589 foreach ($subList as $subKey => $fI) {
590 $this->export_addFile($fI, $k, $fieldname);
591 // Remove the absolute reference to the file so it doesn't expose absolute paths from source server:
592 unset($this->dat['records'][$k]['rels'][$fieldname]['flexFormRels']['file'][$key][$subKey]['ID_absFile']);
593 }
594 }
595 }
596 // DB oriented soft references in flex form fields:
597 if (is_array($vR['flexFormRels']['softrefs'])) {
598 foreach ($vR['flexFormRels']['softrefs'] as $key => $subList) {
599 foreach ($subList['keys'] as $spKey => $elements) {
600 foreach ($elements as $subKey => $el) {
601 if ($el['subst']['type'] === 'file' && $this->includeSoftref($el['subst']['tokenID'])) {
602 // Create abs path and ID for file:
603 $ID_absFile = GeneralUtility::getFileAbsFileName(PATH_site . $el['subst']['relFileName']);
604 $ID = md5($el['subst']['relFileName']);
605 if ($ID_absFile) {
606 if (!$this->dat['files'][$ID]) {
607 $fI = array(
608 'filename' => PathUtility::basename($ID_absFile),
609 'ID_absFile' => $ID_absFile,
610 'ID' => $ID,
611 'relFileName' => $el['subst']['relFileName']
612 );
613 $this->export_addFile($fI, '_SOFTREF_');
614 }
615 $this->dat['records'][$k]['rels'][$fieldname]['flexFormRels']['softrefs'][$key]['keys'][$spKey][$subKey]['file_ID'] = $ID;
616 }
617 }
618 }
619 }
620 }
621 }
622 }
623 // In any case, if there are soft refs:
624 if (is_array($vR['softrefs']['keys'])) {
625 foreach ($vR['softrefs']['keys'] as $spKey => $elements) {
626 foreach ($elements as $subKey => $el) {
627 if ($el['subst']['type'] === 'file' && $this->includeSoftref($el['subst']['tokenID'])) {
628 // Create abs path and ID for file:
629 $ID_absFile = GeneralUtility::getFileAbsFileName(PATH_site . $el['subst']['relFileName']);
630 $ID = md5($el['subst']['relFileName']);
631 if ($ID_absFile) {
632 if (!$this->dat['files'][$ID]) {
633 $fI = array(
634 'filename' => PathUtility::basename($ID_absFile),
635 'ID_absFile' => $ID_absFile,
636 'ID' => $ID,
637 'relFileName' => $el['subst']['relFileName']
638 );
639 $this->export_addFile($fI, '_SOFTREF_');
640 }
641 $this->dat['records'][$k]['rels'][$fieldname]['softrefs']['keys'][$spKey][$subKey]['file_ID'] = $ID;
642 }
643 }
644 }
645 }
646 }
647 }
648 }
649 }
650
651 /**
652 * This adds all files from sys_file records
653 *
654 * @return void
655 */
656 public function export_addFilesFromSysFilesRecords()
657 {
658 if (!isset($this->dat['header']['records']['sys_file']) || !is_array($this->dat['header']['records']['sys_file'])) {
659 return;
660 }
661 foreach ($this->dat['header']['records']['sys_file'] as $sysFileUid => $_) {
662 $recordData = $this->dat['records']['sys_file:' . $sysFileUid]['data'];
663 $file = ResourceFactory::getInstance()->createFileObject($recordData);
664 $this->export_addSysFile($file);
665 }
666 }
667
668 /**
669 * Adds a files content from a sys file record to the export memory
670 *
671 * @param File $file
672 * @return void
673 */
674 public function export_addSysFile(File $file)
675 {
676 if ($file->getProperty('size') >= $this->maxFileSize) {
677 $this->error('File ' . $file->getPublicUrl() . ' was larger (' . GeneralUtility::formatSize($file->getProperty('size')) . ') than the maxFileSize (' . GeneralUtility::formatSize($this->maxFileSize) . ')! Skipping.');
678 return;
679 }
680 $fileContent = '';
681 try {
682 if (!$this->saveFilesOutsideExportFile) {
683 $fileContent = $file->getContents();
684 } else {
685 $file->checkActionPermission('read');
686 }
687 } catch (\Exception $e) {
688 $this->error('Error when trying to add file ' . $file->getCombinedIdentifier() . ': ' . $e->getMessage());
689 return;
690 }
691 $fileUid = $file->getUid();
692 $fileInfo = $file->getStorage()->getFileInfo($file);
693 // we sadly have to cast it to string here, because the size property is also returning a string
694 $fileSize = (string)$fileInfo['size'];
695 if ($fileSize !== $file->getProperty('size')) {
696 $this->error('File size of ' . $file->getCombinedIdentifier() . ' is not up-to-date in index! File added with current size.');
697 $this->dat['records']['sys_file:' . $fileUid]['data']['size'] = $fileSize;
698 }
699 $fileSha1 = $file->getStorage()->hashFile($file, 'sha1');
700 if ($fileSha1 !== $file->getProperty('sha1')) {
701 $this->error('File sha1 hash of ' . $file->getCombinedIdentifier() . ' is not up-to-date in index! File added on current sha1.');
702 $this->dat['records']['sys_file:' . $fileUid]['data']['sha1'] = $fileSha1;
703 }
704
705 $fileRec = array();
706 $fileRec['filesize'] = $fileSize;
707 $fileRec['filename'] = $file->getProperty('name');
708 $fileRec['filemtime'] = $file->getProperty('modification_date');
709
710 // build unique id based on the storage and the file identifier
711 $fileId = md5($file->getStorage()->getUid() . ':' . $file->getProperty('identifier_hash'));
712
713 // Setting this data in the header
714 $this->dat['header']['files_fal'][$fileId] = $fileRec;
715
716 if (!$this->saveFilesOutsideExportFile) {
717 // ... and finally add the heavy stuff:
718 $fileRec['content'] = $fileContent;
719 } else {
720 GeneralUtility::upload_copy_move($file->getForLocalProcessing(false), $this->getTemporaryFilesPathForExport() . $file->getProperty('sha1'));
721 }
722 $fileRec['content_sha1'] = $fileSha1;
723
724 $this->dat['files_fal'][$fileId] = $fileRec;
725 }
726
727 /**
728 * Adds a files content to the export memory
729 *
730 * @param array $fI File information with three keys: "filename" = filename without path, "ID_absFile" = absolute filepath to the file (including the filename), "ID" = md5 hash of "ID_absFile". "relFileName" is optional for files attached to records, but mandatory for soft referenced files (since the relFileName determines where such a file should be stored!)
731 * @param string $recordRef If the file is related to a record, this is the id on the form [table]:[id]. Information purposes only.
732 * @param string $fieldname If the file is related to a record, this is the field name it was related to. Information purposes only.
733 * @return void
734 */
735 public function export_addFile($fI, $recordRef = '', $fieldname = '')
736 {
737 if (!@is_file($fI['ID_absFile'])) {
738 $this->error($fI['ID_absFile'] . ' was not a file! Skipping.');
739 return;
740 }
741 if (filesize($fI['ID_absFile']) >= $this->maxFileSize) {
742 $this->error($fI['ID_absFile'] . ' was larger (' . GeneralUtility::formatSize(filesize($fI['ID_absFile'])) . ') than the maxFileSize (' . GeneralUtility::formatSize($this->maxFileSize) . ')! Skipping.');
743 return;
744 }
745 $fileInfo = stat($fI['ID_absFile']);
746 $fileRec = array();
747 $fileRec['filesize'] = $fileInfo['size'];
748 $fileRec['filename'] = PathUtility::basename($fI['ID_absFile']);
749 $fileRec['filemtime'] = $fileInfo['mtime'];
750 //for internal type file_reference
751 $fileRec['relFileRef'] = PathUtility::stripPathSitePrefix($fI['ID_absFile']);
752 if ($recordRef) {
753 $fileRec['record_ref'] = $recordRef . '/' . $fieldname;
754 }
755 if ($fI['relFileName']) {
756 $fileRec['relFileName'] = $fI['relFileName'];
757 }
758 // Setting this data in the header
759 $this->dat['header']['files'][$fI['ID']] = $fileRec;
760 // ... and for the recordlisting, why not let us know WHICH relations there was...
761 if ($recordRef && $recordRef !== '_SOFTREF_') {
762 $refParts = explode(':', $recordRef, 2);
763 if (!is_array($this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'])) {
764 $this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'] = array();
765 }
766 $this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'][] = $fI['ID'];
767 }
768 $fileMd5 = md5_file($fI['ID_absFile']);
769 if (!$this->saveFilesOutsideExportFile) {
770 // ... and finally add the heavy stuff:
771 $fileRec['content'] = file_get_contents($fI['ID_absFile']);
772 } else {
773 GeneralUtility::upload_copy_move($fI['ID_absFile'], $this->getTemporaryFilesPathForExport() . $fileMd5);
774 }
775 $fileRec['content_md5'] = $fileMd5;
776 $this->dat['files'][$fI['ID']] = $fileRec;
777 // For soft references, do further processing:
778 if ($recordRef === '_SOFTREF_') {
779 // RTE files?
780 if ($RTEoriginal = $this->getRTEoriginalFilename(PathUtility::basename($fI['ID_absFile']))) {
781 $RTEoriginal_absPath = PathUtility::dirname($fI['ID_absFile']) . '/' . $RTEoriginal;
782 if (@is_file($RTEoriginal_absPath)) {
783 $RTEoriginal_ID = md5($RTEoriginal_absPath);
784 $fileInfo = stat($RTEoriginal_absPath);
785 $fileRec = array();
786 $fileRec['filesize'] = $fileInfo['size'];
787 $fileRec['filename'] = PathUtility::basename($RTEoriginal_absPath);
788 $fileRec['filemtime'] = $fileInfo['mtime'];
789 $fileRec['record_ref'] = '_RTE_COPY_ID:' . $fI['ID'];
790 $this->dat['header']['files'][$fI['ID']]['RTE_ORIG_ID'] = $RTEoriginal_ID;
791 // Setting this data in the header
792 $this->dat['header']['files'][$RTEoriginal_ID] = $fileRec;
793 $fileMd5 = md5_file($RTEoriginal_absPath);
794 if (!$this->saveFilesOutsideExportFile) {
795 // ... and finally add the heavy stuff:
796 $fileRec['content'] = file_get_contents($RTEoriginal_absPath);
797 } else {
798 GeneralUtility::upload_copy_move($RTEoriginal_absPath, $this->getTemporaryFilesPathForExport() . $fileMd5);
799 }
800 $fileRec['content_md5'] = $fileMd5;
801 $this->dat['files'][$RTEoriginal_ID] = $fileRec;
802 } else {
803 $this->error('RTE original file "' . PathUtility::stripPathSitePrefix($RTEoriginal_absPath) . '" was not found!');
804 }
805 }
806 // Files with external media?
807 // This is only done with files grabbed by a softreference parser since it is deemed improbable that hard-referenced files should undergo this treatment.
808 $html_fI = pathinfo(PathUtility::basename($fI['ID_absFile']));
809 if ($this->includeExtFileResources && GeneralUtility::inList($this->extFileResourceExtensions, strtolower($html_fI['extension']))) {
810 $uniquePrefix = '###' . md5($GLOBALS['EXEC_TIME']) . '###';
811 if (strtolower($html_fI['extension']) === 'css') {
812 $prefixedMedias = explode($uniquePrefix, preg_replace('/(url[[:space:]]*\\([[:space:]]*["\']?)([^"\')]*)(["\']?[[:space:]]*\\))/i', '\\1' . $uniquePrefix . '\\2' . $uniquePrefix . '\\3', $fileRec['content']));
813 } else {
814 // html, htm:
815 $htmlParser = GeneralUtility::makeInstance(HtmlParser::class);
816 $prefixedMedias = explode($uniquePrefix, $htmlParser->prefixResourcePath($uniquePrefix, $fileRec['content'], array(), $uniquePrefix));
817 }
818 $htmlResourceCaptured = false;
819 foreach ($prefixedMedias as $k => $v) {
820 if ($k % 2) {
821 $EXTres_absPath = GeneralUtility::resolveBackPath(PathUtility::dirname($fI['ID_absFile']) . '/' . $v);
822 $EXTres_absPath = GeneralUtility::getFileAbsFileName($EXTres_absPath);
823 if ($EXTres_absPath && GeneralUtility::isFirstPartOfStr($EXTres_absPath, PATH_site . $this->fileadminFolderName . '/') && @is_file($EXTres_absPath)) {
824 $htmlResourceCaptured = true;
825 $EXTres_ID = md5($EXTres_absPath);
826 $this->dat['header']['files'][$fI['ID']]['EXT_RES_ID'][] = $EXTres_ID;
827 $prefixedMedias[$k] = '{EXT_RES_ID:' . $EXTres_ID . '}';
828 // Add file to memory if it is not set already:
829 if (!isset($this->dat['header']['files'][$EXTres_ID])) {
830 $fileInfo = stat($EXTres_absPath);
831 $fileRec = array();
832 $fileRec['filesize'] = $fileInfo['size'];
833 $fileRec['filename'] = PathUtility::basename($EXTres_absPath);
834 $fileRec['filemtime'] = $fileInfo['mtime'];
835 $fileRec['record_ref'] = '_EXT_PARENT_:' . $fI['ID'];
836 // Media relative to the HTML file.
837 $fileRec['parentRelFileName'] = $v;
838 // Setting this data in the header
839 $this->dat['header']['files'][$EXTres_ID] = $fileRec;
840 // ... and finally add the heavy stuff:
841 $fileRec['content'] = file_get_contents($EXTres_absPath);
842 $fileRec['content_md5'] = md5($fileRec['content']);
843 $this->dat['files'][$EXTres_ID] = $fileRec;
844 }
845 }
846 }
847 }
848 if ($htmlResourceCaptured) {
849 $this->dat['files'][$fI['ID']]['tokenizedContent'] = implode('', $prefixedMedias);
850 }
851 }
852 }
853 }
854
855 /**
856 * If saveFilesOutsideExportFile is enabled, this function returns the path
857 * where the files referenced in the export are copied to.
858 *
859 * @return string
860 * @throws \RuntimeException
861 * @see setSaveFilesOutsideExportFile()
862 */
863 public function getTemporaryFilesPathForExport()
864 {
865 if (!$this->saveFilesOutsideExportFile) {
866 throw new \RuntimeException('You need to set saveFilesOutsideExportFile to TRUE before you want to get the temporary files path for export.', 1401205213);
867 }
868 if ($this->temporaryFilesPathForExport === null) {
869 $temporaryFolderName = $this->getTemporaryFolderName();
870 $this->temporaryFilesPathForExport = $temporaryFolderName . '/';
871 }
872 return $this->temporaryFilesPathForExport;
873 }
874
875 /**
876 * DB relations flattend to 1-dim array.
877 * The list will be unique, no table/uid combination will appear twice.
878 *
879 * @param array $dbrels 2-dim Array of database relations organized by table key
880 * @return array 1-dim array where entries are table:uid and keys are array with table/id
881 */
882 public function flatDBrels($dbrels)
883 {
884 $list = array();
885 foreach ($dbrels as $dat) {
886 if ($dat['type'] == 'db') {
887 foreach ($dat['itemArray'] as $i) {
888 $list[$i['table'] . ':' . $i['id']] = $i;
889 }
890 }
891 if ($dat['type'] == 'flex' && is_array($dat['flexFormRels']['db'])) {
892 foreach ($dat['flexFormRels']['db'] as $subList) {
893 foreach ($subList as $i) {
894 $list[$i['table'] . ':' . $i['id']] = $i;
895 }
896 }
897 }
898 }
899 return $list;
900 }
901
902 /**
903 * Soft References flattend to 1-dim array.
904 *
905 * @param array $dbrels 2-dim Array of database relations organized by table key
906 * @return array 1-dim array where entries are arrays with properties of the soft link found and keys are a unique combination of field, spKey, structure path if applicable and token ID
907 */
908 public function flatSoftRefs($dbrels)
909 {
910 $list = array();
911 foreach ($dbrels as $field => $dat) {
912 if (is_array($dat['softrefs']['keys'])) {
913 foreach ($dat['softrefs']['keys'] as $spKey => $elements) {
914 if (is_array($elements)) {
915 foreach ($elements as $subKey => $el) {
916 $lKey = $field . ':' . $spKey . ':' . $subKey;
917 $list[$lKey] = array_merge(array('field' => $field, 'spKey' => $spKey), $el);
918 // Add file_ID key to header - slightly "risky" way of doing this because if the calculation
919 // changes for the same value in $this->records[...] this will not work anymore!
920 if ($el['subst'] && $el['subst']['relFileName']) {
921 $list[$lKey]['file_ID'] = md5(PATH_site . $el['subst']['relFileName']);
922 }
923 }
924 }
925 }
926 }
927 if ($dat['type'] == 'flex' && is_array($dat['flexFormRels']['softrefs'])) {
928 foreach ($dat['flexFormRels']['softrefs'] as $structurePath => $subSoftrefs) {
929 if (is_array($subSoftrefs['keys'])) {
930 foreach ($subSoftrefs['keys'] as $spKey => $elements) {
931 foreach ($elements as $subKey => $el) {
932 $lKey = $field . ':' . $structurePath . ':' . $spKey . ':' . $subKey;
933 $list[$lKey] = array_merge(array('field' => $field, 'spKey' => $spKey, 'structurePath' => $structurePath), $el);
934 // Add file_ID key to header - slightly "risky" way of doing this because if the calculation
935 // changes for the same value in $this->records[...] this will not work anymore!
936 if ($el['subst'] && $el['subst']['relFileName']) {
937 $list[$lKey]['file_ID'] = md5(PATH_site . $el['subst']['relFileName']);
938 }
939 }
940 }
941 }
942 }
943 }
944 }
945 return $list;
946 }
947
948 /**
949 * If include fields for a specific record type are set, the data
950 * are filtered out with fields are not included in the fields.
951 *
952 * @param string $table The record type to be filtered
953 * @param array $row The data to be filtered
954 * @return array The filtered record row
955 */
956 protected function filterRecordFields($table, array $row)
957 {
958 if (isset($this->recordTypesIncludeFields[$table])) {
959 $includeFields = array_unique(array_merge(
960 $this->recordTypesIncludeFields[$table],
961 $this->defaultRecordIncludeFields
962 ));
963 $newRow = array();
964 foreach ($row as $key => $value) {
965 if (in_array($key, $includeFields)) {
966 $newRow[$key] = $value;
967 }
968 }
969 } else {
970 $newRow = $row;
971 }
972 return $newRow;
973 }
974
975 /**************************
976 * File Output
977 *************************/
978
979 /**
980 * This compiles and returns the data content for an exported file
981 *
982 * @param string $type Type of output; "xml" gives xml, otherwise serialized array, possibly compressed.
983 * @return string The output file stream
984 */
985 public function compileMemoryToFileContent($type = '')
986 {
987 if ($type == 'xml') {
988 $out = $this->createXML();
989 } else {
990 $compress = $this->doOutputCompress();
991 $out = '';
992 // adding header:
993 $out .= $this->addFilePart(serialize($this->dat['header']), $compress);
994 // adding records:
995 $out .= $this->addFilePart(serialize($this->dat['records']), $compress);
996 // adding files:
997 $out .= $this->addFilePart(serialize($this->dat['files']), $compress);
998 // adding files_fal:
999 $out .= $this->addFilePart(serialize($this->dat['files_fal']), $compress);
1000 }
1001 return $out;
1002 }
1003
1004 /**
1005 * Creates XML string from input array
1006 *
1007 * @return string XML content
1008 */
1009 public function createXML()
1010 {
1011 // Options:
1012 $options = array(
1013 'alt_options' => array(
1014 '/header' => array(
1015 'disableTypeAttrib' => true,
1016 'clearStackPath' => true,
1017 'parentTagMap' => array(
1018 'files' => 'file',
1019 'files_fal' => 'file',
1020 'records' => 'table',
1021 'table' => 'rec',
1022 'rec:rels' => 'relations',
1023 'relations' => 'element',
1024 'filerefs' => 'file',
1025 'pid_lookup' => 'page_contents',
1026 'header:relStaticTables' => 'static_tables',
1027 'static_tables' => 'tablename',
1028 'excludeMap' => 'item',
1029 'softrefCfg' => 'softrefExportMode',
1030 'extensionDependencies' => 'extkey',
1031 'softrefs' => 'softref_element'
1032 ),
1033 'alt_options' => array(
1034 '/pagetree' => array(
1035 'disableTypeAttrib' => true,
1036 'useIndexTagForNum' => 'node',
1037 'parentTagMap' => array(
1038 'node:subrow' => 'node'
1039 )
1040 ),
1041 '/pid_lookup/page_contents' => array(
1042 'disableTypeAttrib' => true,
1043 'parentTagMap' => array(
1044 'page_contents' => 'table'
1045 ),
1046 'grandParentTagMap' => array(
1047 'page_contents/table' => 'item'
1048 )
1049 )
1050 )
1051 ),
1052 '/records' => array(
1053 'disableTypeAttrib' => true,
1054 'parentTagMap' => array(
1055 'records' => 'tablerow',
1056 'tablerow:data' => 'fieldlist',
1057 'tablerow:rels' => 'related',
1058 'related' => 'field',
1059 'field:itemArray' => 'relations',
1060 'field:newValueFiles' => 'filerefs',
1061 'field:flexFormRels' => 'flexform',
1062 'relations' => 'element',
1063 'filerefs' => 'file',
1064 'flexform:db' => 'db_relations',
1065 'flexform:file' => 'file_relations',
1066 'flexform:softrefs' => 'softref_relations',
1067 'softref_relations' => 'structurePath',
1068 'db_relations' => 'path',
1069 'file_relations' => 'path',
1070 'path' => 'element',
1071 'keys' => 'softref_key',
1072 'softref_key' => 'softref_element'
1073 ),
1074 'alt_options' => array(
1075 '/records/tablerow/fieldlist' => array(
1076 'useIndexTagForAssoc' => 'field'
1077 )
1078 )
1079 ),
1080 '/files' => array(
1081 'disableTypeAttrib' => true,
1082 'parentTagMap' => array(
1083 'files' => 'file'
1084 )
1085 ),
1086 '/files_fal' => array(
1087 'disableTypeAttrib' => true,
1088 'parentTagMap' => array(
1089 'files_fal' => 'file'
1090 )
1091 )
1092 )
1093 );
1094 // Creating XML file from $outputArray:
1095 $charset = $this->dat['header']['charset'] ?: 'utf-8';
1096 $XML = '<?xml version="1.0" encoding="' . $charset . '" standalone="yes" ?>' . LF;
1097 $XML .= GeneralUtility::array2xml($this->dat, '', 0, 'T3RecordDocument', 0, $options);
1098 return $XML;
1099 }
1100
1101 /**
1102 * Returns TRUE if the output should be compressed.
1103 *
1104 * @return bool TRUE if compression is possible AND requested.
1105 */
1106 public function doOutputCompress()
1107 {
1108 return $this->compress && !$this->dontCompress;
1109 }
1110
1111 /**
1112 * Returns a content part for a filename being build.
1113 *
1114 * @param array $data Data to store in part
1115 * @param bool $compress Compress file?
1116 * @return string Content stream.
1117 */
1118 public function addFilePart($data, $compress = false)
1119 {
1120 if ($compress) {
1121 $data = gzcompress($data);
1122 }
1123 return md5($data) . ':' . ($compress ? '1' : '0') . ':' . str_pad(strlen($data), 10, '0', STR_PAD_LEFT) . ':' . $data . ':';
1124 }
1125 }