a583292cc3f367e363142a4a6a0e07ff2ec324ad
[Packages/TYPO3.CMS.git] / typo3 / sysext / lowlevel / Classes / DoubleFilesCommand.php
1 <?php
2 namespace TYPO3\CMS\Lowlevel;
3
4 /*
5 * This file is part of the TYPO3 CMS project.
6 *
7 * It is free software; you can redistribute it and/or modify it under
8 * the terms of the GNU General Public License, either version 2
9 * of the License, or any later version.
10 *
11 * For the full copyright and license information, please read the
12 * LICENSE.txt file that was distributed with this source code.
13 *
14 * The TYPO3 project - inspiring people to share!
15 */
16
17 use TYPO3\CMS\Core\Database\ConnectionPool;
18 use TYPO3\CMS\Core\Database\ReferenceIndex;
19 use TYPO3\CMS\Core\Utility\File\BasicFileUtility;
20 use TYPO3\CMS\Core\Utility\GeneralUtility;
21
22 /**
23 * Looking for double files
24 */
25 class DoubleFilesCommand extends CleanerCommand
26 {
27 /**
28 * @var bool
29 */
30 public $checkRefIndex = true;
31
32 /**
33 * Constructor
34 */
35 public function __construct()
36 {
37 parent::__construct();
38 // Setting up help:
39 $this->cli_help['name'] = 'double_files -- Looking for files from TYPO3 managed records which are referenced more than one time (only one time allowed)';
40 $this->cli_help['description'] = trim('
41 Assumptions:
42 - a perfect integrity of the reference index table (always update the reference index table before using this tool!)
43 - files found in deleted records are included (otherwise you would see a false list of lost files)
44
45 Files attached to records in TYPO3 using a "group" type configuration in TCA or FlexForm DataStructure are managed exclusively by the system and there must always exist a 1-1 reference between the file and the reference in the record.
46 This tool will expose when such files are referenced from multiple locations which is considered an integrity error.
47 If a multi-reference is found it was typically created because the record was copied or modified outside of DataHandler which will otherwise maintain the relations correctly.
48 Multi-references should be resolved to 1-1 references as soon as possible. The danger of keeping multi-references is that if the file is removed from one of the refering records it will actually be deleted in the file system, leaving missing files for the remaining referers!
49
50 Automatic Repair of Errors:
51 - The multi-referenced file is copied under a new name and references updated.
52
53 Manual repair suggestions:
54 - None that can not be handled by the automatic repair.
55 ');
56 $this->cli_help['examples'] = '/.../cli_dispatch.phpsh lowlevel_cleaner double_files -s -r
57 This will check the system for double files relations.';
58 }
59
60 /**
61 * Find managed files which are referred to more than one time
62 * Fix methods: API in \TYPO3\CMS\Core\Database\ReferenceIndex that allows to
63 * change the value of a reference (we could copy the file) or remove reference
64 *
65 * @return array
66 */
67 public function main()
68 {
69 // Initialize result array:
70 $resultArray = [
71 'message' => $this->cli_help['name'] . LF . LF . $this->cli_help['description'],
72 'headers' => [
73 'multipleReferencesList_count' => ['Number of multi-reference files', '(See below)', 0],
74 'singleReferencesList_count' => ['Number of files correctly referenced', 'The amount of correct 1-1 references', 0],
75 'multipleReferencesList' => ['Entries with files having multiple references', 'These are serious problems that should be resolved ASAP to prevent data loss! ' . $this->label_infoString, 3],
76 'dirname_registry' => ['Registry of directories in which files are found.', 'Registry includes which table/field pairs store files in them plus how many files their store.', 0],
77 'missingFiles' => ['Tracking missing files', '(Extra feature, not related to tracking of double references. Further, the list may include more files than found in the missing_files()-test because this list includes missing files from deleted records.)', 0],
78 'warnings' => ['Warnings picked up', '', 2]
79 ],
80 'multipleReferencesList_count' => ['count' => 0],
81 'singleReferencesList_count' => ['count' => 0],
82 'multipleReferencesList' => [],
83 'dirname_registry' => [],
84 'missingFiles' => [],
85 'warnings' => []
86 ];
87 // Select all files in the reference table not found by a soft reference parser (thus TCA configured)
88 $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
89 ->getQueryBuilderForTable('sys_refindex');
90
91 $result = $queryBuilder
92 ->select('*')
93 ->from('sys_refindex')
94 ->where(
95 $queryBuilder->expr()->eq('ref_table', $queryBuilder->createNamedParameter('_FILE', \PDO::PARAM_STR)),
96 $queryBuilder->expr()->eq('softref_key', $queryBuilder->createNamedParameter('', \PDO::PARAM_STR))
97 )
98 ->orderBy('sorting', 'DESC')
99 ->execute();
100
101 // Traverse the files and put into a large table:
102 $tempCount = [];
103 while ($rec = $result->fetch()) {
104 // Compile info string for location of reference:
105 $infoString = $this->infoStr($rec);
106 // Registering occurencies in directories:
107 $resultArray['dirname_registry'][dirname($rec['ref_string'])][$rec['tablename'] . ':' . $rec['field']]++;
108 // Handle missing file:
109 if (!@is_file((PATH_site . $rec['ref_string']))) {
110 $resultArray['missingFiles'][$rec['ref_string']][$rec['hash']] = $infoString;
111 ksort($resultArray['missingFiles'][$rec['ref_string']]);
112 }
113 // Add entry if file has multiple references pointing to it:
114 if (isset($tempCount[$rec['ref_string']])) {
115 if (!is_array($resultArray['multipleReferencesList'][$rec['ref_string']])) {
116 $resultArray['multipleReferencesList'][$rec['ref_string']] = [];
117 $resultArray['multipleReferencesList'][$rec['ref_string']][$tempCount[$rec['ref_string']][1]] = $tempCount[$rec['ref_string']][0];
118 }
119 $resultArray['multipleReferencesList'][$rec['ref_string']][$rec['hash']] = $infoString;
120 ksort($resultArray['multipleReferencesList'][$rec['ref_string']]);
121 } else {
122 $tempCount[$rec['ref_string']] = [$infoString, $rec['hash']];
123 }
124 }
125
126 ksort($resultArray['missingFiles']);
127 ksort($resultArray['multipleReferencesList']);
128 // Add count for multi-references:
129 $resultArray['multipleReferencesList_count']['count'] = count($resultArray['multipleReferencesList']);
130 $resultArray['singleReferencesList_count']['count'] = count($tempCount) - $resultArray['multipleReferencesList_count']['count'];
131 // Sort dirname registry and add warnings for directories outside uploads/
132 ksort($resultArray['dirname_registry']);
133 foreach ($resultArray['dirname_registry'] as $dir => $temp) {
134 ksort($resultArray['dirname_registry'][$dir]);
135 if (!GeneralUtility::isFirstPartOfStr($dir, 'uploads/')) {
136 $resultArray['warnings'][GeneralUtility::shortMD5($dir)] = 'Directory "' . $dir . '" was outside uploads/ which is unusual practice in TYPO3 although not forbidden. Directory used by the following table:field pairs: ' . implode(',', array_keys($temp));
137 }
138 }
139 return $resultArray;
140 }
141
142 /**
143 * Mandatory autofix function
144 * Will run auto-fix on the result array. Echos status during processing.
145 *
146 * @param array $resultArray Result array from main() function
147 * @return void
148 */
149 public function main_autoFix($resultArray)
150 {
151 foreach ($resultArray['multipleReferencesList'] as $key => $value) {
152 $absFileName = GeneralUtility::getFileAbsFileName($key);
153 if ($absFileName && @is_file($absFileName)) {
154 echo 'Processing file: ' . $key . LF;
155 $c = 0;
156 foreach ($value as $hash => $recReference) {
157 if ($c == 0) {
158 echo ' Keeping ' . $key . ' for record "' . $recReference . '"' . LF;
159 } else {
160 // Create unique name for file:
161 $fileFunc = GeneralUtility::makeInstance(BasicFileUtility::class);
162 $newName = $fileFunc->getUniqueName(basename($key), dirname($absFileName));
163 echo ' Copying ' . $key . ' to ' . \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($newName) . ' for record "' . $recReference . '": ';
164 if ($bypass = $this->cli_noExecutionCheck($recReference)) {
165 echo $bypass;
166 } else {
167 GeneralUtility::upload_copy_move($absFileName, $newName);
168 clearstatcache();
169 if (@is_file($newName)) {
170 $sysRefObj = GeneralUtility::makeInstance(ReferenceIndex::class);
171 $error = $sysRefObj->setReferenceValue($hash, basename($newName));
172 if ($error) {
173 echo ' ERROR: TYPO3\\CMS\\Core\\Database\\ReferenceIndex::setReferenceValue(): ' . $error . LF;
174 die;
175 } else {
176 echo 'DONE';
177 }
178 } else {
179 echo ' ERROR: File "' . $newName . '" was not created!';
180 }
181 }
182 echo LF;
183 }
184 $c++;
185 }
186 } else {
187 echo ' ERROR: File "' . $absFileName . '" was not found!';
188 }
189 }
190 }
191 }