c6163d4647a48e8061c61a30a22c3e60ff719e1f
[Packages/TYPO3.CMS.git] / typo3 / sysext / lowlevel / Classes / Command / RteImagesCommand.php
1 <?php
2 declare(strict_types = 1);
3 namespace TYPO3\CMS\Lowlevel\Command;
4
5 /*
6 * This file is part of the TYPO3 CMS project.
7 *
8 * It is free software; you can redistribute it and/or modify it under
9 * the terms of the GNU General Public License, either version 2
10 * of the License, or any later version.
11 *
12 * For the full copyright and license information, please read the
13 * LICENSE.txt file that was distributed with this source code.
14 *
15 * The TYPO3 project - inspiring people to share!
16 */
17
18 use Symfony\Component\Console\Command\Command;
19 use Symfony\Component\Console\Input\InputInterface;
20 use Symfony\Component\Console\Input\InputOption;
21 use Symfony\Component\Console\Output\OutputInterface;
22 use Symfony\Component\Console\Style\SymfonyStyle;
23 use TYPO3\CMS\Core\Core\Bootstrap;
24 use TYPO3\CMS\Core\Database\ConnectionPool;
25 use TYPO3\CMS\Core\Database\ReferenceIndex;
26 use TYPO3\CMS\Core\Utility\File\BasicFileUtility;
27 use TYPO3\CMS\Core\Utility\GeneralUtility;
28 use TYPO3\CMS\Core\Utility\PathUtility;
29
30 /**
31 * Looking up all occurencies of RTEmagic images in the database and check existence of parent and
32 * copy files on the file system plus report possibly lost files of this type
33 */
34 class RteImagesCommand extends Command
35 {
36
37 /**
38 * Configure the command by defining the name, options and arguments
39 */
40 public function configure()
41 {
42 $this
43 ->setDescription('Looking up all occurrences of RTEmagic images in the database and check existence of parent and copy files on the file system plus report possibly lost RTE files.')
44 ->setHelp('
45 Assumptions:
46 - a perfect integrity of the reference index table (always update the reference index table before using this tool!)
47 - that all RTEmagic image files in the database are registered with the soft reference parser "images"
48 - images found in deleted records are included (means that you might find lost RTEmagic images after flushing deleted records)
49
50 The assumptions are not requirements by the TYPO3 API but reflects the de facto implementation of most TYPO3 installations.
51 However, many custom fields using an RTE will probably not have the "images" soft reference parser registered and so the index will be incomplete and not listing all RTEmagic image files.
52 The consequence of this limitation is that you should be careful if you wish to delete lost RTEmagic images - they could be referenced from a field not parsed by the "images" soft reference parser!
53
54 Automatic Repair of Errors:
55 - Will search for double-usages of RTEmagic images and make copies as required.
56 - Lost files can be deleted automatically, but it is recommended to delete them manually if you do not recognize them as used somewhere the system does not know about.
57
58 Manual repair suggestions:
59 - Missing files: Re-insert missing files or edit record where the reference is found.
60
61 If the option "--dry-run" is not set, the files are then deleted automatically.
62 Warning: First, make sure those files are not used somewhere TYPO3 does not know about! See the assumptions above.
63
64 If you want to get more detailed information, use the --verbose option.')
65 ->addOption(
66 'dry-run',
67 null,
68 InputOption::VALUE_NONE,
69 'If this option is set, the files will not actually be deleted, but just the output which files would be deleted are shown'
70 )
71 ->addOption(
72 'update-refindex',
73 null,
74 InputOption::VALUE_NONE,
75 'Setting this option automatically updates the reference index and does not ask on command line. Alternatively, use -n to avoid the interactive mode'
76 );
77 }
78
79 /**
80 * Executes the command to
81 * - optionally update the reference index (to have clean data)
82 * - find files within uploads/* which are not connected to the reference index
83 * - remove these files if --dry-run is not set
84 *
85 * @param InputInterface $input
86 * @param OutputInterface $output
87 */
88 protected function execute(InputInterface $input, OutputInterface $output)
89 {
90 // Make sure the _cli_ user is loaded
91 Bootstrap::initializeBackendAuthentication();
92
93 $io = new SymfonyStyle($input, $output);
94 $io->title($this->getDescription());
95
96 $dryRun = $input->hasOption('dry-run') && $input->getOption('dry-run') != false ? true : false;
97
98 $this->updateReferenceIndex($input, $io);
99
100 // Find the RTE files
101 $allRteImagesInUse = $this->findAllReferencedRteImagesWithOriginals();
102
103 if (count($allRteImagesInUse)) {
104 $allRteImagesWithOriginals = [];
105 $multipleReferenced = [];
106 $missingFiles = [];
107 $lostFiles = [];
108
109 // Searching for duplicates, and missing files (also missing originals)
110 foreach ($allRteImagesInUse as $fileName => $fileInfo) {
111 $allRteImagesWithOriginals[$fileName]++;
112 $allRteImagesWithOriginals[$fileInfo['original']]++;
113 if ($fileInfo['count'] > 1 && $fileInfo['exists'] && $fileInfo['original_exists']) {
114 $multipleReferenced[$fileName] = $fileInfo['softReferences'];
115 }
116 // Missing files:
117 if (!$fileInfo['exists']) {
118 $missingFiles[$fileName] = $fileInfo['softReferences'];
119 }
120 if (!$fileInfo['original_exists']) {
121 $missingFiles[$fileInfo['original']] = $fileInfo['softReferences'];
122 }
123 }
124
125 // Now, ask for RTEmagic files inside uploads/ folder:
126 $magicFiles = $this->findAllRteFilesInDirectory();
127 foreach ($magicFiles as $fileName) {
128 if (!isset($allRteImagesWithOriginals[$fileName])) {
129 $lostFiles[$fileName] = $fileName;
130 }
131 }
132 ksort($missingFiles);
133 ksort($multipleReferenced);
134
135 // Output info about missing files
136 if (!$io->isQuiet()) {
137 $io->note('Found ' . count($missingFiles) . ' RTE images that are referenced, but missing.');
138 if ($io->isVerbose()) {
139 $io->listing($missingFiles);
140 }
141 }
142
143 // Duplicate RTEmagic image files
144 // These files are RTEmagic images found used in multiple records! RTEmagic images should be used by only
145 // one record at a time. A large amount of such images probably stems from previous versions of TYPO3 (before 4.2)
146 // which did not support making copies automatically of RTEmagic images in case of new copies / versions.
147 $this->copyMultipleReferencedRteImages($multipleReferenced, $dryRun, $io);
148
149 // Delete lost files
150 // Lost RTEmagic files from uploads/
151 // These files you might be able to delete but only if _all_ RTEmagic images are found by the soft reference parser.
152 // If you are using the RTE in third-party extensions it is likely that the soft reference parser is not applied
153 // correctly to their RTE and thus these "lost" files actually represent valid RTEmagic images,
154 // just not registered. Lost files can be auto-fixed but only if you specifically
155 // set "lostFiles" as parameter to the --AUTOFIX option.
156 if (count($lostFiles)) {
157 ksort($lostFiles);
158 $this->deleteLostFiles($lostFiles, $dryRun, $io);
159 $io->success('Deleted ' . count($lostFiles) . ' lost files.');
160 }
161 } else {
162 $io->success('Nothing to do, your system does not have any RTE images.');
163 }
164 }
165
166 /**
167 * Function to update the reference index
168 * - if the option --update-refindex is set, do it
169 * - otherwise, if in interactive mode (not having -n set), ask the user
170 * - otherwise assume everything is fine
171 *
172 * @param InputInterface $input holds information about entered parameters
173 * @param SymfonyStyle $io necessary for outputting information
174 */
175 protected function updateReferenceIndex(InputInterface $input, SymfonyStyle $io)
176 {
177 // Check for reference index to update
178 $io->note('Finding RTE images used in TYPO3 requires a clean reference index (sys_refindex)');
179 $updateReferenceIndex = false;
180 if ($input->hasOption('update-refindex') && $input->getOption('update-refindex')) {
181 $updateReferenceIndex = true;
182 } elseif ($input->isInteractive()) {
183 $updateReferenceIndex = $io->confirm('Should the reference index be updated right now?', false);
184 }
185
186 // Update the reference index
187 if ($updateReferenceIndex) {
188 $referenceIndex = GeneralUtility::makeInstance(ReferenceIndex::class);
189 $referenceIndex->updateIndex(false, !$io->isQuiet());
190 } else {
191 $io->writeln('Reference index is assumed to be up to date, continuing.');
192 }
193 }
194
195 /**
196 * Find lost files in uploads/ folder
197 *
198 * @return array an array of files (relative to PATH_site) that are not connected
199 */
200 protected function findAllReferencedRteImagesWithOriginals(): array
201 {
202 $allRteImagesInUse = [];
203
204 // Select all RTEmagic files in the reference table (only from soft references of course)
205 $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
206 ->getQueryBuilderForTable('sys_refindex');
207
208 $result = $queryBuilder
209 ->select('*')
210 ->from('sys_refindex')
211 ->where(
212 $queryBuilder->expr()->eq(
213 'ref_table',
214 $queryBuilder->createNamedParameter('_FILE', \PDO::PARAM_STR)
215 ),
216 $queryBuilder->expr()->like(
217 'ref_string',
218 $queryBuilder->createNamedParameter('%/RTEmagic%', \PDO::PARAM_STR)
219 ),
220 $queryBuilder->expr()->eq(
221 'softref_key',
222 $queryBuilder->createNamedParameter('images', \PDO::PARAM_STR)
223 )
224 )
225 ->execute();
226
227 // Traverse the files and put into a large table:
228 while ($rec = $result->fetch()) {
229 $file = $rec['ref_string'];
230 $filename = PathUtility::basenameDuringBootstrap($file);
231 if (strpos($filename, 'RTEmagicC_') === 0) {
232 // First time the file is referenced => build index
233 if (!is_array($allRteImagesInUse[$file])) {
234 $original = 'RTEmagicP_' . preg_replace('/\\.[[:alnum:]]+$/', '', substr($filename, 10));
235 $original = substr($file, 0, -strlen($filename)) . $original;
236 $allRteImagesInUse[$file] = [
237 'exists' => @is_file(PATH_site . $file),
238 'original' => $original,
239 'original_exists' => @is_file(PATH_site . $original),
240 'count' => 0,
241 'softReferences' => []
242 ];
243 }
244 $allRteImagesInUse[$file]['count']++;
245 $allRteImagesInUse[$file]['softReferences'][$rec['hash']] = $this->formatReferenceIndexEntryToString($rec);
246 }
247 }
248
249 ksort($allRteImagesInUse);
250 return $allRteImagesInUse;
251 }
252
253 /**
254 * Find all RTE files in uploads/ folder
255 *
256 * @param string $folder the name of the folder to start from
257 * @return array an array of files (relative to PATH_site) that are not connected
258 */
259 protected function findAllRteFilesInDirectory($folder = 'uploads/'): array
260 {
261 $filesFound = [];
262
263 // Get all files
264 $files = [];
265 $files = GeneralUtility::getAllFilesAndFoldersInPath($files, PATH_site . $folder);
266 $files = GeneralUtility::removePrefixPathFromList($files, PATH_site);
267
268 // Traverse files
269 foreach ($files as $key => $value) {
270 // If the file is a RTEmagic-image name
271 if (preg_match('/^RTEmagic[P|C]_/', PathUtility::basenameDuringBootstrap($value))) {
272 $filesFound[] = $value;
273 continue;
274 }
275 }
276
277 return $filesFound;
278 }
279
280 /**
281 * Removes given files from the uploads/ folder
282 *
283 * @param array $lostFiles Contains the lost files found
284 * @param bool $dryRun if set, the files are just displayed, but not deleted
285 * @param SymfonyStyle $io the IO object for output
286 */
287 protected function deleteLostFiles(array $lostFiles, bool $dryRun, SymfonyStyle $io)
288 {
289 foreach ($lostFiles as $lostFile) {
290 $absoluteFileName = GeneralUtility::getFileAbsFileName($lostFile);
291 if ($io->isVeryVerbose()) {
292 $io->writeln('Deleting file "' . $absoluteFileName . '"');
293 }
294 if (!$dryRun) {
295 if ($absoluteFileName && @is_file($absoluteFileName)) {
296 unlink($absoluteFileName);
297 if (!$io->isQuiet()) {
298 $io->writeln('Permanently deleted file "' . $absoluteFileName . '".');
299 }
300 } else {
301 $io->error('File "' . $absoluteFileName . '" was not found!');
302 }
303 }
304 }
305 }
306
307 /**
308 * Duplicate RTEmagic image files which are used on several records. RTEmagic images should be used by only
309 * one record at a time. A large amount of such images probably stems from previous versions of TYPO3 (before 4.2)
310 * which did not support making copies automatically of RTEmagic images in case of new copies / versions.
311 *
312 * @param array $multipleReferencedImages
313 * @param bool $dryRun
314 * @param SymfonyStyle $io
315 */
316 protected function copyMultipleReferencedRteImages(array $multipleReferencedImages, bool $dryRun, SymfonyStyle $io)
317 {
318 $fileProcObj = GeneralUtility::makeInstance(BasicFileUtility::class);
319 foreach ($multipleReferencedImages as $fileName => $fileInfo) {
320 // Traverse all records using the file
321 $c = 0;
322 foreach ($fileInfo['usedIn'] as $hash => $recordID) {
323 if ($c === 0) {
324 $io->writeln('Keeping file ' . $fileName . ' for record ' . $recordID);
325 } else {
326 $io->writeln('Copying file ' . PathUtility::basenameDuringBootstrap($fileName) . ' for record ' . $recordID);
327 // Get directory prefix for file and set the original name
328 $dirPrefix = PathUtility::dirnameDuringBootstrap($fileName) . '/';
329 $rteOrigName = PathUtility::basenameDuringBootstrap($fileInfo['original']);
330 // If filename looks like an RTE file, and the directory is in "uploads/", then process as a RTE file!
331 if ($rteOrigName && strpos($dirPrefix, 'uploads/') === 0 && @is_dir(PATH_site . $dirPrefix)) {
332 // From the "original" RTE filename, produce a new "original" destination filename which is unused.
333 $origDestName = $fileProcObj->getUniqueName($rteOrigName, PATH_site . $dirPrefix);
334 // Create copy file name
335 $pI = pathinfo($fileName);
336 $copyDestName = PathUtility::dirnameDuringBootstrap($origDestName) . '/RTEmagicC_' . substr(PathUtility::basenameDuringBootstrap($origDestName), 10) . '.' . $pI['extension'];
337 if (!@is_file($copyDestName) && !@is_file($origDestName) && $origDestName === GeneralUtility::getFileAbsFileName($origDestName) && $copyDestName === GeneralUtility::getFileAbsFileName($copyDestName)) {
338 $io->writeln('Copying file ' . PathUtility::basenameDuringBootstrap($fileName) . ' for record ' . $recordID . ' to ' . PathUtility::basenameDuringBootstrap($copyDestName));
339 if (!$dryRun) {
340 // Making copies
341 GeneralUtility::upload_copy_move(PATH_site . $fileInfo['original'], $origDestName);
342 GeneralUtility::upload_copy_move(PATH_site . $fileName, $copyDestName);
343 clearstatcache();
344 if (@is_file($copyDestName)) {
345 $referenceIndex = GeneralUtility::makeInstance(ReferenceIndex::class);
346 $error = $referenceIndex->setReferenceValue($hash, PathUtility::stripPathSitePrefix($copyDestName));
347 if ($error) {
348 $io->error('ReferenceIndex::setReferenceValue() reported "' . $error . '"');
349 }
350 } else {
351 $io->error('File "' . $copyDestName . '" could not be created.');
352 }
353 }
354 } else {
355 $io->error('Could not construct new unique names for file.');
356 }
357 } else {
358 $io->error('Maybe directory of file was not within "uploads/"?');
359 }
360 }
361 $c++;
362 }
363 }
364 }
365
366 /**
367 * Formats a sys_refindex entry to something readable
368 *
369 * @param array $record
370 * @return string
371 */
372 protected function formatReferenceIndexEntryToString(array $record): string
373 {
374 return $record['tablename']
375 . ':' . $record['recuid']
376 . ':' . $record['field']
377 . ($record['flexpointer'] ? ':' . $record['flexpointer'] : '')
378 . ($record['softref_key'] ? ':' . $record['softref_key'] . ' (Soft Reference) ' : '')
379 . ($record['deleted'] ? ' (DELETED)' : '');
380 }
381 }