[!!!][TASK] Migrate lowlevel rte_images command to Symfony Console
[Packages/TYPO3.CMS.git] / typo3 / sysext / lowlevel / Classes / Command / RteImagesCommand.php
1 <?php
2 declare(strict_types=1);
3 namespace TYPO3\CMS\Lowlevel\Command;
4
5 /*
6 * This file is part of the TYPO3 CMS project.
7 *
8 * It is free software; you can redistribute it and/or modify it under
9 * the terms of the GNU General Public License, either version 2
10 * of the License, or any later version.
11 *
12 * For the full copyright and license information, please read the
13 * LICENSE.txt file that was distributed with this source code.
14 *
15 * The TYPO3 project - inspiring people to share!
16 */
17
18 use Symfony\Component\Console\Command\Command;
19 use Symfony\Component\Console\Input\InputInterface;
20 use Symfony\Component\Console\Input\InputOption;
21 use Symfony\Component\Console\Output\OutputInterface;
22 use Symfony\Component\Console\Style\SymfonyStyle;
23 use TYPO3\CMS\Core\Database\ConnectionPool;
24 use TYPO3\CMS\Core\Database\ReferenceIndex;
25 use TYPO3\CMS\Core\Utility\File\BasicFileUtility;
26 use TYPO3\CMS\Core\Utility\GeneralUtility;
27 use TYPO3\CMS\Core\Utility\PathUtility;
28
29 /**
30 * Looking up all occurencies of RTEmagic images in the database and check existence of parent and
31 * copy files on the file system plus report possibly lost files of this type
32 */
33 class RteImagesCommand extends Command
34 {
35
36 /**
37 * Configure the command by defining the name, options and arguments
38 */
39 public function configure()
40 {
41 $this
42 ->setDescription('Looking up all occurrences of RTEmagic images in the database and check existence of parent and copy files on the file system plus report possibly lost RTE files.')
43 ->setHelp('
44 Assumptions:
45 - a perfect integrity of the reference index table (always update the reference index table before using this tool!)
46 - that all RTEmagic image files in the database are registered with the soft reference parser "images"
47 - images found in deleted records are included (means that you might find lost RTEmagic images after flushing deleted records)
48
49 The assumptions are not requirements by the TYPO3 API but reflects the de facto implementation of most TYPO3 installations.
50 However, many custom fields using an RTE will probably not have the "images" soft reference parser registered and so the index will be incomplete and not listing all RTEmagic image files.
51 The consequence of this limitation is that you should be careful if you wish to delete lost RTEmagic images - they could be referenced from a field not parsed by the "images" soft reference parser!
52
53 Automatic Repair of Errors:
54 - Will search for double-usages of RTEmagic images and make copies as required.
55 - Lost files can be deleted automatically, but it is recommended to delete them manually if you do not recognize them as used somewhere the system does not know about.
56
57 Manual repair suggestions:
58 - Missing files: Re-insert missing files or edit record where the reference is found.
59
60 If the option "--dry-run" is not set, the files are then deleted automatically.
61 Warning: First, make sure those files are not used somewhere TYPO3 does not know about! See the assumptions above.
62
63 If you want to get more detailed information, use the --verbose option.')
64 ->addOption(
65 'dry-run',
66 null,
67 InputOption::VALUE_NONE,
68 'If this option is set, the files will not actually be deleted, but just the output which files would be deleted are shown'
69 )
70 ->addOption(
71 'update-refindex',
72 null,
73 InputOption::VALUE_NONE,
74 'Setting this option automatically updates the reference index and does not ask on command line. Alternatively, use -n to avoid the interactive mode'
75 );
76 }
77
78 /**
79 * Executes the command to
80 * - optionally update the reference index (to have clean data)
81 * - find files within uploads/* which are not connected to the reference index
82 * - remove these files if --dry-run is not set
83 *
84 * @param InputInterface $input
85 * @param OutputInterface $output
86 *
87 * @return void
88 */
89 protected function execute(InputInterface $input, OutputInterface $output)
90 {
91 $io = new SymfonyStyle($input, $output);
92 $io->title($this->getDescription());
93
94 $dryRun = $input->hasOption('dry-run') && $input->getOption('dry-run') != false ? true : false;
95
96 $this->updateReferenceIndex($input, $io);
97
98 // Find the RTE files
99 $allRteImagesInUse = $this->findAllReferencedRteImagesWithOriginals();
100
101 if (count($allRteImagesInUse)) {
102 $allRteImagesWithOriginals = [];
103 $multipleReferenced = [];
104 $missingFiles = [];
105 $lostFiles = [];
106
107 // Searching for duplicates, and missing files (also missing originals)
108 foreach ($allRteImagesInUse as $fileName => $fileInfo) {
109 $allRteImagesWithOriginals[$fileName]++;
110 $allRteImagesWithOriginals[$fileInfo['original']]++;
111 if ($fileInfo['count'] > 1 && $fileInfo['exists'] && $fileInfo['original_exists']) {
112 $multipleReferenced[$fileName] = $fileInfo['softReferences'];
113 }
114 // Missing files:
115 if (!$fileInfo['exists']) {
116 $missingFiles[$fileName] = $fileInfo['softReferences'];
117 }
118 if (!$fileInfo['original_exists']) {
119 $missingFiles[$fileInfo['original']] = $fileInfo['softReferences'];
120 }
121 }
122
123 // Now, ask for RTEmagic files inside uploads/ folder:
124 $magicFiles = $this->findAllRteFilesInDirectory();
125 foreach ($magicFiles as $fileName) {
126 if (!isset($allRteImagesWithOriginals[$fileName])) {
127 $lostFiles[$fileName] = $fileName;
128 }
129 }
130 ksort($missingFiles);
131 ksort($multipleReferenced);
132
133 // Output info about missing files
134 if (!$io->isQuiet()) {
135 $io->note('Found ' . count($missingFiles) . ' RTE images that are referenced, but missing.');
136 if ($io->isVerbose()) {
137 $io->listing($missingFiles);
138 }
139 }
140
141 // Duplicate RTEmagic image files
142 // These files are RTEmagic images found used in multiple records! RTEmagic images should be used by only
143 // one record at a time. A large amount of such images probably stems from previous versions of TYPO3 (before 4.2)
144 // which did not support making copies automatically of RTEmagic images in case of new copies / versions.
145 $this->copyMultipleReferencedRteImages($multipleReferenced, $dryRun, $io);
146
147 // Delete lost files
148 // Lost RTEmagic files from uploads/
149 // These files you might be able to delete but only if _all_ RTEmagic images are found by the soft reference parser.
150 // If you are using the RTE in third-party extensions it is likely that the soft reference parser is not applied
151 // correctly to their RTE and thus these "lost" files actually represent valid RTEmagic images,
152 // just not registered. Lost files can be auto-fixed but only if you specifically
153 // set "lostFiles" as parameter to the --AUTOFIX option.
154 if (count($lostFiles)) {
155 ksort($lostFiles);
156 $this->deleteLostFiles($lostFiles, $dryRun, $io);
157 $io->success('Deleted ' . count($lostFiles) . ' lost files.');
158 }
159 } else {
160 $io->success('Nothing to do, your system does not have any RTE images.');
161 }
162 }
163
164 /**
165 * Function to update the reference index
166 * - if the option --update-refindex is set, do it
167 * - otherwise, if in interactive mode (not having -n set), ask the user
168 * - otherwise assume everything is fine
169 *
170 * @param InputInterface $input holds information about entered parameters
171 * @param SymfonyStyle $io necessary for outputting information
172 * @return void
173 */
174 protected function updateReferenceIndex(InputInterface $input, SymfonyStyle $io)
175 {
176 // Check for reference index to update
177 $io->note('Finding RTE images used in TYPO3 requires a clean reference index (sys_refindex)');
178 $updateReferenceIndex = false;
179 if ($input->hasOption('update-refindex') && $input->getOption('update-refindex')) {
180 $updateReferenceIndex = true;
181 } elseif ($input->isInteractive()) {
182 $updateReferenceIndex = $io->confirm('Should the reference index be updated right now?', false);
183 }
184
185 // Update the reference index
186 if ($updateReferenceIndex) {
187 $referenceIndex = GeneralUtility::makeInstance(ReferenceIndex::class);
188 $referenceIndex->updateIndex(false, !$io->isQuiet());
189 } else {
190 $io->writeln('Reference index is assumed to be up to date, continuing.');
191 }
192 }
193
194 /**
195 * Find lost files in uploads/ folder
196 *
197 * @return array an array of files (relative to PATH_site) that are not connected
198 */
199 protected function findAllReferencedRteImagesWithOriginals(): array
200 {
201 $allRteImagesInUse = [];
202
203 // Select all RTEmagic files in the reference table (only from soft references of course)
204 $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
205 ->getQueryBuilderForTable('sys_refindex');
206
207 $result = $queryBuilder
208 ->select('*')
209 ->from('sys_refindex')
210 ->where(
211 $queryBuilder->expr()->eq(
212 'ref_table',
213 $queryBuilder->createNamedParameter('_FILE', \PDO::PARAM_STR)
214 ),
215 $queryBuilder->expr()->like(
216 'ref_string',
217 $queryBuilder->createNamedParameter('%/RTEmagic%', \PDO::PARAM_STR)
218 ),
219 $queryBuilder->expr()->eq(
220 'softref_key',
221 $queryBuilder->createNamedParameter('images', \PDO::PARAM_STR)
222 )
223 )
224 ->execute();
225
226 // Traverse the files and put into a large table:
227 while ($rec = $result->fetch()) {
228 $file = $rec['ref_string'];
229 $filename = basename($file);
230 if (strpos($filename, 'RTEmagicC_') === 0) {
231 // First time the file is referenced => build index
232 if (!is_array($allRteImagesInUse[$file])) {
233 $original = 'RTEmagicP_' . preg_replace('/\\.[[:alnum:]]+$/', '', substr($filename, 10));
234 $original = substr($file, 0, -strlen($filename)) . $original;
235 $allRteImagesInUse[$file] = [
236 'exists' => @is_file(PATH_site . $file),
237 'original' => $original,
238 'original_exists' => @is_file(PATH_site . $original),
239 'count' => 0,
240 'softReferences' => []
241 ];
242 }
243 $allRteImagesInUse[$file]['count']++;
244 $allRteImagesInUse[$file]['softReferences'][$rec['hash']] = $this->formatReferenceIndexEntryToString($rec);
245 }
246 }
247
248 ksort($allRteImagesInUse);
249 return $allRteImagesInUse;
250 }
251
252 /**
253 * Find all RTE files in uploads/ folder
254 *
255 * @param string $folder the name of the folder to start from
256 * @return array an array of files (relative to PATH_site) that are not connected
257 */
258 protected function findAllRteFilesInDirectory($folder = 'uploads/'): array
259 {
260 $filesFound = [];
261
262 // Get all files
263 $files = [];
264 $files = GeneralUtility::getAllFilesAndFoldersInPath($files, PATH_site . $folder);
265 $files = GeneralUtility::removePrefixPathFromList($files, PATH_site);
266
267 // Traverse files
268 foreach ($files as $key => $value) {
269 // If the file is a RTEmagic-image name
270 if (preg_match('/^RTEmagic[P|C]_/', basename($value))) {
271 $filesFound[] = $value;
272 continue;
273 }
274 }
275
276 return $filesFound;
277 }
278
279 /**
280 * Removes given files from the uploads/ folder
281 *
282 * @param array $lostFiles Contains the lost files found
283 * @param bool $dryRun if set, the files are just displayed, but not deleted
284 * @param SymfonyStyle $io the IO object for output
285 * @return void
286 */
287 protected function deleteLostFiles(array $lostFiles, bool $dryRun, SymfonyStyle $io)
288 {
289 foreach ($lostFiles as $lostFile) {
290 $absoluteFileName = GeneralUtility::getFileAbsFileName($lostFile);
291 if ($io->isVeryVerbose()) {
292 $io->writeln('Deleting file "' . $absoluteFileName . '"');
293 }
294 if (!$dryRun) {
295 if ($absoluteFileName && @is_file($absoluteFileName)) {
296 unlink($absoluteFileName);
297 if (!$io->isQuiet()) {
298 $io->writeln('Permanently deleted file "' . $absoluteFileName . '".');
299 }
300 } else {
301 $io->error('File "' . $absoluteFileName . '" was not found!');
302 }
303 }
304 }
305 }
306
307 /**
308 * Duplicate RTEmagic image files which are used on several records. RTEmagic images should be used by only
309 * one record at a time. A large amount of such images probably stems from previous versions of TYPO3 (before 4.2)
310 * which did not support making copies automatically of RTEmagic images in case of new copies / versions.
311 *
312 * @param array $multipleReferencedImages
313 * @param bool $dryRun
314 * @param SymfonyStyle $io
315 */
316 protected function copyMultipleReferencedRteImages(array $multipleReferencedImages, bool $dryRun, SymfonyStyle $io)
317 {
318 $fileProcObj = GeneralUtility::makeInstance(BasicFileUtility::class);
319 foreach ($multipleReferencedImages as $fileName => $fileInfo) {
320 // Traverse all records using the file
321 $c = 0;
322 foreach ($fileInfo['usedIn'] as $hash => $recordID) {
323 if ($c === 0) {
324 $io->writeln('Keeping file ' . $fileName . ' for record ' . $recordID);
325 } else {
326 $io->writeln('Copying file ' . basename($fileName) . ' for record ' . $recordID);
327 // Get directory prefix for file and set the original name
328 $dirPrefix = dirname($fileName) . '/';
329 $rteOrigName = basename($fileInfo['original']);
330 // If filename looks like an RTE file, and the directory is in "uploads/", then process as a RTE file!
331 if ($rteOrigName && strpos($dirPrefix, 'uploads/') === 0 && @is_dir((PATH_site . $dirPrefix))) {
332 // From the "original" RTE filename, produce a new "original" destination filename which is unused.
333 $origDestName = $fileProcObj->getUniqueName($rteOrigName, PATH_site . $dirPrefix);
334 // Create copy file name
335 $pI = pathinfo($fileName);
336 $copyDestName = dirname($origDestName) . '/RTEmagicC_' . substr(basename($origDestName), 10) . '.' . $pI['extension'];
337 if (!@is_file($copyDestName) && !@is_file($origDestName) && $origDestName === GeneralUtility::getFileAbsFileName($origDestName) && $copyDestName === GeneralUtility::getFileAbsFileName($copyDestName)) {
338 $io->writeln('Copying file ' . basename($fileName) . ' for record ' . $recordID . ' to ' . basename($copyDestName));
339 if (!$dryRun) {
340 // Making copies
341 GeneralUtility::upload_copy_move(PATH_site . $fileInfo['original'], $origDestName);
342 GeneralUtility::upload_copy_move(PATH_site . $fileName, $copyDestName);
343 clearstatcache();
344 if (@is_file($copyDestName)) {
345 $referenceIndex = GeneralUtility::makeInstance(ReferenceIndex::class);
346 $error = $referenceIndex->setReferenceValue($hash, PathUtility::stripPathSitePrefix($copyDestName));
347 if ($error) {
348 $io->error('ReferenceIndex::setReferenceValue() reported "' . $error . '"');
349 }
350 } else {
351 $io->error('File "' . $copyDestName . '" could not be created.');
352 }
353 }
354 } else {
355 $io->error('Could not construct new unique names for file.');
356 }
357 } else {
358 $io->error('Maybe directory of file was not within "uploads/"?');
359 }
360 }
361 $c++;
362 }
363 }
364 }
365
366 /**
367 * Formats a sys_refindex entry to something readable
368 *
369 * @param array $record
370 * @return string
371 */
372 protected function formatReferenceIndexEntryToString(array $record): string
373 {
374 return $record['tablename']
375 . ':' . $record['recuid']
376 . ':' . $record['field']
377 . ($record['flexpointer'] ? ':' . $record['flexpointer'] : '')
378 . ($record['softref_key'] ? ':' . $record['softref_key'] . ' (Soft Reference) ' : '')
379 . ($record['deleted'] ? ' (DELETED)' : '');
380 }
381 }