[TASK] Replace PATH_site with Environment API in various exts
[Packages/TYPO3.CMS.git] / typo3 / sysext / lowlevel / Classes / Command / RteImagesCommand.php
1 <?php
2 declare(strict_types = 1);
3 namespace TYPO3\CMS\Lowlevel\Command;
4
5 /*
6 * This file is part of the TYPO3 CMS project.
7 *
8 * It is free software; you can redistribute it and/or modify it under
9 * the terms of the GNU General Public License, either version 2
10 * of the License, or any later version.
11 *
12 * For the full copyright and license information, please read the
13 * LICENSE.txt file that was distributed with this source code.
14 *
15 * The TYPO3 project - inspiring people to share!
16 */
17
18 use Symfony\Component\Console\Command\Command;
19 use Symfony\Component\Console\Input\InputInterface;
20 use Symfony\Component\Console\Input\InputOption;
21 use Symfony\Component\Console\Output\OutputInterface;
22 use Symfony\Component\Console\Style\SymfonyStyle;
23 use TYPO3\CMS\Core\Core\Bootstrap;
24 use TYPO3\CMS\Core\Core\Environment;
25 use TYPO3\CMS\Core\Database\ConnectionPool;
26 use TYPO3\CMS\Core\Database\ReferenceIndex;
27 use TYPO3\CMS\Core\Utility\File\BasicFileUtility;
28 use TYPO3\CMS\Core\Utility\GeneralUtility;
29 use TYPO3\CMS\Core\Utility\PathUtility;
30
31 /**
32 * Looking up all occurencies of RTEmagic images in the database and check existence of parent and
33 * copy files on the file system plus report possibly lost files of this type
34 */
35 class RteImagesCommand extends Command
36 {
37
38 /**
39 * Configure the command by defining the name, options and arguments
40 */
41 public function configure()
42 {
43 $this
44 ->setDescription('Looking up all occurrences of RTEmagic images in the database and check existence of parent and copy files on the file system plus report possibly lost RTE files.')
45 ->setHelp('
46 Assumptions:
47 - a perfect integrity of the reference index table (always update the reference index table before using this tool!)
48 - that all RTEmagic image files in the database are registered with the soft reference parser "images"
49 - images found in deleted records are included (means that you might find lost RTEmagic images after flushing deleted records)
50
51 The assumptions are not requirements by the TYPO3 API but reflects the de facto implementation of most TYPO3 installations.
52 However, many custom fields using an RTE will probably not have the "images" soft reference parser registered and so the index will be incomplete and not listing all RTEmagic image files.
53 The consequence of this limitation is that you should be careful if you wish to delete lost RTEmagic images - they could be referenced from a field not parsed by the "images" soft reference parser!
54
55 Automatic Repair of Errors:
56 - Will search for double-usages of RTEmagic images and make copies as required.
57 - Lost files can be deleted automatically, but it is recommended to delete them manually if you do not recognize them as used somewhere the system does not know about.
58
59 Manual repair suggestions:
60 - Missing files: Re-insert missing files or edit record where the reference is found.
61
62 If the option "--dry-run" is not set, the files are then deleted automatically.
63 Warning: First, make sure those files are not used somewhere TYPO3 does not know about! See the assumptions above.
64
65 If you want to get more detailed information, use the --verbose option.')
66 ->addOption(
67 'dry-run',
68 null,
69 InputOption::VALUE_NONE,
70 'If this option is set, the files will not actually be deleted, but just the output which files would be deleted are shown'
71 )
72 ->addOption(
73 'update-refindex',
74 null,
75 InputOption::VALUE_NONE,
76 'Setting this option automatically updates the reference index and does not ask on command line. Alternatively, use -n to avoid the interactive mode'
77 );
78 }
79
80 /**
81 * Executes the command to
82 * - optionally update the reference index (to have clean data)
83 * - find files within uploads/* which are not connected to the reference index
84 * - remove these files if --dry-run is not set
85 *
86 * @param InputInterface $input
87 * @param OutputInterface $output
88 */
89 protected function execute(InputInterface $input, OutputInterface $output)
90 {
91 // Make sure the _cli_ user is loaded
92 Bootstrap::initializeBackendAuthentication();
93
94 $io = new SymfonyStyle($input, $output);
95 $io->title($this->getDescription());
96
97 $dryRun = $input->hasOption('dry-run') && $input->getOption('dry-run') != false ? true : false;
98
99 $this->updateReferenceIndex($input, $io);
100
101 // Find the RTE files
102 $allRteImagesInUse = $this->findAllReferencedRteImagesWithOriginals();
103
104 if (count($allRteImagesInUse)) {
105 $allRteImagesWithOriginals = [];
106 $multipleReferenced = [];
107 $missingFiles = [];
108 $lostFiles = [];
109
110 // Searching for duplicates, and missing files (also missing originals)
111 foreach ($allRteImagesInUse as $fileName => $fileInfo) {
112 $allRteImagesWithOriginals[$fileName]++;
113 $allRteImagesWithOriginals[$fileInfo['original']]++;
114 if ($fileInfo['count'] > 1 && $fileInfo['exists'] && $fileInfo['original_exists']) {
115 $multipleReferenced[$fileName] = $fileInfo['softReferences'];
116 }
117 // Missing files:
118 if (!$fileInfo['exists']) {
119 $missingFiles[$fileName] = $fileInfo['softReferences'];
120 }
121 if (!$fileInfo['original_exists']) {
122 $missingFiles[$fileInfo['original']] = $fileInfo['softReferences'];
123 }
124 }
125
126 // Now, ask for RTEmagic files inside uploads/ folder:
127 $magicFiles = $this->findAllRteFilesInDirectory();
128 foreach ($magicFiles as $fileName) {
129 if (!isset($allRteImagesWithOriginals[$fileName])) {
130 $lostFiles[$fileName] = $fileName;
131 }
132 }
133 ksort($missingFiles);
134 ksort($multipleReferenced);
135
136 // Output info about missing files
137 if (!$io->isQuiet()) {
138 $io->note('Found ' . count($missingFiles) . ' RTE images that are referenced, but missing.');
139 if ($io->isVerbose()) {
140 $io->listing($missingFiles);
141 }
142 }
143
144 // Duplicate RTEmagic image files
145 // These files are RTEmagic images found used in multiple records! RTEmagic images should be used by only
146 // one record at a time. A large amount of such images probably stems from previous versions of TYPO3 (before 4.2)
147 // which did not support making copies automatically of RTEmagic images in case of new copies / versions.
148 $this->copyMultipleReferencedRteImages($multipleReferenced, $dryRun, $io);
149
150 // Delete lost files
151 // Lost RTEmagic files from uploads/
152 // These files you might be able to delete but only if _all_ RTEmagic images are found by the soft reference parser.
153 // If you are using the RTE in third-party extensions it is likely that the soft reference parser is not applied
154 // correctly to their RTE and thus these "lost" files actually represent valid RTEmagic images,
155 // just not registered. Lost files can be auto-fixed but only if you specifically
156 // set "lostFiles" as parameter to the --AUTOFIX option.
157 if (count($lostFiles)) {
158 ksort($lostFiles);
159 $this->deleteLostFiles($lostFiles, $dryRun, $io);
160 $io->success('Deleted ' . count($lostFiles) . ' lost files.');
161 }
162 } else {
163 $io->success('Nothing to do, your system does not have any RTE images.');
164 }
165 }
166
167 /**
168 * Function to update the reference index
169 * - if the option --update-refindex is set, do it
170 * - otherwise, if in interactive mode (not having -n set), ask the user
171 * - otherwise assume everything is fine
172 *
173 * @param InputInterface $input holds information about entered parameters
174 * @param SymfonyStyle $io necessary for outputting information
175 */
176 protected function updateReferenceIndex(InputInterface $input, SymfonyStyle $io)
177 {
178 // Check for reference index to update
179 $io->note('Finding RTE images used in TYPO3 requires a clean reference index (sys_refindex)');
180 $updateReferenceIndex = false;
181 if ($input->hasOption('update-refindex') && $input->getOption('update-refindex')) {
182 $updateReferenceIndex = true;
183 } elseif ($input->isInteractive()) {
184 $updateReferenceIndex = $io->confirm('Should the reference index be updated right now?', false);
185 }
186
187 // Update the reference index
188 if ($updateReferenceIndex) {
189 $referenceIndex = GeneralUtility::makeInstance(ReferenceIndex::class);
190 $referenceIndex->updateIndex(false, !$io->isQuiet());
191 } else {
192 $io->writeln('Reference index is assumed to be up to date, continuing.');
193 }
194 }
195
196 /**
197 * Find lost files in uploads/ folder
198 *
199 * @return array an array of files (relative to Environment::getPublicPath()) that are not connected
200 */
201 protected function findAllReferencedRteImagesWithOriginals(): array
202 {
203 $allRteImagesInUse = [];
204
205 // Select all RTEmagic files in the reference table (only from soft references of course)
206 $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
207 ->getQueryBuilderForTable('sys_refindex');
208
209 $result = $queryBuilder
210 ->select('*')
211 ->from('sys_refindex')
212 ->where(
213 $queryBuilder->expr()->eq(
214 'ref_table',
215 $queryBuilder->createNamedParameter('_FILE', \PDO::PARAM_STR)
216 ),
217 $queryBuilder->expr()->like(
218 'ref_string',
219 $queryBuilder->createNamedParameter('%/RTEmagic%', \PDO::PARAM_STR)
220 ),
221 $queryBuilder->expr()->eq(
222 'softref_key',
223 $queryBuilder->createNamedParameter('images', \PDO::PARAM_STR)
224 )
225 )
226 ->execute();
227
228 // Traverse the files and put into a large table:
229 while ($rec = $result->fetch()) {
230 $file = $rec['ref_string'];
231 $filename = PathUtility::basenameDuringBootstrap($file);
232 if (strpos($filename, 'RTEmagicC_') === 0) {
233 // First time the file is referenced => build index
234 if (!is_array($allRteImagesInUse[$file])) {
235 $original = 'RTEmagicP_' . preg_replace('/\\.[[:alnum:]]+$/', '', substr($filename, 10));
236 $original = substr($file, 0, -strlen($filename)) . $original;
237 $allRteImagesInUse[$file] = [
238 'exists' => @is_file(Environment::getPublicPath() . '/' . $file),
239 'original' => $original,
240 'original_exists' => @is_file(Environment::getPublicPath() . '/' . $original),
241 'count' => 0,
242 'softReferences' => []
243 ];
244 }
245 $allRteImagesInUse[$file]['count']++;
246 $allRteImagesInUse[$file]['softReferences'][$rec['hash']] = $this->formatReferenceIndexEntryToString($rec);
247 }
248 }
249
250 ksort($allRteImagesInUse);
251 return $allRteImagesInUse;
252 }
253
254 /**
255 * Find all RTE files in uploads/ folder
256 *
257 * @param string $folder the name of the folder to start from
258 * @return array an array of files (relative to Environment::getPublicPath()) that are not connected
259 */
260 protected function findAllRteFilesInDirectory($folder = 'uploads/'): array
261 {
262 $filesFound = [];
263
264 // Get all files
265 $files = [];
266 $files = GeneralUtility::getAllFilesAndFoldersInPath($files, Environment::getPublicPath() . '/' . $folder);
267 $files = GeneralUtility::removePrefixPathFromList($files, Environment::getPublicPath() . '/');
268
269 // Traverse files
270 foreach ($files as $key => $value) {
271 // If the file is a RTEmagic-image name
272 if (preg_match('/^RTEmagic[P|C]_/', PathUtility::basenameDuringBootstrap($value))) {
273 $filesFound[] = $value;
274 continue;
275 }
276 }
277
278 return $filesFound;
279 }
280
281 /**
282 * Removes given files from the uploads/ folder
283 *
284 * @param array $lostFiles Contains the lost files found
285 * @param bool $dryRun if set, the files are just displayed, but not deleted
286 * @param SymfonyStyle $io the IO object for output
287 */
288 protected function deleteLostFiles(array $lostFiles, bool $dryRun, SymfonyStyle $io)
289 {
290 foreach ($lostFiles as $lostFile) {
291 $absoluteFileName = GeneralUtility::getFileAbsFileName($lostFile);
292 if ($io->isVeryVerbose()) {
293 $io->writeln('Deleting file "' . $absoluteFileName . '"');
294 }
295 if (!$dryRun) {
296 if ($absoluteFileName && @is_file($absoluteFileName)) {
297 unlink($absoluteFileName);
298 if (!$io->isQuiet()) {
299 $io->writeln('Permanently deleted file "' . $absoluteFileName . '".');
300 }
301 } else {
302 $io->error('File "' . $absoluteFileName . '" was not found!');
303 }
304 }
305 }
306 }
307
308 /**
309 * Duplicate RTEmagic image files which are used on several records. RTEmagic images should be used by only
310 * one record at a time. A large amount of such images probably stems from previous versions of TYPO3 (before 4.2)
311 * which did not support making copies automatically of RTEmagic images in case of new copies / versions.
312 *
313 * @param array $multipleReferencedImages
314 * @param bool $dryRun
315 * @param SymfonyStyle $io
316 */
317 protected function copyMultipleReferencedRteImages(array $multipleReferencedImages, bool $dryRun, SymfonyStyle $io)
318 {
319 $fileProcObj = GeneralUtility::makeInstance(BasicFileUtility::class);
320 foreach ($multipleReferencedImages as $fileName => $fileInfo) {
321 // Traverse all records using the file
322 $c = 0;
323 foreach ($fileInfo['usedIn'] as $hash => $recordID) {
324 if ($c === 0) {
325 $io->writeln('Keeping file ' . $fileName . ' for record ' . $recordID);
326 } else {
327 $io->writeln('Copying file ' . PathUtility::basenameDuringBootstrap($fileName) . ' for record ' . $recordID);
328 // Get directory prefix for file and set the original name
329 $dirPrefix = PathUtility::dirnameDuringBootstrap($fileName) . '/';
330 $rteOrigName = PathUtility::basenameDuringBootstrap($fileInfo['original']);
331 // If filename looks like an RTE file, and the directory is in "uploads/", then process as a RTE file!
332 if ($rteOrigName && strpos($dirPrefix, 'uploads/') === 0 && @is_dir(Environment::getPublicPath() . '/' . $dirPrefix)) {
333 // From the "original" RTE filename, produce a new "original" destination filename which is unused.
334 $origDestName = $fileProcObj->getUniqueName($rteOrigName, Environment::getPublicPath() . '/' . $dirPrefix);
335 // Create copy file name
336 $pI = pathinfo($fileName);
337 $copyDestName = PathUtility::dirnameDuringBootstrap($origDestName) . '/RTEmagicC_' . substr(PathUtility::basenameDuringBootstrap($origDestName), 10) . '.' . $pI['extension'];
338 if (!@is_file($copyDestName) && !@is_file($origDestName) && $origDestName === GeneralUtility::getFileAbsFileName($origDestName) && $copyDestName === GeneralUtility::getFileAbsFileName($copyDestName)) {
339 $io->writeln('Copying file ' . PathUtility::basenameDuringBootstrap($fileName) . ' for record ' . $recordID . ' to ' . PathUtility::basenameDuringBootstrap($copyDestName));
340 if (!$dryRun) {
341 // Making copies
342 GeneralUtility::upload_copy_move(Environment::getPublicPath() . '/' . $fileInfo['original'], $origDestName);
343 GeneralUtility::upload_copy_move(Environment::getPublicPath() . '/' . $fileName, $copyDestName);
344 clearstatcache();
345 if (@is_file($copyDestName)) {
346 $referenceIndex = GeneralUtility::makeInstance(ReferenceIndex::class);
347 $error = $referenceIndex->setReferenceValue($hash, PathUtility::stripPathSitePrefix($copyDestName));
348 if ($error) {
349 $io->error('ReferenceIndex::setReferenceValue() reported "' . $error . '"');
350 }
351 } else {
352 $io->error('File "' . $copyDestName . '" could not be created.');
353 }
354 }
355 } else {
356 $io->error('Could not construct new unique names for file.');
357 }
358 } else {
359 $io->error('Maybe directory of file was not within "uploads/"?');
360 }
361 }
362 $c++;
363 }
364 }
365 }
366
367 /**
368 * Formats a sys_refindex entry to something readable
369 *
370 * @param array $record
371 * @return string
372 */
373 protected function formatReferenceIndexEntryToString(array $record): string
374 {
375 return $record['tablename']
376 . ':' . $record['recuid']
377 . ':' . $record['field']
378 . ($record['flexpointer'] ? ':' . $record['flexpointer'] : '')
379 . ($record['softref_key'] ? ':' . $record['softref_key'] . ' (Soft Reference) ' : '')
380 . ($record['deleted'] ? ' (DELETED)' : '');
381 }
382 }