3ea622b7d654b23ad978a7b92dee582cf5ed3353
[Packages/TYPO3.CMS.git] / typo3 / sysext / lowlevel / Classes / Command / MissingFilesCommand.php
1 <?php
2 declare(strict_types = 1);
3 namespace TYPO3\CMS\Lowlevel\Command;
4
5 /*
6 * This file is part of the TYPO3 CMS project.
7 *
8 * It is free software; you can redistribute it and/or modify it under
9 * the terms of the GNU General Public License, either version 2
10 * of the License, or any later version.
11 *
12 * For the full copyright and license information, please read the
13 * LICENSE.txt file that was distributed with this source code.
14 *
15 * The TYPO3 project - inspiring people to share!
16 */
17
18 use Symfony\Component\Console\Command\Command;
19 use Symfony\Component\Console\Input\InputInterface;
20 use Symfony\Component\Console\Input\InputOption;
21 use Symfony\Component\Console\Output\OutputInterface;
22 use Symfony\Component\Console\Style\SymfonyStyle;
23 use TYPO3\CMS\Core\Core\Bootstrap;
24 use TYPO3\CMS\Core\Database\ConnectionPool;
25 use TYPO3\CMS\Core\Database\ReferenceIndex;
26 use TYPO3\CMS\Core\Utility\GeneralUtility;
27
28 /**
29 * Finds files which are referenced by TYPO3 but not found in the file system
30 */
31 class MissingFilesCommand extends Command
32 {
33
34 /**
35 * Configure the command by defining the name, options and arguments
36 */
37 public function configure()
38 {
39 $this
40 ->setDescription('Find all file references from records pointing to a missing (non-existing) file.')
41 ->setHelp('
42 Assumptions:
43 - a perfect integrity of the reference index table (always update the reference index table before using this tool!)
44 - relevant soft reference parsers applied everywhere file references are used inline
45
46 Files may be missing for these reasons (except software bugs):
47 - someone manually deleted the file inside fileadmin/ or another user maintained folder. If the reference was a soft reference (opposite to a DataHandler managed file relation from "group" type fields), technically it is not an error although it might be a mistake that someone did so.
48 - someone manually deleted the file inside the uploads/ folder (typically containing managed files) which is an error since no user interaction should take place there.
49
50 Manual repair suggestions (using --dry-run):
51 - Managed files: You might be able to locate the file and re-insert it in the correct location. However, no automatic fix can do that for you.
52 - Soft References: You should investigate each case and edit the content accordingly. A soft reference to a file could be in an HTML image tag (for example <img src="missing_file.jpg" />) and you would have to either remove the whole tag, change the filename or re-create the missing file.
53
54 If the option "--dry-run" is not set, all managed files (TCA/FlexForm attachments) will silently remove the reference
55 from the record since the file is missing. For this reason you might prefer a manual approach instead.
56 All soft references with missing files require manual fix if you consider it an error.
57
58 If you want to get more detailed information, use the --verbose option.')
59 ->addOption(
60 'dry-run',
61 null,
62 InputOption::VALUE_NONE,
63 'If this option is set, the references will not be removed, but just the output which files would be deleted are shown'
64 )
65 ->addOption(
66 'update-refindex',
67 null,
68 InputOption::VALUE_NONE,
69 'Setting this option automatically updates the reference index and does not ask on command line. Alternatively, use -n to avoid the interactive mode'
70 );
71 }
72
73 /**
74 * Executes the command to
75 * - optionally update the reference index (to have clean data)
76 * - find data in sys_refindex (softrefs and regular references) where the actual file does not exist (anymore)
77 * - remove these files if --dry-run is not set (not possible for refindexes)
78 *
79 * @param InputInterface $input
80 * @param OutputInterface $output
81 */
82 protected function execute(InputInterface $input, OutputInterface $output)
83 {
84 // Make sure the _cli_ user is loaded
85 Bootstrap::initializeBackendAuthentication();
86
87 $io = new SymfonyStyle($input, $output);
88 $io->title($this->getDescription());
89
90 $dryRun = $input->hasOption('dry-run') && $input->getOption('dry-run') != false ? true : false;
91
92 // Update the reference index
93 $this->updateReferenceIndex($input, $io);
94
95 // Find missing soft references (cannot be updated / deleted)
96 $missingSoftReferencedFiles = $this->findMissingSoftReferencedFiles();
97 if (count($missingSoftReferencedFiles)) {
98 $io->note('Found ' . count($missingSoftReferencedFiles) . ' soft-referenced files that need manual repair.');
99 $io->listing($missingSoftReferencedFiles);
100 }
101
102 // Find missing references
103 $missingReferencedFiles = $this->findMissingReferencedFiles();
104 if (count($missingReferencedFiles)) {
105 $io->note('Found ' . count($missingReferencedFiles) . ' references to non-existing files.');
106
107 $this->removeReferencesToMissingFiles($missingReferencedFiles, $dryRun, $io);
108 $io->success('All references were updated accordingly.');
109 }
110
111 if (!count($missingSoftReferencedFiles) && !count($missingReferencedFiles)) {
112 $io->success('Nothing to do, no missing files found. Everything is in place.');
113 }
114 }
115
116 /**
117 * Function to update the reference index
118 * - if the option --update-refindex is set, do it
119 * - otherwise, if in interactive mode (not having -n set), ask the user
120 * - otherwise assume everything is fine
121 *
122 * @param InputInterface $input holds information about entered parameters
123 * @param SymfonyStyle $io necessary for outputting information
124 */
125 protected function updateReferenceIndex(InputInterface $input, SymfonyStyle $io)
126 {
127 // Check for reference index to update
128 $io->note('Finding missing files referenced by TYPO3 requires a clean reference index (sys_refindex)');
129 if ($input->hasOption('update-refindex') && $input->getOption('update-refindex')) {
130 $updateReferenceIndex = true;
131 } elseif ($input->isInteractive()) {
132 $updateReferenceIndex = $io->confirm('Should the reference index be updated right now?', false);
133 } else {
134 $updateReferenceIndex = false;
135 }
136
137 // Update the reference index
138 if ($updateReferenceIndex) {
139 $referenceIndex = GeneralUtility::makeInstance(ReferenceIndex::class);
140 $referenceIndex->updateIndex(false, !$io->isQuiet());
141 } else {
142 $io->writeln('Reference index is assumed to be up to date, continuing.');
143 }
144 }
145
146 /**
147 * Find file references that points to non-existing files in system
148 * Fix methods: API in \TYPO3\CMS\Core\Database\ReferenceIndex that allows to
149 * change the value of a reference (or remove it)
150 *
151 * @return array an array of records within sys_refindex
152 */
153 protected function findMissingReferencedFiles(): array
154 {
155 $missingReferences = [];
156 // Select all files in the reference table
157 $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
158 ->getQueryBuilderForTable('sys_refindex');
159
160 $result = $queryBuilder
161 ->select('*')
162 ->from('sys_refindex')
163 ->where(
164 $queryBuilder->expr()->eq('ref_table', $queryBuilder->createNamedParameter('_FILE', \PDO::PARAM_STR)),
165 $queryBuilder->expr()->isNull('softref_key')
166 )
167 ->execute();
168
169 // Traverse the references and check if the files exists
170 while ($record = $result->fetch()) {
171 $fileName = $record['ref_string'];
172 if (empty($record['softref_key']) && !@is_file(PATH_site . $fileName)) {
173 $missingReferences[$fileName][$record['hash']] = $this->formatReferenceIndexEntryToString($record);
174 }
175 }
176
177 return $missingReferences;
178 }
179
180 /**
181 * Find file references that points to non-existing files in system
182 * registered as soft references (checked for "softref_key")
183 *
184 * @return array an array of the data within soft references
185 */
186 protected function findMissingSoftReferencedFiles(): array
187 {
188 $missingReferences = [];
189 // Select all files in the reference table
190 $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
191 ->getQueryBuilderForTable('sys_refindex');
192
193 $result = $queryBuilder
194 ->select('*')
195 ->from('sys_refindex')
196 ->where(
197 $queryBuilder->expr()->eq('ref_table', $queryBuilder->createNamedParameter('_FILE', \PDO::PARAM_STR)),
198 $queryBuilder->expr()->isNotNull('softref_key')
199 )
200 ->execute();
201
202 // Traverse the references and check if the files exists
203 while ($record = $result->fetch()) {
204 $fileName = $record['ref_string'];
205 if (!@is_file(PATH_site . $fileName)) {
206 $missingReferences[] = $fileName . ' - ' . $record['hash'] . ' - ' . $this->formatReferenceIndexEntryToString($record);
207 }
208 }
209 return $missingReferences;
210 }
211
212 /**
213 * Removes all references in the sys_file_reference where files were not found
214 *
215 * @param array $missingManagedFiles Contains the records of sys_refindex which need to be updated
216 * @param bool $dryRun if set, the references are just displayed, but not removed
217 * @param SymfonyStyle $io the IO object for output
218 */
219 protected function removeReferencesToMissingFiles(array $missingManagedFiles, bool $dryRun, SymfonyStyle $io)
220 {
221 foreach ($missingManagedFiles as $fileName => $references) {
222 if ($io->isVeryVerbose()) {
223 $io->writeln('Deleting references to missing file "' . $fileName . '"');
224 }
225 foreach ($references as $hash => $recordReference) {
226 $io->writeln('Removing reference in record "' . $recordReference . '"');
227 if (!$dryRun) {
228 $sysRefObj = GeneralUtility::makeInstance(ReferenceIndex::class);
229 $error = $sysRefObj->setReferenceValue($hash, null);
230 if ($error) {
231 $io->error('ReferenceIndex::setReferenceValue() reported "' . $error . '"');
232 }
233 }
234 }
235 }
236 }
237
238 /**
239 * Formats a sys_refindex entry to something readable
240 *
241 * @param array $record
242 * @return string
243 */
244 protected function formatReferenceIndexEntryToString(array $record): string
245 {
246 return $record['tablename'] . ':' . $record['recuid'] . ':' . $record['field'] . ':' . $record['flexpointer'] . ':' . $record['softref_key'] . ($record['deleted'] ? ' (DELETED)' : '');
247 }
248 }