[TASK] Replace PATH_site with Environment API in various exts
[Packages/TYPO3.CMS.git] / typo3 / sysext / lowlevel / Classes / Command / LostFilesCommand.php
1 <?php
2 declare(strict_types = 1);
3 namespace TYPO3\CMS\Lowlevel\Command;
4
5 /*
6 * This file is part of the TYPO3 CMS project.
7 *
8 * It is free software; you can redistribute it and/or modify it under
9 * the terms of the GNU General Public License, either version 2
10 * of the License, or any later version.
11 *
12 * For the full copyright and license information, please read the
13 * LICENSE.txt file that was distributed with this source code.
14 *
15 * The TYPO3 project - inspiring people to share!
16 */
17
18 use Symfony\Component\Console\Command\Command;
19 use Symfony\Component\Console\Input\InputInterface;
20 use Symfony\Component\Console\Input\InputOption;
21 use Symfony\Component\Console\Output\OutputInterface;
22 use Symfony\Component\Console\Style\SymfonyStyle;
23 use TYPO3\CMS\Core\Core\Bootstrap;
24 use TYPO3\CMS\Core\Core\Environment;
25 use TYPO3\CMS\Core\Database\ConnectionPool;
26 use TYPO3\CMS\Core\Database\ReferenceIndex;
27 use TYPO3\CMS\Core\Utility\GeneralUtility;
28 use TYPO3\CMS\Core\Utility\PathUtility;
29
30 /**
31 * Finds files within uploads/ which are not needed anymore
32 */
33 class LostFilesCommand extends Command
34 {
35
36 /**
37 * Configure the command by defining the name, options and arguments
38 */
39 public function configure()
40 {
41 $this
42 ->setDescription('Looking for files in the uploads/ folder which does not have a reference in TYPO3 managed records.')
43 ->setHelp('
44 Assumptions:
45 - a perfect integrity of the reference index table (always update the reference index table before using this tool!)
46 - that all contents in the uploads folder are files attached to TCA records and exclusively managed by DataHandler through "group" type fields
47 - index.html, .htaccess files and RTEmagic* image files (ignored)
48 - Files found in deleted records are included (otherwise you would see a false list of lost files)
49
50 The assumptions are not requirements by the TYPO3 API but reflects the de facto implementation of most TYPO3 installations and therefore a practical approach to cleaning up the uploads/ or costum folder.
51 Therefore, if all "group" type fields in TCA and flexforms are positioned inside the uploads/ folder and if no files inside are managed manually it should be safe to clean out files with no relations found in the system.
52 Under such circumstances there should theoretically be no lost files in the uploads/ or custom folder since DataHandler should have managed relations automatically including adding and deleting files.
53 However, there is at least one reason known to why files might be found lost and that is when FlexForms are used. In such a case a change of/in the Data Structure XML (or the ability of the system to find the Data Structure definition!) used for the flexform could leave lost files behind. This is not unlikely to happen when records are deleted. More details can be found in a note to the function FlexFormTools->getDataStructureIdentifier()
54 Another scenario could of course be de-installation of extensions which managed files in the uploads/ or custom folders.
55
56 If the option "--dry-run" is not set, the files are then deleted automatically.
57 Warning: First, make sure those files are not used somewhere TYPO3 does not know about! See the assumptions above.
58
59 If you want to get more detailed information, use the --verbose option.')
60 ->addOption(
61 'exclude',
62 null,
63 InputOption::VALUE_REQUIRED,
64 'Comma-separated list of paths that should be excluded, e.g. "uploads/pics,uploads/media"'
65 )
66 ->addOption(
67 'dry-run',
68 null,
69 InputOption::VALUE_NONE,
70 'If this option is set, the files will not actually be deleted, but just the output which files would be deleted are shown'
71 )
72 ->addOption(
73 'update-refindex',
74 null,
75 InputOption::VALUE_NONE,
76 'Setting this option automatically updates the reference index and does not ask on command line. Alternatively, use -n to avoid the interactive mode'
77 )
78 ->addOption(
79 'custom-path',
80 null,
81 InputOption::VALUE_REQUIRED,
82 'Comma separated list of paths to process. Example: "fileadmin/[path1],fileadmin/[path2],...", if not passed, uploads/ will be used by default.'
83 );
84 }
85
86 /**
87 * Executes the command to
88 * - optionally update the reference index (to have clean data)
89 * - find files within uploads/* which are not connected to the reference index
90 * - remove these files if --dry-run is not set
91 *
92 * @param InputInterface $input
93 * @param OutputInterface $output
94 */
95 protected function execute(InputInterface $input, OutputInterface $output)
96 {
97 // Make sure the _cli_ user is loaded
98 Bootstrap::initializeBackendAuthentication();
99
100 $io = new SymfonyStyle($input, $output);
101 $io->title($this->getDescription());
102
103 $dryRun = $input->hasOption('dry-run') && $input->getOption('dry-run') != false ? true : false;
104
105 $this->updateReferenceIndex($input, $io);
106
107 // Find the lost files
108 if ($input->hasOption('exclude') && !empty($input->getOption('exclude'))) {
109 $excludedPaths = GeneralUtility::trimExplode(',', $input->getOption('exclude'), true);
110 } else {
111 $excludedPaths = [];
112 }
113
114 // Use custom-path
115 $customPaths = '';
116 if ($input->hasOption('custom-path') && !empty($input->getOption('custom-path'))) {
117 $customPaths = $input->getOption('custom-path');
118 }
119
120 $lostFiles = $this->findLostFiles($excludedPaths, $customPaths);
121
122 if (count($lostFiles)) {
123 if (!$io->isQuiet()) {
124 $io->note('Found ' . count($lostFiles) . ' lost files, ready to be deleted.');
125 if ($io->isVerbose()) {
126 $io->listing($lostFiles);
127 }
128 }
129
130 // Delete them
131 $this->deleteLostFiles($lostFiles, $dryRun, $io);
132
133 $io->success('Deleted ' . count($lostFiles) . ' lost files.');
134 } else {
135 $io->success('Nothing to do, no lost files found');
136 }
137 }
138
139 /**
140 * Function to update the reference index
141 * - if the option --update-refindex is set, do it
142 * - otherwise, if in interactive mode (not having -n set), ask the user
143 * - otherwise assume everything is fine
144 *
145 * @param InputInterface $input holds information about entered parameters
146 * @param SymfonyStyle $io necessary for outputting information
147 */
148 protected function updateReferenceIndex(InputInterface $input, SymfonyStyle $io)
149 {
150 // Check for reference index to update
151 $io->note('Finding lost files managed by TYPO3 requires a clean reference index (sys_refindex)');
152 $updateReferenceIndex = false;
153 if ($input->hasOption('update-refindex') && $input->getOption('update-refindex')) {
154 $updateReferenceIndex = true;
155 } elseif ($input->isInteractive()) {
156 $updateReferenceIndex = $io->confirm('Should the reference index be updated right now?', false);
157 }
158
159 // Update the reference index
160 if ($updateReferenceIndex) {
161 $referenceIndex = GeneralUtility::makeInstance(ReferenceIndex::class);
162 $referenceIndex->updateIndex(false, !$io->isQuiet());
163 } else {
164 $io->writeln('Reference index is assumed to be up to date, continuing.');
165 }
166 }
167
168 /**
169 * Find lost files in uploads/ or custom folder
170 *
171 * @param array $excludedPaths list of paths to be excluded, can be uploads/pics/
172 * @param string $customPaths list of paths to be checked instead of uploads/
173 * @return array an array of files (relative to Environment::getPublicPath()) that are not connected
174 */
175 protected function findLostFiles($excludedPaths = [], $customPaths = ''): array
176 {
177 $lostFiles = [];
178
179 // Get all files
180 $files = [];
181 if (!empty($customPaths)) {
182 $customPaths = GeneralUtility::trimExplode(',', $customPaths, true);
183 foreach ($customPaths as $customPath) {
184 if (false === realpath(Environment::getPublicPath() . '/' . $customPath)
185 || !GeneralUtility::isFirstPartOfStr(realpath(Environment::getPublicPath() . '/' . $customPath), realpath(Environment::getPublicPath()))) {
186 throw new \Exception('The path: "' . $customPath . '" is invalid', 1450086736);
187 }
188 $files = GeneralUtility::getAllFilesAndFoldersInPath($files, Environment::getPublicPath() . '/' . $customPath);
189 }
190 } else {
191 $files = GeneralUtility::getAllFilesAndFoldersInPath($files, Environment::getPublicPath() . '/uploads/');
192 }
193
194 $files = GeneralUtility::removePrefixPathFromList($files, Environment::getPublicPath() . '/');
195
196 $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
197 ->getQueryBuilderForTable('sys_refindex');
198
199 // Traverse files and for each, look up if its found in the reference index.
200 foreach ($files as $key => $value) {
201
202 // First, allow "index.html", ".htaccess" files since they are often used for good reasons
203 if (substr($value, -11) === '/index.html' || substr($value, -10) === '/.htaccess') {
204 continue;
205 }
206
207 // If the file is a RTEmagic-image name and if so, we allow it
208 if (preg_match('/^RTEmagic[P|C]_/', PathUtility::basenameDuringBootstrap($value))) {
209 continue;
210 }
211
212 $fileIsInExcludedPath = false;
213 foreach ($excludedPaths as $exclPath) {
214 if (GeneralUtility::isFirstPartOfStr($value, $exclPath)) {
215 $fileIsInExcludedPath = true;
216 break;
217 }
218 }
219
220 if ($fileIsInExcludedPath) {
221 continue;
222 }
223
224 // Looking for a reference from a field which is NOT a soft reference (thus, only fields with a proper TCA/Flexform configuration)
225 $result = $queryBuilder
226 ->select('hash')
227 ->from('sys_refindex')
228 ->where(
229 $queryBuilder->expr()->eq(
230 'ref_table',
231 $queryBuilder->createNamedParameter('_FILE', \PDO::PARAM_STR)
232 ),
233 $queryBuilder->expr()->eq(
234 'ref_string',
235 $queryBuilder->createNamedParameter($value, \PDO::PARAM_STR)
236 ),
237 $queryBuilder->expr()->eq(
238 'softref_key',
239 $queryBuilder->createNamedParameter('', \PDO::PARAM_STR)
240 )
241 )
242 ->orderBy('sorting', 'DESC')
243 ->execute();
244
245 $rowCount = $queryBuilder->count('hash')->execute()->fetchColumn(0);
246 // We conclude that the file is lost
247 if ($rowCount === 0) {
248 $lostFiles[] = $value;
249 }
250 }
251
252 return $lostFiles;
253 }
254
255 /**
256 * Removes given files from the uploads/ folder
257 *
258 * @param array $lostFiles Contains the lost files found
259 * @param bool $dryRun if set, the files are just displayed, but not deleted
260 * @param SymfonyStyle $io the IO object for output
261 */
262 protected function deleteLostFiles(array $lostFiles, bool $dryRun, SymfonyStyle $io)
263 {
264 foreach ($lostFiles as $lostFile) {
265 $absoluteFileName = GeneralUtility::getFileAbsFileName($lostFile);
266 if ($io->isVeryVerbose()) {
267 $io->writeln('Deleting file "' . $absoluteFileName . '"');
268 }
269 if (!$dryRun) {
270 if ($absoluteFileName && @is_file($absoluteFileName)) {
271 unlink($absoluteFileName);
272 if (!$io->isQuiet()) {
273 $io->writeln('Permanently deleted file record "' . $absoluteFileName . '".');
274 }
275 } else {
276 $io->error('File "' . $absoluteFileName . '" was not found!');
277 }
278 }
279 }
280 }
281 }