[BUGFIX] Improve install tool temporary assets removal 37/57837/11
authorChristian Kuhn <lolli@schwarzbu.ch>
Fri, 14 Sep 2018 21:01:16 +0000 (23:01 +0200)
committerMarkus Klein <markus.klein@typo3.org>
Sat, 15 Sep 2018 09:42:08 +0000 (11:42 +0200)
Maintenance > Remove temporary assets now properly lists
statistics for FAL processed files and allows to clear them.

Releases: master
Resolves: #83034
Change-Id: Ic9238c88648d98a76a0cbd4015820d51bffde0fa
Reviewed-on: https://review.typo3.org/57837
Reviewed-by: Guido Schmechel <guido.schmechel@brandung.de>
Tested-by: Guido Schmechel <guido.schmechel@brandung.de>
Reviewed-by: Christian Kuhn <lolli@schwarzbu.ch>
Tested-by: Christian Kuhn <lolli@schwarzbu.ch>
Tested-by: TYPO3com <no-reply@typo3.com>
Reviewed-by: Mathias Schreiber <mathias.schreiber@typo3.com>
Tested-by: Mathias Schreiber <mathias.schreiber@typo3.com>
Reviewed-by: Markus Klein <markus.klein@typo3.org>
Tested-by: Markus Klein <markus.klein@typo3.org>
typo3/sysext/core/Classes/Resource/ProcessedFileRepository.php
typo3/sysext/install/Classes/Controller/MaintenanceController.php
typo3/sysext/install/Classes/Service/Typo3tempFileService.php
typo3/sysext/install/Resources/Private/Templates/Maintenance/ClearTypo3tempFiles.html
typo3/sysext/install/Resources/Public/JavaScript/Modules/ClearTypo3tempFiles.js
typo3/sysext/install/Tests/Unit/Service/Typo3tempFileServiceTest.php

index 6dbb916..f99aef1 100644 (file)
@@ -131,6 +131,31 @@ class ProcessedFileRepository extends AbstractRepository implements LoggerAwareI
         }
         return $processedFileObject;
     }
+
+    /**
+     * Count processed files by storage. This is used in the install tool
+     * to render statistics of processed files.
+     *
+     * @param ResourceStorage $storage
+     * @return int
+     */
+    public function countByStorage(ResourceStorage $storage): int
+    {
+        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
+            ->getQueryBuilderForTable($this->table);
+        return (int)$queryBuilder
+            ->count('uid')
+            ->from($this->table)
+            ->where(
+                $queryBuilder->expr()->eq(
+                    'storage',
+                    $queryBuilder->createNamedParameter($storage->getUid(), \PDO::PARAM_INT)
+                )
+            )
+            ->execute()
+            ->fetchColumn(0);
+    }
+
     /**
      * Adds a processedfile object in the database
      *
@@ -248,20 +273,29 @@ class ProcessedFileRepository extends AbstractRepository implements LoggerAwareI
     }
 
     /**
-     * Removes all processed files and also deletes the associated physical files
+     * Removes all processed files and also deletes the associated physical files.
+     * If a storageUid is given, only db entries and files of this storage are removed.
      *
      * @param int|null $storageUid If not NULL, only the processed files of the given storage are removed
      * @return int Number of failed deletions
      */
     public function removeAll($storageUid = null)
     {
-        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable($this->table);
+        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
+            ->getQueryBuilderForTable($this->table);
+        $where = [
+            $queryBuilder->expr()->neq('identifier', $queryBuilder->createNamedParameter('', \PDO::PARAM_STR))
+        ];
+        if ($storageUid !== null) {
+            $where[] = $queryBuilder->expr()->eq(
+                'storage',
+                $queryBuilder->createNamedParameter($storageUid, \PDO::PARAM_INT)
+            );
+        }
         $result = $queryBuilder
             ->select('*')
             ->from($this->table)
-            ->where(
-                $queryBuilder->expr()->neq('identifier', $queryBuilder->createNamedParameter('', \PDO::PARAM_STR))
-            )
+            ->where(...$where)
             ->execute();
 
         $errorCount = 0;
@@ -285,9 +319,17 @@ class ProcessedFileRepository extends AbstractRepository implements LoggerAwareI
             }
         }
 
-        GeneralUtility::makeInstance(ConnectionPool::class)
-            ->getConnectionForTable($this->table)
-            ->truncate($this->table);
+        if ($storageUid === null) {
+            // Truncate entire table if not restricted to specific storage
+            GeneralUtility::makeInstance(ConnectionPool::class)
+                ->getConnectionForTable($this->table)
+                ->truncate($this->table);
+        } else {
+            // else remove db rows of this storage only
+            GeneralUtility::makeInstance(ConnectionPool::class)
+                ->getConnectionForTable($this->table)
+                ->delete($this->table, ['storage' => $storageUid], [\PDO::PARAM_INT]);
+        }
 
         return $errorCount;
     }
index fba04b7..5ad4da1 100644 (file)
@@ -99,7 +99,7 @@ class MaintenanceController extends AbstractController
     }
 
     /**
-     * Clear Processed Files
+     * Clear typo3temp/assets or FAL processed Files
      *
      * @param ServerRequestInterface $request
      * @return ResponseInterface
@@ -109,8 +109,14 @@ class MaintenanceController extends AbstractController
         $messageQueue = new FlashMessageQueue('install');
         $typo3tempFileService = new Typo3tempFileService();
         $folder = $request->getParsedBody()['install']['folder'];
-        if ($folder === '_processed_') {
-            $failedDeletions = $typo3tempFileService->clearProcessedFiles();
+        // storageUid is an optional post param if FAL storages should be cleaned
+        $storageUid = $request->getParsedBody()['install']['storageUid'] ?? null;
+        if ($storageUid === null) {
+            $typo3tempFileService->clearAssetsFolder($folder);
+            $messageQueue->enqueue(new FlashMessage('Cleared files in "' . $folder . '" folder'));
+        } else {
+            $storageUid = (int)$storageUid;
+            $failedDeletions = $typo3tempFileService->clearProcessedFiles($storageUid);
             if ($failedDeletions) {
                 $messageQueue->enqueue(new FlashMessage(
                     'Failed to delete ' . $failedDeletions . ' processed files. See TYPO3 log (by default typo3temp/var/log/typo3_*.log)',
@@ -120,9 +126,6 @@ class MaintenanceController extends AbstractController
             } else {
                 $messageQueue->enqueue(new FlashMessage('Cleared processed files'));
             }
-        } else {
-            $typo3tempFileService->clearAssetsFolder($folder);
-            $messageQueue->enqueue(new FlashMessage('Cleared files in "' . $folder . '" folder'));
         }
         return new JsonResponse([
             'success' => true,
index c6344b5..94f9a7b 100644 (file)
@@ -18,10 +18,12 @@ use Symfony\Component\Finder\Finder;
 use Symfony\Component\Finder\SplFileInfo;
 use TYPO3\CMS\Core\Core\Environment;
 use TYPO3\CMS\Core\Resource\ProcessedFileRepository;
+use TYPO3\CMS\Core\Resource\StorageRepository;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 
 /**
- * Service class to manage typo3temp/assets folder cleanup
+ * Service class to manage typo3temp/assets and FAL storage
+ * processed file statistics / cleanup.
  */
 class Typo3tempFileService
 {
@@ -30,40 +32,84 @@ class Typo3tempFileService
      *
      * @return array
      */
-    public function getDirectoryStatistics()
+    public function getDirectoryStatistics(): array
     {
-        $basePath = Environment::getPublicPath() . '/typo3temp/assets';
-        if (!is_dir($basePath)) {
-            return [];
-        }
+        return array_merge(
+            $this->statsFromTypo3temp(),
+            $this->statsFromStorages()
+        );
+    }
 
-        $dirFinder = new Finder();
-        $dirsInAssets = $dirFinder->directories()->in($basePath)->depth(0)->sortByName();
+    /**
+     * Directory statistics for typo3temp/assets folders with some
+     * special handling for legacy processed file storage _processed_
+     *
+     * @return array
+     */
+    protected function statsFromTypo3temp(): array
+    {
         $stats = [];
-        foreach ($dirsInAssets as $dirInAssets) {
-            /** @var SplFileInfo $dirInAssets */
-            $fileFinder = new Finder();
-            $fileCount = $fileFinder->files()->in($dirInAssets->getPathname())->count();
-            $stats[] = [
-                'directory' => $dirInAssets->getFilename(),
-                'numberOfFiles' => $fileCount,
-            ];
+        $typo3TempAssetsPath = '/typo3temp/assets/';
+        $basePath = Environment::getPublicPath() . $typo3TempAssetsPath;
+        if (is_dir($basePath)) {
+            $dirFinder = new Finder();
+            $dirsInAssets = $dirFinder->directories()->in($basePath)->depth(0)->sortByName();
+            foreach ($dirsInAssets as $dirInAssets) {
+                /** @var SplFileInfo $dirInAssets */
+                $fileFinder = new Finder();
+                $fileCount = $fileFinder->files()->in($dirInAssets->getPathname())->count();
+                $folderName = $dirInAssets->getFilename();
+                $stat = [
+                    'directory' => $typo3TempAssetsPath . $folderName,
+                    'numberOfFiles' => $fileCount,
+                ];
+                if ($folderName === '_processed_') {
+                    // The processed file storage for legacy files (eg. TCA type=group internal_type=file)
+                    // gets the storageUid set, so this one can be removed via FAL functionality
+                    $stat['storageUid'] = 0;
+                }
+                $stats[] = $stat;
+            }
         }
-
         return $stats;
     }
 
     /**
-     * Clear processed files
+     * Directory statistics for configured FAL storages.
      *
-     * The sys_file_processedfile table is truncated and the physical files of local storages are deleted.
+     * @return array
+     */
+    protected function statsFromStorages(): array
+    {
+        $stats = [];
+        $processedFileRepository = GeneralUtility::makeInstance(ProcessedFileRepository::class);
+        $storages = GeneralUtility::makeInstance(StorageRepository::class)->findAll();
+        foreach ($storages as $storage) {
+            if ($storage->isOnline()) {
+                $storageConfiguration = $storage->getConfiguration();
+                $storageBasePath = rtrim($storageConfiguration['basePath'], '/');
+                $processedPath = '/' . $storageBasePath . $storage->getProcessingFolder()->getIdentifier();
+                $numberOfFiles = $processedFileRepository->countByStorage($storage);
+                $stats[] = [
+                    'directory' => $processedPath,
+                    'numberOfFiles' => $numberOfFiles,
+                    'storageUid' => $storage->getUid()
+                ];
+            }
+        }
+        return $stats;
+    }
+
+    /**
+     * Clear processed files. The sys_file_processedfile table is cleared for
+     * given storage uid and the physical files of local processed storages are deleted.
      *
-     * @return int 0 if all went well, if >0 this number of files couldn't be deleted
+     * @return int 0 if all went well, if >0 this number of files that could not be deleted
      */
-    public function clearProcessedFiles()
+    public function clearProcessedFiles(int $storageUid): int
     {
         $repository = GeneralUtility::makeInstance(ProcessedFileRepository::class);
-        return $repository->removeAll();
+        return $repository->removeAll($storageUid);
     }
 
     /**
@@ -75,8 +121,11 @@ class Typo3tempFileService
      */
     public function clearAssetsFolder(string $folderName)
     {
-        $basePath = Environment::getPublicPath() . '/typo3temp/assets/' . $folderName;
-        if (empty($folderName) || !GeneralUtility::isAllowedAbsPath($basePath)) {
+        $basePath = Environment::getPublicPath() . $folderName;
+        if (empty($folderName)
+            || !GeneralUtility::isAllowedAbsPath($basePath)
+            || strpos($folderName, '/typo3temp/assets/') !== 0
+        ) {
             throw new \RuntimeException(
                 'Path to folder ' . $folderName . ' not allowed.',
                 1501781453
index 8bcf91d..2ca913a 100644 (file)
@@ -31,7 +31,7 @@
                                type="button"
                        >
                                Delete <span class="t3js-clearTypo3temp-stat-numberOfFiles"></span> files in
-                               typo3temp/assets/<span class="t3js-clearTypo3temp-stat-directory"></span>
+                               <span class="t3js-clearTypo3temp-stat-directory"></span>
                        </button>
                        <hr class="t3js-clearTypo3temp-stat-lastRuler">
                </div>
index 12b071d..19c8d2d 100644 (file)
@@ -52,8 +52,9 @@ define([
       });
       currentModal.on('click', this.selectorDeleteTrigger, function(e) {
         var folder = $(this).data('folder');
+        var storageUid = $(this).data('storage-uid');
         e.preventDefault();
-        self.delete(folder);
+        self.delete(folder, storageUid);
       });
     },
 
@@ -73,6 +74,7 @@ define([
                   aStat.find(self.selectorStatNumberOfFiles).text(element.numberOfFiles);
                   aStat.find(self.selectorStatDirectory).text(element.directory);
                   aStat.find(self.selectorDeleteTrigger).attr('data-folder', element.directory);
+                  aStat.find(self.selectorDeleteTrigger).attr('data-storage-uid', element.storageUid);
                   modalContent.find(self.selectorStatContainer).append(aStat.html());
                 }
               });
@@ -87,7 +89,7 @@ define([
       });
     },
 
-    delete: function(folder) {
+    delete: function(folder, storageUid) {
       var self = this;
       var executeToken = self.currentModal.find(this.selectorModuleContent).data('clear-typo3temp-delete-token');
       $.ajax({
@@ -98,7 +100,8 @@ define([
           'install': {
             'action': 'clearTypo3tempFiles',
             'token': executeToken,
-            'folder': folder
+            'folder': folder,
+            'storageUid': storageUid
           }
         },
         cache: false,
index 4f14b75..0325be3 100644 (file)
@@ -36,11 +36,11 @@ class Typo3tempFileServiceTest extends UnitTestCase
     /**
      * @test
      */
-    public function clearAssetsFolderThrowsWithNotExistingPath()
+    public function clearAssetsFolderThrowsIfPathDoesNotStartWithTypotempAssets()
     {
         $this->expectException(\RuntimeException::class);
-        $this->expectExceptionCode(1501781454);
+        $this->expectExceptionCode(1501781453);
         $subject = new Typo3tempFileService();
-        $subject->clearAssetsFolder('bar');
+        $subject->clearAssetsFolder('typo3temp/foo');
     }
 }