[BUGFIX] Processing folder has now nested subfolders 23/36523/9
authorAlina Fleser <afleser@arxia.com>
Sat, 31 Jan 2015 15:57:55 +0000 (17:57 +0200)
committerPhilipp Gampe <philipp.gampe@typo3.org>
Fri, 22 Apr 2016 12:02:21 +0000 (14:02 +0200)
In case of big installations the amount of processed files in
one folder is too much and causes performance issues and other
problems.
To prevent this kind of issues, nested subfolders are now
created in the processing folder in order to split the processed
files into more than one folder.

Resolves: #56557
Releases: master, 7.6
Change-Id: Id9a4fc3b4bb4b28ca26ff96221097171c3835eb7
Reviewed-on: https://review.typo3.org/36523
Reviewed-by: Frank Naegler <frank.naegler@typo3.org>
Tested-by: Frank Naegler <frank.naegler@typo3.org>
Reviewed-by: Philipp Gampe <philipp.gampe@typo3.org>
Tested-by: Philipp Gampe <philipp.gampe@typo3.org>
typo3/sysext/core/Classes/Resource/ProcessedFile.php
typo3/sysext/core/Classes/Resource/Processing/LocalImageProcessor.php
typo3/sysext/core/Classes/Resource/ResourceStorage.php
typo3/sysext/core/Tests/Unit/Resource/ResourceStorageTest.php

index 10f4ed9..0412dd4 100644 (file)
@@ -184,7 +184,7 @@ class ProcessedFile extends AbstractFile
         if ($this->identifier === null) {
             throw new \RuntimeException('Cannot update original file!', 1350582054);
         }
-        $processingFolder = $this->originalFile->getStorage()->getProcessingFolder();
+        $processingFolder = $this->originalFile->getStorage()->getProcessingFolder($this->originalFile);
         $addedFile = $this->storage->updateProcessedFile($filePath, $this, $processingFolder);
 
         // Update some related properties
@@ -256,7 +256,7 @@ class ProcessedFile extends AbstractFile
 
         $this->name = $name;
         // @todo this is a *weird* hack that will fail if the storage is non-hierarchical!
-        $this->identifier = $this->storage->getProcessingFolder()->getIdentifier() . $this->name;
+        $this->identifier = $this->storage->getProcessingFolder($this->originalFile)->getIdentifier() . $this->name;
 
         $this->updated = true;
     }
index 10c559b..b6b222e 100644 (file)
@@ -108,7 +108,7 @@ class LocalImageProcessor implements ProcessorInterface
     {
         // the storage of the processed file, not of the original file!
         $storage = $task->getTargetFile()->getStorage();
-        $processingFolder = $storage->getProcessingFolder();
+        $processingFolder = $storage->getProcessingFolder($task->getSourceFile());
 
         // explicitly check for the raw filename here, as we check for files that existed before we even started
         // processing, i.e. that were processed earlier
index bf210e7..b8bb973 100644 (file)
@@ -155,6 +155,11 @@ class ResourceStorage implements ResourceStorageInterface
     protected $fileAndFolderNameFilters = array();
 
     /**
+     * Levels numbers used to generate hashed subfolders in the processing folder
+     */
+    const PROCESSING_FOLDER_LEVELS = 2;
+
+    /**
      * Constructor for a storage object.
      *
      * @param Driver\DriverInterface $driver
@@ -1204,10 +1209,9 @@ class ResourceStorage implements ResourceStorageInterface
             throw new \InvalidArgumentException('File "' . $localFilePath . '" does not exist.', 1319552746);
         }
         if ($processingFolder === null) {
-            $processingFolder = $this->getProcessingFolder();
+            $processingFolder = $this->getProcessingFolder($processedFile->getOriginalFile());
         }
         $fileIdentifier = $this->driver->addFile($localFilePath, $processingFolder->getIdentifier(), $processedFile->getName());
-
         // @todo check if we have to update the processed file other then the identifier
         $processedFile->setIdentifier($fileIdentifier);
         return $processedFile;
@@ -2856,9 +2860,10 @@ class ResourceStorage implements ResourceStorageInterface
      * Getter function to return the folder where the files can
      * be processed. Does not check for access rights here.
      *
+     * @param File $file Specific file you want to have the processing folder for
      * @return Folder
      */
-    public function getProcessingFolder()
+    public function getProcessingFolder(File $file = null)
     {
         if (!isset($this->processingFolder)) {
             $processingFolder = self::DEFAULT_ProcessingFolder;
@@ -2892,7 +2897,62 @@ class ResourceStorage implements ResourceStorageInterface
                 );
             }
         }
-        return $this->processingFolder;
+
+        $processingFolder = $this->processingFolder;
+        if (!empty($file)) {
+            $processingFolder = $this->getNestedProcessingFolder($file, $processingFolder);
+        }
+        return $processingFolder;
+    }
+
+    /**
+     * Getter function to return the the file's corresponding hashed subfolder
+     * of the processed folder
+     *
+     * @param File $file
+     * @param Folder $rootProcessingFolder
+     * @return Folder
+     * @throws Exception\InsufficientFolderWritePermissionsException
+     */
+    protected function getNestedProcessingFolder(File $file, Folder $rootProcessingFolder)
+    {
+        $processingFolder = $rootProcessingFolder;
+        $nestedFolderNames = $this->getNamesForNestedProcessingFolder(
+            $file->getIdentifier(),
+            self::PROCESSING_FOLDER_LEVELS
+        );
+
+        try {
+            foreach ($nestedFolderNames as $folderName) {
+                if ($processingFolder->hasFolder($folderName)) {
+                    $processingFolder = $processingFolder->getSubfolder($folderName);
+                } else {
+                    $processingFolder = $processingFolder->createFolder($folderName);
+                }
+            }
+        } catch (Exception\FolderDoesNotExistException $e) {}
+
+        return $processingFolder;
+    }
+
+    /**
+     * Generates appropriate hashed sub-folder path for a given file identifier
+     *
+     * @param string $fileIdentifier
+     * @param int $levels
+     * @return []
+     */
+    protected function getNamesForNestedProcessingFolder($fileIdentifier, $levels)
+    {
+        $names = [];
+        if ($levels === 0) {
+            return $names;
+        }
+        $hash = md5($fileIdentifier);
+        for ($i = 1; $i <= $levels; $i++) {
+            $names[] = substr($hash, $i, 1);
+        }
+        return $names;
     }
 
     /**
index 3357473..c8bdeac 100644 (file)
@@ -752,4 +752,36 @@ class ResourceStorageTest extends BaseTestCase
 
         $this->assertSame(FolderInterface::ROLE_DEFAULT, $role);
     }
+
+    /**
+     * @test
+     */
+    public function getProcessingRootFolderTest()
+    {
+        $this->prepareSubject(array());
+        $processingFolder = $this->subject->getProcessingFolder();
+
+        $this->assertInstanceOf(Folder::class, $processingFolder);
+    }
+
+    /**
+     * @test
+     */
+    public function getNestedProcessingFolderTest()
+    {
+        $mockedDriver = $this->createDriverMock(array('basePath' => $this->getMountRootUrl()), null, null);
+        $this->prepareSubject(array(), true, $mockedDriver);
+        $mockedFile = $this->getSimpleFileMock('/someFile');
+
+        $rootProcessingFolder = $this->subject->getProcessingFolder();
+        $processingFolder = $this->subject->getProcessingFolder($mockedFile);
+
+        $this->assertInstanceOf(Folder::class, $processingFolder);
+        $this->assertNotEquals($rootProcessingFolder, $processingFolder);
+
+        for ($i = ResourceStorage::PROCESSING_FOLDER_LEVELS; $i>0; $i--) {
+            $processingFolder = $processingFolder->getParentFolder();
+        }
+        $this->assertEquals($rootProcessingFolder, $processingFolder);
+    }
 }