[BUGFIX] Processing folder has now nested subfolders 54/47854/2
authorAlina Fleser <afleser@arxia.com>
Sat, 31 Jan 2015 15:57:55 +0000 (17:57 +0200)
committerPhilipp Gampe <philipp.gampe@typo3.org>
Fri, 22 Apr 2016 12:19:45 +0000 (14:19 +0200)
In case of big installations the amount of processed files in
one folder is too much and causes performance issues and other
problems.
To prevent this kind of issues, nested subfolders are now
created in the processing folder in order to split the processed
files into more than one folder.

Resolves: #56557
Releases: master, 7.6
Change-Id: Id9a4fc3b4bb4b28ca26ff96221097171c3835eb7
Reviewed-on: https://review.typo3.org/47854
Reviewed-by: Philipp Gampe <philipp.gampe@typo3.org>
Tested-by: Philipp Gampe <philipp.gampe@typo3.org>
typo3/sysext/core/Classes/Resource/ProcessedFile.php
typo3/sysext/core/Classes/Resource/Processing/LocalImageProcessor.php
typo3/sysext/core/Classes/Resource/ResourceStorage.php
typo3/sysext/core/Tests/Unit/Resource/ResourceStorageTest.php

index d2beda0..3304d44 100644 (file)
@@ -181,7 +181,7 @@ class ProcessedFile extends AbstractFile
         if ($this->identifier === null) {
             throw new \RuntimeException('Cannot update original file!', 1350582054);
         }
-        $processingFolder = $this->originalFile->getStorage()->getProcessingFolder();
+        $processingFolder = $this->originalFile->getStorage()->getProcessingFolder($this->originalFile);
         $addedFile = $this->storage->updateProcessedFile($filePath, $this, $processingFolder);
 
         // Update some related properties
@@ -251,7 +251,7 @@ class ProcessedFile extends AbstractFile
 
         $this->name = $name;
         // @todo this is a *weird* hack that will fail if the storage is non-hierarchical!
-        $this->identifier = $this->storage->getProcessingFolder()->getIdentifier() . $this->name;
+        $this->identifier = $this->storage->getProcessingFolder($this->originalFile)->getIdentifier() . $this->name;
 
         $this->updated = true;
     }
index fc6edaa..52964f1 100644 (file)
@@ -106,7 +106,7 @@ class LocalImageProcessor implements ProcessorInterface
     {
         // the storage of the processed file, not of the original file!
         $storage = $task->getTargetFile()->getStorage();
-        $processingFolder = $storage->getProcessingFolder();
+        $processingFolder = $storage->getProcessingFolder($task->getSourceFile());
 
         // explicitly check for the raw filename here, as we check for files that existed before we even started
         // processing, i.e. that were processed earlier
index e3f4cb1..1c364d9 100644 (file)
@@ -151,6 +151,11 @@ class ResourceStorage implements ResourceStorageInterface
     protected $fileAndFolderNameFilters = array();
 
     /**
+     * Levels numbers used to generate hashed subfolders in the processing folder
+     */
+    const PROCESSING_FOLDER_LEVELS = 2;
+
+    /**
      * Constructor for a storage object.
      *
      * @param Driver\DriverInterface $driver
@@ -1198,10 +1203,9 @@ class ResourceStorage implements ResourceStorageInterface
             throw new \InvalidArgumentException('File "' . $localFilePath . '" does not exist.', 1319552746);
         }
         if ($processingFolder === null) {
-            $processingFolder = $this->getProcessingFolder();
+            $processingFolder = $this->getProcessingFolder($processedFile->getOriginalFile());
         }
         $fileIdentifier = $this->driver->addFile($localFilePath, $processingFolder->getIdentifier(), $processedFile->getName());
-
         // @todo check if we have to update the processed file other then the identifier
         $processedFile->setIdentifier($fileIdentifier);
         return $processedFile;
@@ -2837,9 +2841,10 @@ class ResourceStorage implements ResourceStorageInterface
      * Getter function to return the folder where the files can
      * be processed. Does not check for access rights here.
      *
+     * @param File $file Specific file you want to have the processing folder for
      * @return Folder
      */
-    public function getProcessingFolder()
+    public function getProcessingFolder(File $file = null)
     {
         if (!isset($this->processingFolder)) {
             $processingFolder = self::DEFAULT_ProcessingFolder;
@@ -2873,7 +2878,62 @@ class ResourceStorage implements ResourceStorageInterface
                 );
             }
         }
-        return $this->processingFolder;
+
+        $processingFolder = $this->processingFolder;
+        if (!empty($file)) {
+            $processingFolder = $this->getNestedProcessingFolder($file, $processingFolder);
+        }
+        return $processingFolder;
+    }
+
+    /**
+     * Getter function to return the the file's corresponding hashed subfolder
+     * of the processed folder
+     *
+     * @param File $file
+     * @param Folder $rootProcessingFolder
+     * @return Folder
+     * @throws Exception\InsufficientFolderWritePermissionsException
+     */
+    protected function getNestedProcessingFolder(File $file, Folder $rootProcessingFolder)
+    {
+        $processingFolder = $rootProcessingFolder;
+        $nestedFolderNames = $this->getNamesForNestedProcessingFolder(
+            $file->getIdentifier(),
+            self::PROCESSING_FOLDER_LEVELS
+        );
+
+        try {
+            foreach ($nestedFolderNames as $folderName) {
+                if ($processingFolder->hasFolder($folderName)) {
+                    $processingFolder = $processingFolder->getSubfolder($folderName);
+                } else {
+                    $processingFolder = $processingFolder->createFolder($folderName);
+                }
+            }
+        } catch (Exception\FolderDoesNotExistException $e) {}
+
+        return $processingFolder;
+    }
+
+    /**
+     * Generates appropriate hashed sub-folder path for a given file identifier
+     *
+     * @param string $fileIdentifier
+     * @param int $levels
+     * @return []
+     */
+    protected function getNamesForNestedProcessingFolder($fileIdentifier, $levels)
+    {
+        $names = [];
+        if ($levels === 0) {
+            return $names;
+        }
+        $hash = md5($fileIdentifier);
+        for ($i = 1; $i <= $levels; $i++) {
+            $names[] = substr($hash, $i, 1);
+        }
+        return $names;
     }
 
     /**
index e848a41..0383b18 100644 (file)
@@ -750,4 +750,36 @@ class ResourceStorageTest extends BaseTestCase
 
         $this->assertSame(FolderInterface::ROLE_DEFAULT, $role);
     }
+
+    /**
+     * @test
+     */
+    public function getProcessingRootFolderTest()
+    {
+        $this->prepareSubject(array());
+        $processingFolder = $this->subject->getProcessingFolder();
+
+        $this->assertInstanceOf(Folder::class, $processingFolder);
+    }
+
+    /**
+     * @test
+     */
+    public function getNestedProcessingFolderTest()
+    {
+        $mockedDriver = $this->createDriverMock(array('basePath' => $this->getMountRootUrl()), null, null);
+        $this->prepareSubject(array(), true, $mockedDriver);
+        $mockedFile = $this->getSimpleFileMock('/someFile');
+
+        $rootProcessingFolder = $this->subject->getProcessingFolder();
+        $processingFolder = $this->subject->getProcessingFolder($mockedFile);
+
+        $this->assertInstanceOf(Folder::class, $processingFolder);
+        $this->assertNotEquals($rootProcessingFolder, $processingFolder);
+
+        for ($i = ResourceStorage::PROCESSING_FOLDER_LEVELS; $i>0; $i--) {
+            $processingFolder = $processingFolder->getParentFolder();
+        }
+        $this->assertEquals($rootProcessingFolder, $processingFolder);
+    }
 }