[TASK] Performance optimizations for the form manager module 75/58375/4
authorRalf Zimmermann <ralf.zimmermann@tritum.de>
Mon, 27 Aug 2018 23:10:07 +0000 (01:10 +0200)
committerBenni Mack <benni@typo3.org>
Sun, 28 Oct 2018 12:35:39 +0000 (13:35 +0100)
Speeds up the form manager module and the form plugin especially
if there are many forms within the TYPO3 instance.

Resolves: #86000
Releases: master, 8.7
Change-Id: Ic483029e0d1b1955d58e04496f97862c00b6d6a4
Reviewed-on: https://review.typo3.org/58375
Tested-by: TYPO3com <no-reply@typo3.com>
Reviewed-by: Susanne Moog <susanne.moog@typo3.org>
Tested-by: Susanne Moog <susanne.moog@typo3.org>
Reviewed-by: Benni Mack <benni@typo3.org>
Tested-by: Benni Mack <benni@typo3.org>
typo3/sysext/form/Classes/Controller/FormManagerController.php
typo3/sysext/form/Classes/Hooks/FormFileExtensionUpdate.php
typo3/sysext/form/Classes/Mvc/Persistence/FormPersistenceManager.php
typo3/sysext/form/Classes/Service/DatabaseService.php [new file with mode: 0644]
typo3/sysext/form/Tests/Unit/Controller/FormManagerControllerTest.php
typo3/sysext/form/Tests/Unit/Mvc/Persistence/FormPersistenceManagerTest.php

index 5bb5597..8a84b0b 100644 (file)
@@ -20,17 +20,16 @@ use TYPO3\CMS\Backend\Template\Components\ButtonBar;
 use TYPO3\CMS\Backend\Utility\BackendUtility;
 use TYPO3\CMS\Backend\View\BackendTemplateView;
 use TYPO3\CMS\Core\Authentication\BackendUserAuthentication;
-use TYPO3\CMS\Core\Database\ConnectionPool;
 use TYPO3\CMS\Core\Imaging\Icon;
 use TYPO3\CMS\Core\Imaging\IconFactory;
 use TYPO3\CMS\Core\Messaging\AbstractMessage;
 use TYPO3\CMS\Core\Page\PageRenderer;
-use TYPO3\CMS\Core\Resource\ResourceFactory;
 use TYPO3\CMS\Core\Utility\ArrayUtility;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 use TYPO3\CMS\Extbase\Mvc\View\JsonView;
 use TYPO3\CMS\Form\Exception as FormException;
 use TYPO3\CMS\Form\Mvc\Persistence\Exception\PersistenceManagerException;
+use TYPO3\CMS\Form\Service\DatabaseService;
 use TYPO3\CMS\Form\Service\TranslationService;
 use TYPO3\CMS\Lang\LanguageService;
 
@@ -43,6 +42,20 @@ class FormManagerController extends AbstractBackendController
 {
 
     /**
+     * @var DatabaseService
+     */
+    protected $databaseService;
+
+    /**
+     * @param \TYPO3\CMS\Form\Service\DatabaseService $databaseService
+     * @internal
+     */
+    public function injectDatabaseService(\TYPO3\CMS\Form\Service\DatabaseService $databaseService)
+    {
+        $this->databaseService = $databaseService;
+    }
+
+    /**
      * Default View Container
      *
      * @var BackendTemplateView
@@ -247,7 +260,7 @@ class FormManagerController extends AbstractBackendController
      */
     public function deleteAction(string $formPersistenceIdentifier)
     {
-        if (empty($this->getReferences($formPersistenceIdentifier))) {
+        if (empty($this->databaseService->getReferencesByPersistenceIdentifier($formPersistenceIdentifier))) {
             if (
                 isset($GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['ext/form']['beforeFormDelete'])
                 && is_array($GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['ext/form']['beforeFormDelete'])
@@ -345,12 +358,25 @@ class FormManagerController extends AbstractBackendController
      */
     protected function getAvailableFormDefinitions(): array
     {
+        $allReferencesForFileUid = $this->databaseService->getAllReferencesForFileUid();
+        $allReferencesForPersistenceIdentifier = $this->databaseService->getAllReferencesForPersistenceIdentifier();
+
         $availableFormDefinitions = [];
         foreach ($this->formPersistenceManager->listForms() as $formDefinition) {
-            $referenceCount = count($this->getReferences($formDefinition['persistenceIdentifier']));
+            $referenceCount  = 0;
+            if (
+                isset($formDefinition['fileUid'])
+                && array_key_exists($formDefinition['fileUid'], $allReferencesForFileUid)
+            ) {
+                $referenceCount = $allReferencesForFileUid[$formDefinition['fileUid']];
+            } elseif (array_key_exists($formDefinition['persistenceIdentifier'], $allReferencesForPersistenceIdentifier)) {
+                $referenceCount = $allReferencesForPersistenceIdentifier[$formDefinition['persistenceIdentifier']];
+            }
+
             $formDefinition['referenceCount'] = $referenceCount;
             $availableFormDefinitions[] = $formDefinition;
         }
+
         return $availableFormDefinitions;
     }
 
@@ -371,7 +397,7 @@ class FormManagerController extends AbstractBackendController
         $references = [];
         $iconFactory = GeneralUtility::makeInstance(IconFactory::class);
 
-        $referenceRows = $this->getReferences($persistenceIdentifier);
+        $referenceRows = $this->databaseService->getReferencesByPersistenceIdentifier($persistenceIdentifier);
         foreach ($referenceRows as &$referenceRow) {
             $record = $this->getRecord($referenceRow['tablename'], $referenceRow['recuid']);
             if (!$record) {
@@ -399,41 +425,6 @@ class FormManagerController extends AbstractBackendController
     }
 
     /**
-     * Returns an array with all sys_refindex database rows which be
-     * connected to a formDefinition identified by $persistenceIdentifier
-     *
-     * @param string $persistenceIdentifier
-     * @return array
-     * @throws \InvalidArgumentException
-     */
-    protected function getReferences(string $persistenceIdentifier): array
-    {
-        if (empty($persistenceIdentifier)) {
-            throw new \InvalidArgumentException('$persistenceIdentifier must not be empty.', 1472238493);
-        }
-
-        $resourceFactory = GeneralUtility::makeInstance(ResourceFactory::class);
-        $file = $resourceFactory->retrieveFileOrFolderObject($persistenceIdentifier);
-
-        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('sys_refindex');
-        $referenceRows = $queryBuilder
-            ->select('*')
-            ->from('sys_refindex')
-            ->where(
-                $queryBuilder->expr()->eq('deleted', 0),
-                $queryBuilder->expr()->eq('softref_key', $queryBuilder->createNamedParameter('formPersistenceIdentifier', \PDO::PARAM_STR)),
-                $queryBuilder->expr()->orX(
-                    $queryBuilder->expr()->eq('ref_string', $queryBuilder->createNamedParameter($persistenceIdentifier, \PDO::PARAM_STR)),
-                    $queryBuilder->expr()->eq('ref_uid', $queryBuilder->createNamedParameter($file->getUid(), \PDO::PARAM_INT))
-                ),
-                $queryBuilder->expr()->eq('tablename', $queryBuilder->createNamedParameter('tt_content', \PDO::PARAM_STR))
-            )
-            ->execute()
-            ->fetchAll();
-        return $referenceRows;
-    }
-
-    /**
      * Check if a given $templatePath for a given $prototypeName is valid
      * and accessible.
      *
index 0ac1c13..7f51228 100644 (file)
@@ -27,7 +27,6 @@ use TYPO3\CMS\Core\Utility\GeneralUtility;
 use TYPO3\CMS\Core\Utility\PathUtility;
 use TYPO3\CMS\Core\Utility\StringUtility;
 use TYPO3\CMS\Extbase\Object\ObjectManager;
-use TYPO3\CMS\Form\Mvc\Configuration\YamlSource;
 use TYPO3\CMS\Form\Mvc\Persistence\FormPersistenceManager;
 use TYPO3\CMS\Form\Slot\FilePersistenceSlot;
 use TYPO3\CMS\Install\Updates\AbstractUpdate;
@@ -50,11 +49,6 @@ class FormFileExtensionUpdate extends AbstractUpdate
     protected $persistenceManager;
 
     /**
-     * @var YamlSource
-     */
-    protected $yamlSource;
-
-    /**
      * @var ResourceFactory
      */
     protected $resourceFactory;
@@ -86,7 +80,6 @@ class FormFileExtensionUpdate extends AbstractUpdate
         $information = [];
 
         $this->persistenceManager = $this->getObjectManager()->get(FormPersistenceManager::class);
-        $this->yamlSource = $this->getObjectManager()->get(YamlSource::class);
         $this->resourceFactory = GeneralUtility::makeInstance(ResourceFactory::class);
 
         foreach ($this->getFormDefinitionsInformation() as $formDefinitionInformation) {
@@ -164,7 +157,6 @@ class FormFileExtensionUpdate extends AbstractUpdate
 
         $this->connection = $connectionPool->getConnectionForTable('tt_content');
         $this->persistenceManager = $this->getObjectManager()->get(FormPersistenceManager::class);
-        $this->yamlSource = $this->getObjectManager()->get(YamlSource::class);
         $this->resourceFactory = GeneralUtility::makeInstance(ResourceFactory::class);
         $this->referenceIndex = GeneralUtility::makeInstance(ReferenceIndex::class);
         $this->flexFormTools = GeneralUtility::makeInstance(FlexFormTools::class);
@@ -684,7 +676,8 @@ class FormFileExtensionUpdate extends AbstractUpdate
     protected function getFormDefinition(File $file): array
     {
         try {
-            $formDefinition = $this->yamlSource->load([$file]);
+            $rawYamlContent = $file->getContents();
+            $formDefinition = $this->extractMetaDataFromCouldBeFormDefinition($rawYamlContent);
 
             if (!$this->looksLikeAFormDefinition($formDefinition)) {
                 $formDefinition = [];
@@ -697,6 +690,35 @@ class FormFileExtensionUpdate extends AbstractUpdate
     }
 
     /**
+     * @param string $maybeRawFormDefinition
+     * @return array
+     */
+    protected function extractMetaDataFromCouldBeFormDefinition(string $maybeRawFormDefinition): array
+    {
+        $metaDataProperties = ['identifier', 'type', 'label', 'prototypeName'];
+        $metaData = [];
+        foreach (explode("\n", $maybeRawFormDefinition) as $line) {
+            if (empty($line) || $line[0] === ' ') {
+                continue;
+            }
+
+            list($key, $value) = explode(':', $line);
+            if (
+                empty($key)
+                || empty($value)
+                || !in_array($key, $metaDataProperties)
+            ) {
+                continue;
+            }
+
+            $value = trim($value, ' \'"');
+            $metaData[$key] = $value;
+        }
+
+        return $metaData;
+    }
+
+    /**
      * @return array
      */
     protected function getAllFlexformFieldsFromFormPlugins(): array
index 63551b3..4ca4cda 100644 (file)
@@ -17,11 +17,14 @@ namespace TYPO3\CMS\Form\Mvc\Persistence;
  * The TYPO3 project - inspiring people to share!
  */
 
+use TYPO3\CMS\Core\Cache\CacheManager;
+use TYPO3\CMS\Core\Cache\Frontend\FrontendInterface;
 use TYPO3\CMS\Core\Resource\Exception\FolderDoesNotExistException;
 use TYPO3\CMS\Core\Resource\Exception\InsufficientFolderAccessPermissionsException;
 use TYPO3\CMS\Core\Resource\File;
 use TYPO3\CMS\Core\Resource\Filter\FileExtensionFilter;
 use TYPO3\CMS\Core\Resource\Folder;
+use TYPO3\CMS\Core\Resource\ResourceFactory;
 use TYPO3\CMS\Core\Resource\ResourceStorage;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 use TYPO3\CMS\Core\Utility\PathUtility;
@@ -29,6 +32,7 @@ use TYPO3\CMS\Core\Utility\StringUtility;
 use TYPO3\CMS\Extbase\Object\ObjectManager;
 use TYPO3\CMS\Form\Mvc\Configuration\ConfigurationManagerInterface;
 use TYPO3\CMS\Form\Mvc\Configuration\Exception\FileWriteException;
+use TYPO3\CMS\Form\Mvc\Configuration\Exception\NoSuchFileException;
 use TYPO3\CMS\Form\Mvc\Persistence\Exception\NoUniqueIdentifierException;
 use TYPO3\CMS\Form\Mvc\Persistence\Exception\NoUniquePersistenceIdentifierException;
 use TYPO3\CMS\Form\Mvc\Persistence\Exception\PersistenceManagerException;
@@ -64,6 +68,16 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
     protected $filePersistenceSlot;
 
     /**
+     * @var FrontendInterface
+     */
+    protected $runtimeCache;
+
+    /**
+     * @var ResourceFactory
+     */
+    protected $resourceFactory;
+
+    /**
      * @param \TYPO3\CMS\Form\Mvc\Configuration\YamlSource $yamlSource
      * @internal
      */
@@ -82,14 +96,22 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
     }
 
     /**
-     * @param FilePersistenceSlot $filePersistenceSlot
+     * @param \TYPO3\CMS\Form\Slot\FilePersistenceSlot $filePersistenceSlot
      */
-    public function injectFilePersistenceSlot(FilePersistenceSlot $filePersistenceSlot)
+    public function injectFilePersistenceSlot(\TYPO3\CMS\Form\Slot\FilePersistenceSlot $filePersistenceSlot)
     {
         $this->filePersistenceSlot = $filePersistenceSlot;
     }
 
     /**
+     * @param \TYPO3\CMS\Core\Resource\ResourceFactory $resourceFactory
+     */
+    public function injectResourceFactory(\TYPO3\CMS\Core\Resource\ResourceFactory $resourceFactory)
+    {
+        $this->resourceFactory = $resourceFactory;
+    }
+
+    /**
      * @internal
      */
     public function initializeObject()
@@ -97,6 +119,7 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
         $this->formSettings = GeneralUtility::makeInstance(ObjectManager::class)
             ->get(ConfigurationManagerInterface::class)
             ->getConfiguration(ConfigurationManagerInterface::CONFIGURATION_TYPE_YAML_SETTINGS, 'form');
+        $this->runtimeCache = GeneralUtility::makeInstance(CacheManager::class)->getCache('cache_runtime');
     }
 
     /**
@@ -105,34 +128,22 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
      *
      * @param string $persistenceIdentifier
      * @return array
-     * @throws PersistenceManagerException
      * @internal
      */
     public function load(string $persistenceIdentifier): array
     {
-        if (pathinfo($persistenceIdentifier, PATHINFO_EXTENSION) !== 'yaml') {
-            throw new PersistenceManagerException(sprintf('The file "%s" could not be loaded.', $persistenceIdentifier), 1477679819);
-        }
+        $cacheKey = 'formLoad' . md5($persistenceIdentifier);
 
-        if (strpos($persistenceIdentifier, 'EXT:') === 0) {
-            if (!array_key_exists(pathinfo($persistenceIdentifier, PATHINFO_DIRNAME) . '/', $this->getAccessibleExtensionFolders())) {
-                throw new PersistenceManagerException(sprintf('The file "%s" could not be loaded.', $persistenceIdentifier), 1484071985);
-            }
-            $file = $persistenceIdentifier;
-        } else {
-            $file = $this->getFileByIdentifier($persistenceIdentifier);
+        $yaml = $this->runtimeCache->get($cacheKey);
+        if ($yaml !== false) {
+            return $yaml;
         }
 
+        $file = $this->retrieveFileByPersistenceIdentifier($persistenceIdentifier);
+
         try {
             $yaml = $this->yamlSource->load([$file]);
-
-            if (isset($yaml['identifier'], $yaml['type'])
-                && $yaml['type'] === 'Form'
-                && !$this->hasValidFileExtension($persistenceIdentifier)
-                && strpos($persistenceIdentifier, 'EXT:') !== 0
-            ) {
-                throw new PersistenceManagerException(sprintf('Form definition "%s" does not end with ".form.yaml".', $persistenceIdentifier), 1531160649);
-            }
+            $this->generateErrorsIfFormDefinitionIsValidButHasInvalidFileExtension($yaml, $persistenceIdentifier);
         } catch (\Exception $e) {
             $yaml = [
                 'type' => 'Form',
@@ -141,6 +152,7 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
                 'invalid' => true,
             ];
         }
+        $this->runtimeCache->set($cacheKey, $yaml);
 
         return $yaml;
     }
@@ -168,8 +180,9 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
             if (!$this->formSettings['persistenceManager']['allowSaveToExtensionPaths']) {
                 throw new PersistenceManagerException('Save to extension paths is not allowed.', 1477680881);
             }
-            if (!array_key_exists(pathinfo($persistenceIdentifier, PATHINFO_DIRNAME) . '/', $this->getAccessibleExtensionFolders())) {
-                throw new PersistenceManagerException(sprintf('The file "%s" could not be saved.', $persistenceIdentifier), 1484073571);
+            if (!$this->isFileWithinAccessibleExtensionFolders($persistenceIdentifier)) {
+                $message = sprintf('The file "%s" could not be saved. Please check your configuration option "persistenceManager.allowedExtensionPaths"', $persistenceIdentifier);
+                throw new PersistenceManagerException($message, 1484073571);
             }
             $fileToSave = GeneralUtility::getFileAbsFileName($persistenceIdentifier);
         } else {
@@ -207,8 +220,9 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
             if (!$this->formSettings['persistenceManager']['allowDeleteFromExtensionPaths']) {
                 throw new PersistenceManagerException(sprintf('The file "%s" could not be removed.', $persistenceIdentifier), 1472239536);
             }
-            if (!array_key_exists(pathinfo($persistenceIdentifier, PATHINFO_DIRNAME) . '/', $this->getAccessibleExtensionFolders())) {
-                throw new PersistenceManagerException(sprintf('The file "%s" could not be removed.', $persistenceIdentifier), 1484073878);
+            if (!$this->isFileWithinAccessibleExtensionFolders($persistenceIdentifier)) {
+                $message = sprintf('The file "%s" could not be removed. Please check your configuration option "persistenceManager.allowedExtensionPaths"', $persistenceIdentifier);
+                throw new PersistenceManagerException($message, 1484073878);
             }
             $fileToDelete = GeneralUtility::getFileAbsFileName($persistenceIdentifier);
             unlink($fileToDelete);
@@ -235,7 +249,7 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
         $exists = false;
         if ($this->hasValidFileExtension($persistenceIdentifier)) {
             if (strpos($persistenceIdentifier, 'EXT:') === 0) {
-                if (array_key_exists(pathinfo($persistenceIdentifier, PATHINFO_DIRNAME) . '/', $this->getAccessibleExtensionFolders())) {
+                if ($this->isFileWithinAccessibleExtensionFolders($persistenceIdentifier)) {
                     $exists = file_exists(GeneralUtility::getFileAbsFileName($persistenceIdentifier));
                 }
             } else {
@@ -263,13 +277,13 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
         $forms = [];
 
         foreach ($this->retrieveYamlFilesFromStorageFolders() as $file) {
-            $persistenceIdentifier = $file->getCombinedIdentifier();
+            $form = $this->loadMetaData($file);
 
-            $form = $this->load($persistenceIdentifier);
-            if (empty($form['identifier']) || ($form['type'] ?? null) !== 'Form') {
+            if (!$this->looksLikeAFormDefinition($form)) {
                 continue;
             }
 
+            $persistenceIdentifier = $file->getCombinedIdentifier();
             if ($this->hasValidFileExtension($persistenceIdentifier)) {
                 $forms[] = [
                     'identifier' => $form['identifier'],
@@ -280,6 +294,7 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
                     'location' => 'storage',
                     'duplicateIdentifier' => false,
                     'invalid' => $form['invalid'],
+                    'fileUid' => $form['fileUid'],
                 ];
                 $identifiers[$form['identifier']]++;
             } else {
@@ -293,13 +308,15 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
                     'duplicateIdentifier' => false,
                     'invalid' => false,
                     'deprecatedFileExtension' => true,
+                    'fileUid' => $form['fileUid'],
                 ];
             }
         }
 
         foreach ($this->retrieveYamlFilesFromExtensionFolders() as $fullPath => $fileName) {
-            $form = $this->load($fullPath);
-            if (isset($form['identifier'], $form['type']) && $form['type'] === 'Form') {
+            $form = $this->loadMetaData($fullPath);
+
+            if ($this->looksLikeAFormDefinition($form)) {
                 if ($this->hasValidFileExtension($fileName)) {
                     $forms[] = [
                         'identifier' => $form['identifier'],
@@ -310,6 +327,7 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
                         'location' => 'extension',
                         'duplicateIdentifier' => false,
                         'invalid' => $form['invalid'],
+                        'fileUid' => $form['fileUid'],
                     ];
                     $identifiers[$form['identifier']]++;
                 } else {
@@ -323,6 +341,7 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
                         'duplicateIdentifier' => false,
                         'invalid' => false,
                         'deprecatedFileExtension' => true,
+                        'fileUid' => $form['fileUid'],
                     ];
                 }
             }
@@ -454,12 +473,19 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
      */
     public function getAccessibleExtensionFolders(): array
     {
+        $extensionFolders = $this->runtimeCache->get('formAccessibleExtensionFolders');
+
+        if ($extensionFolders !== false) {
+            return $extensionFolders;
+        }
+
         $extensionFolders = [];
         if (
             !isset($this->formSettings['persistenceManager']['allowedExtensionPaths'])
             || !is_array($this->formSettings['persistenceManager']['allowedExtensionPaths'])
             || empty($this->formSettings['persistenceManager']['allowedExtensionPaths'])
         ) {
+            $this->runtimeCache->set('formAccessibleExtensionFolders', $extensionFolders);
             return $extensionFolders;
         }
 
@@ -475,6 +501,8 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
             $allowedExtensionPath = rtrim($allowedExtensionPath, '/') . '/';
             $extensionFolders[$allowedExtensionPath] = $allowedExtensionFullPath;
         }
+
+        $this->runtimeCache->set('formAccessibleExtensionFolders', $extensionFolders);
         return $extensionFolders;
     }
 
@@ -564,24 +592,6 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
     }
 
     /**
-     * Returns a File object for a given $persistenceIdentifier
-     *
-     * @param string $persistenceIdentifier
-     * @return File
-     * @throws PersistenceManagerException
-     */
-    protected function getFileByIdentifier(string $persistenceIdentifier): File
-    {
-        list($storageUid, $fileIdentifier) = explode(':', $persistenceIdentifier, 2);
-        $storage = $this->getStorageByUid((int)$storageUid);
-        $file = $storage->getFile($fileIdentifier);
-        if (!$storage->checkFileActionPermission('read', $file)) {
-            throw new PersistenceManagerException(sprintf('No read access to file "%s".', $persistenceIdentifier), 1471630578);
-        }
-        return $file;
-    }
-
-    /**
      * Returns a File object for a given $persistenceIdentifier.
      * If no file for this identifier exists a new object will be
      * created.
@@ -642,6 +652,125 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
     }
 
     /**
+     * @param string|File $persistenceIdentifier
+     * @return array
+     * @throws NoSuchFileException
+     */
+    protected function loadMetaData($persistenceIdentifier): array
+    {
+        if ($persistenceIdentifier instanceof File) {
+            $file = $persistenceIdentifier;
+            $persistenceIdentifier = $file->getCombinedIdentifier();
+        } else {
+            $file = $this->retrieveFileByPersistenceIdentifier($persistenceIdentifier);
+        }
+
+        try {
+            $rawYamlContent = $file->getContents();
+
+            if ($rawYamlContent === false) {
+                throw new NoSuchFileException(sprintf('YAML file "%s" could not be loaded', $persistenceIdentifier), 1524684462);
+            }
+
+            $yaml = $this->extractMetaDataFromCouldBeFormDefinition($rawYamlContent);
+            $this->generateErrorsIfFormDefinitionIsValidButHasInvalidFileExtension($yaml, $persistenceIdentifier);
+            $yaml['fileUid'] = $file->getUid();
+        } catch (\Exception $e) {
+            $yaml = [
+                'type' => 'Form',
+                'identifier' => $persistenceIdentifier,
+                'label' => $e->getMessage(),
+                'invalid' => true,
+            ];
+        }
+
+        return $yaml;
+    }
+
+    /**
+     * @param string $maybeRawFormDefinition
+     * @return array
+     */
+    protected function extractMetaDataFromCouldBeFormDefinition(string $maybeRawFormDefinition): array
+    {
+        $metaDataProperties = ['identifier', 'type', 'label', 'prototypeName'];
+        $metaData = [];
+        foreach (explode(LF, $maybeRawFormDefinition) as $line) {
+            if (empty($line) || $line[0] === ' ') {
+                continue;
+            }
+
+            list($key, $value) = explode(':', $line);
+            if (
+                empty($key)
+                || empty($value)
+                || !in_array($key, $metaDataProperties, true)
+            ) {
+                continue;
+            }
+
+            $value = trim($value, ' \'"');
+            $metaData[$key] = $value;
+        }
+
+        return $metaData;
+    }
+
+    /**
+     * @param array $formDefinition
+     * @param string $persistenceIdentifier
+     * @throws PersistenceManagerException
+     */
+    protected function generateErrorsIfFormDefinitionIsValidButHasInvalidFileExtension(array $formDefinition, string $persistenceIdentifier)
+    {
+        if (
+            $this->looksLikeAFormDefinition($formDefinition)
+            && !$this->hasValidFileExtension($persistenceIdentifier)
+            && strpos($persistenceIdentifier, 'EXT:') !== 0
+        ) {
+            throw new PersistenceManagerException(sprintf('Form definition "%s" does not end with ".form.yaml".', $persistenceIdentifier), 1531160649);
+        }
+    }
+
+    /**
+     * @param string $persistenceIdentifier
+     * @return File
+     * @throws PersistenceManagerException
+     * @throws NoSuchFileException
+     */
+    protected function retrieveFileByPersistenceIdentifier(string $persistenceIdentifier): File
+    {
+        if (pathinfo($persistenceIdentifier, PATHINFO_EXTENSION) !== 'yaml') {
+            throw new PersistenceManagerException(sprintf('The file "%s" could not be loaded.', $persistenceIdentifier), 1477679819);
+        }
+
+        if (
+            strpos($persistenceIdentifier, 'EXT:') === 0
+            && !$this->isFileWithinAccessibleExtensionFolders($persistenceIdentifier)
+        ) {
+            $message = sprintf('The file "%s" could not be loaded. Please check your configuration option "persistenceManager.allowedExtensionPaths"', $persistenceIdentifier);
+            throw new PersistenceManagerException($message, 1484071985);
+        }
+
+        try {
+            $file = $this->resourceFactory->retrieveFileOrFolderObject($persistenceIdentifier);
+        } catch (\Exception $e) {
+            // Top level catch to ensure useful following exception handling, because FAL throws top level exceptions.
+            $file = null;
+        }
+
+        if ($file === null) {
+            throw new NoSuchFileException(sprintf('YAML file "%s" could not be loaded', $persistenceIdentifier), 1524684442);
+        }
+
+        if (!$file->getStorage()->checkFileActionPermission('read', $file)) {
+            throw new PersistenceManagerException(sprintf('No read access to file "%s".', $persistenceIdentifier), 1471630578);
+        }
+
+        return $file;
+    }
+
+    /**
      * @param string $fileName
      * @return bool
      */
@@ -649,4 +778,23 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
     {
         return StringUtility::endsWith($fileName, self::FORM_DEFINITION_FILE_EXTENSION);
     }
+
+    /**
+     * @param string $fileName
+     * @return bool
+     */
+    protected function isFileWithinAccessibleExtensionFolders(string $fileName): bool
+    {
+        $dirName = rtrim(PathUtility::pathinfo($fileName, PATHINFO_DIRNAME), '/') . '/';
+        return array_key_exists($dirName, $this->getAccessibleExtensionFolders());
+    }
+
+    /**
+     * @param array $data
+     * @return bool
+     */
+    protected function looksLikeAFormDefinition(array $data): bool
+    {
+        return isset($data['identifier'], $data['type']) && !empty($data['identifier']) && $data['type'] === 'Form';
+    }
 }
diff --git a/typo3/sysext/form/Classes/Service/DatabaseService.php b/typo3/sysext/form/Classes/Service/DatabaseService.php
new file mode 100644 (file)
index 0000000..d4c4f40
--- /dev/null
@@ -0,0 +1,134 @@
+<?php
+declare(strict_types = 1);
+namespace TYPO3\CMS\Form\Service;
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+use TYPO3\CMS\Core\Database\ConnectionPool;
+use TYPO3\CMS\Core\Resource\ResourceFactory;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
+/**
+ * This class is subjected to change.
+ * **Do NOT subclass**
+ *
+ * Scope: frontend / backend
+ * @internal
+ */
+class DatabaseService
+{
+
+    /**
+     * Returns an array with all sys_refindex database rows which be
+     * connected to a formDefinition identified by $persistenceIdentifier
+     *
+     * @param string $persistenceIdentifier
+     * @return array
+     * @throws \InvalidArgumentException
+     * @internal
+     */
+    public function getReferencesByPersistenceIdentifier(string $persistenceIdentifier): array
+    {
+        if (empty($persistenceIdentifier)) {
+            throw new \InvalidArgumentException('$persistenceIdentifier must not be empty.', 1472238493);
+        }
+
+        $resourceFactory = GeneralUtility::makeInstance(ResourceFactory::class);
+        $file = $resourceFactory->retrieveFileOrFolderObject($persistenceIdentifier);
+
+        if ($file === null) {
+            return [];
+        }
+
+        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('sys_refindex');
+
+        return $queryBuilder
+            ->select('*')
+            ->from('sys_refindex')
+            ->where(
+                $queryBuilder->expr()->eq('deleted', 0),
+                $queryBuilder->expr()->eq('softref_key', $queryBuilder->createNamedParameter('formPersistenceIdentifier', \PDO::PARAM_STR)),
+                $queryBuilder->expr()->orX(
+                    $queryBuilder->expr()->eq('ref_string', $queryBuilder->createNamedParameter($persistenceIdentifier, \PDO::PARAM_STR)),
+                    $queryBuilder->expr()->eq('ref_uid', $queryBuilder->createNamedParameter($file->getUid(), \PDO::PARAM_INT))
+                ),
+                $queryBuilder->expr()->eq('tablename', $queryBuilder->createNamedParameter('tt_content', \PDO::PARAM_STR))
+            )
+            ->execute()
+            ->fetchAll();
+    }
+
+    /**
+     * Returns an array with all form definition persistenceIdentifiers
+     * as keys and their reference counts as values.
+     *
+     * @return array
+     * @internal
+     */
+    public function getAllReferencesForPersistenceIdentifier(): array
+    {
+        $items = [];
+        foreach ($this->getAllReferences('ref_string') as $item) {
+            $items[$item['identifier']] = $item['items'];
+        }
+        return $items;
+    }
+
+    /**
+     * Returns an array with all form definition file uids as keys
+     * and their reference counts as values.
+     *
+     * @return array
+     * @internal
+     */
+    public function getAllReferencesForFileUid(): array
+    {
+        $items = [];
+        foreach ($this->getAllReferences('ref_uid') as $item) {
+            $items[$item['identifier']] = $item['items'];
+        }
+        return $items;
+    }
+
+    /**
+     * @param string $column
+     * @return array
+     * @throws \InvalidArgumentException
+     */
+    protected function getAllReferences(string $column): array
+    {
+        if ($column !== 'ref_string' && $column !== 'ref_uid') {
+            throw new \InvalidArgumentException('$column must not be "ref_string" or "ref_uid".', 1535406600);
+        }
+
+        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('sys_refindex');
+
+        $constraints = [$queryBuilder->expr()->eq('softref_key', $queryBuilder->createNamedParameter('formPersistenceIdentifier', \PDO::PARAM_STR))];
+
+        if ($column === 'ref_string') {
+            $constraints[] = $queryBuilder->expr()->neq('ref_string', $queryBuilder->createNamedParameter('', \PDO::PARAM_STR));
+        } else {
+            $constraints[] = $queryBuilder->expr()->gt('ref_uid', $queryBuilder->createNamedParameter(0, \PDO::PARAM_INT));
+        }
+
+        return $queryBuilder
+            ->select($column . ' AS identifier')
+            ->addSelectLiteral('COUNT(' . $queryBuilder->quoteIdentifier($column) . ') AS ' . $queryBuilder->quoteIdentifier('items'))
+            ->from('sys_refindex')
+            ->where(...$constraints)
+            ->groupBy($column)
+            ->execute()
+            ->fetchAll();
+    }
+}
index 83e960b..5cc6eb3 100644 (file)
@@ -23,6 +23,7 @@ use TYPO3\CMS\Extbase\Mvc\Web\Routing\UriBuilder;
 use TYPO3\CMS\Extbase\Object\ObjectManager;
 use TYPO3\CMS\Form\Controller\FormManagerController;
 use TYPO3\CMS\Form\Mvc\Persistence\FormPersistenceManager;
+use TYPO3\CMS\Form\Service\DatabaseService;
 use TYPO3\CMS\Form\Service\TranslationService;
 
 /**
@@ -189,12 +190,15 @@ class FormManagerControllerTest extends \TYPO3\TestingFramework\Core\Unit\UnitTe
     public function getAvailableFormDefinitionsReturnsProcessedArray()
     {
         $mockController = $this->getAccessibleMock(FormManagerController::class, [
-            'getReferences'
+            'dummy'
         ], [], '', false);
 
         $formPersistenceManagerProphecy = $this->prophesize(FormPersistenceManager::class);
         $mockController->_set('formPersistenceManager', $formPersistenceManagerProphecy->reveal());
 
+        $databaseService = $this->prophesize(DatabaseService::class);
+        $mockController->_set('databaseService', $databaseService->reveal());
+
         $formPersistenceManagerProphecy->listForms(Argument::cetera())->willReturn([
             0 => [
                 'identifier' => 'ext-form-identifier',
@@ -207,13 +211,13 @@ class FormManagerControllerTest extends \TYPO3\TestingFramework\Core\Unit\UnitTe
             ],
         ]);
 
-        $mockController
-            ->expects($this->any())
-            ->method('getReferences')
-            ->willReturn([
-                'someRow',
-                'anotherRow',
-            ]);
+        $databaseService->getAllReferencesForFileUid(Argument::cetera())->willReturn([
+            0 => 0,
+        ]);
+
+        $databaseService->getAllReferencesForPersistenceIdentifier(Argument::cetera())->willReturn([
+            '1:/user_uploads/someFormName.yaml' => 2,
+        ]);
 
         $expected = [
             0 => [
@@ -255,9 +259,18 @@ class FormManagerControllerTest extends \TYPO3\TestingFramework\Core\Unit\UnitTe
             'getModuleUrl',
             'getRecord',
             'getRecordTitle',
-            'getReferences',
         ], [], '', false);
 
+        $databaseService = $this->prophesize(DatabaseService::class);
+        $mockController->_set('databaseService', $databaseService->reveal());
+
+        $databaseService->getReferencesByPersistenceIdentifier(Argument::cetera())->willReturn([
+            0 => [
+                'tablename' => 'tt_content',
+                'recuid' => -1,
+            ],
+        ]);
+
         $mockController
             ->expects($this->any())
             ->method('getModuleUrl')
@@ -273,16 +286,6 @@ class FormManagerControllerTest extends \TYPO3\TestingFramework\Core\Unit\UnitTe
             ->method('getRecordTitle')
             ->willReturn('record title');
 
-        $mockController
-            ->expects($this->any())
-            ->method('getReferences')
-            ->willReturn([
-                0 => [
-                    'tablename' => 'tt_content',
-                    'recuid' => -1,
-                ],
-            ]);
-
         $expected = [
             0 => [
                 'recordPageTitle' => 'record title',
index 077d5b0..8eeb9b9 100644 (file)
@@ -14,7 +14,9 @@ namespace TYPO3\CMS\Form\Tests\Unit\Mvc\Persistence;
  * The TYPO3 project - inspiring people to share!
  */
 
+use TYPO3\CMS\Core\Cache\Frontend\VariableFrontend;
 use TYPO3\CMS\Core\Resource\File;
+use TYPO3\CMS\Core\Resource\ResourceFactory;
 use TYPO3\CMS\Core\Resource\ResourceStorage;
 use TYPO3\CMS\Core\Resource\StorageRepository;
 use TYPO3\CMS\Form\Mvc\Persistence\Exception\NoUniqueIdentifierException;
@@ -39,6 +41,18 @@ class FormPersistenceManagerTest extends \TYPO3\TestingFramework\Core\Unit\UnitT
             'dummy'
         ], [], '', false);
 
+        $runtimeCache= $this->getMockBuilder(VariableFrontend::class)
+            ->setMethods(['get', 'set'])
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $runtimeCache
+            ->expects($this->any())
+            ->method('get')
+            ->willReturn(false);
+
+        $mockFormPersistenceManager->_set('runtimeCache', $runtimeCache);
+
         $input = '-1:/user_uploads/_example.php';
         $mockFormPersistenceManager->_call('load', $input);
     }
@@ -55,6 +69,18 @@ class FormPersistenceManagerTest extends \TYPO3\TestingFramework\Core\Unit\UnitT
             'dummy'
         ], [], '', false);
 
+        $runtimeCache= $this->getMockBuilder(VariableFrontend::class)
+            ->setMethods(['get', 'set'])
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $runtimeCache
+            ->expects($this->any())
+            ->method('get')
+            ->willReturn(false);
+
+        $mockFormPersistenceManager->_set('runtimeCache', $runtimeCache);
+
         $mockFormPersistenceManager->_set('formSettings', [
             'persistenceManager' => [
                 'allowedExtensionPaths' => [],
@@ -115,6 +141,18 @@ class FormPersistenceManagerTest extends \TYPO3\TestingFramework\Core\Unit\UnitT
             'dummy'
         ], [], '', false);
 
+        $runtimeCache= $this->getMockBuilder(VariableFrontend::class)
+            ->setMethods(['get', 'set'])
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $runtimeCache
+            ->expects($this->any())
+            ->method('get')
+            ->willReturn(false);
+
+        $mockFormPersistenceManager->_set('runtimeCache', $runtimeCache);
+
         $mockFormPersistenceManager->_set('formSettings', [
             'persistenceManager' => [
                 'allowSaveToExtensionPaths' => true,
@@ -202,6 +240,18 @@ class FormPersistenceManagerTest extends \TYPO3\TestingFramework\Core\Unit\UnitT
             'exists'
         ], [], '', false);
 
+        $runtimeCache= $this->getMockBuilder(VariableFrontend::class)
+            ->setMethods(['get', 'set'])
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $runtimeCache
+            ->expects($this->any())
+            ->method('get')
+            ->willReturn(false);
+
+        $mockFormPersistenceManager->_set('runtimeCache', $runtimeCache);
+
         $mockFormPersistenceManager
             ->expects($this->any())
             ->method('exists')
@@ -269,6 +319,18 @@ class FormPersistenceManagerTest extends \TYPO3\TestingFramework\Core\Unit\UnitT
             'dummy'
         ], [], '', false);
 
+        $runtimeCache= $this->getMockBuilder(VariableFrontend::class)
+            ->setMethods(['get', 'set'])
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $runtimeCache
+            ->expects($this->any())
+            ->method('get')
+            ->willReturn(false);
+
+        $mockFormPersistenceManager->_set('runtimeCache', $runtimeCache);
+
         $mockFormPersistenceManager->_set('formSettings', [
             'persistenceManager' => [
                 'allowedExtensionPaths' => [
@@ -557,37 +619,39 @@ class FormPersistenceManagerTest extends \TYPO3\TestingFramework\Core\Unit\UnitT
     /**
      * @test
      */
-    public function getFileByIdentifierThrowsExceptionIfReadFromStorageIsNotAllowed()
+    public function retrieveFileByPersistenceIdentifierThrowsExceptionIfReadFromStorageIsNotAllowed()
     {
         $this->expectException(PersistenceManagerException::class);
         $this->expectExceptionCode(1471630578);
 
         $mockFormPersistenceManager = $this->getAccessibleMock(FormPersistenceManager::class, [
-            'getStorageByUid',
+            'dummy',
         ], [], '', false);
 
-        $mockStorage = $this->getMockBuilder(ResourceStorage::class)
+        $storage = $this->getMockBuilder(ResourceStorage::class)
             ->disableOriginalConstructor()
             ->getMock();
 
-        $mockStorage
+        $storage
             ->expects($this->any())
             ->method('checkFileActionPermission')
             ->willReturn(false);
 
-        $file = new File(['identifier' => '', 'mime_type' => ''], $mockStorage);
-        $mockStorage
+        $file = new File(['identifier' => '', 'mime_type' => ''], $storage);
+
+        $resourceFactory = $this->getMockBuilder(ResourceFactory::class)
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $resourceFactory
             ->expects($this->any())
-            ->method('getFile')
+            ->method('retrieveFileOrFolderObject')
             ->willReturn($file);
 
-        $mockFormPersistenceManager
-            ->expects($this->any())
-            ->method('getStorageByUid')
-            ->willReturn($mockStorage);
+        $mockFormPersistenceManager->_set('resourceFactory', $resourceFactory);
 
         $input = '-1:/user_uploads/example.yaml';
-        $mockFormPersistenceManager->_call('getFileByIdentifier', $input);
+        $mockFormPersistenceManager->_call('retrieveFileByPersistenceIdentifier', $input);
     }
 
     /**