[TASK] Performance optimizations for the form manager module 54/58054/12
authorRalf Zimmermann <ralf.zimmermann@tritum.de>
Mon, 27 Aug 2018 23:10:07 +0000 (01:10 +0200)
committerSusanne Moog <susanne.moog@typo3.org>
Fri, 21 Sep 2018 13:36:42 +0000 (15:36 +0200)
Speeds up the form manager module and the form plugin especially if
there are many forms within the TYPO3 instance.

Resolves: #86000
Releases: master, 8.7
Change-Id: Ic483029e0d1b1955d58e04496f97862c00b6d6a4
Reviewed-on: https://review.typo3.org/58054
Reviewed-by: Frank Naegler <frank.naegler@typo3.org>
Tested-by: Frank Naegler <frank.naegler@typo3.org>
Reviewed-by: Anja Leichsenring <aleichsenring@ab-softlab.de>
Tested-by: Anja Leichsenring <aleichsenring@ab-softlab.de>
Tested-by: TYPO3com <no-reply@typo3.com>
Reviewed-by: Susanne Moog <susanne.moog@typo3.org>
Tested-by: Susanne Moog <susanne.moog@typo3.org>
typo3/sysext/form/Classes/Controller/FormManagerController.php
typo3/sysext/form/Classes/Hooks/FormFileExtensionUpdate.php
typo3/sysext/form/Classes/Mvc/Persistence/FormPersistenceManager.php
typo3/sysext/form/Classes/Service/DatabaseService.php [new file with mode: 0644]
typo3/sysext/form/Tests/Unit/Controller/FormManagerControllerTest.php
typo3/sysext/form/Tests/Unit/Mvc/Persistence/FormPersistenceManagerTest.php

index 48b2109..5439016 100644 (file)
@@ -20,18 +20,17 @@ use TYPO3\CMS\Backend\Template\Components\ButtonBar;
 use TYPO3\CMS\Backend\Utility\BackendUtility;
 use TYPO3\CMS\Backend\View\BackendTemplateView;
 use TYPO3\CMS\Core\Authentication\BackendUserAuthentication;
-use TYPO3\CMS\Core\Database\ConnectionPool;
 use TYPO3\CMS\Core\Imaging\Icon;
 use TYPO3\CMS\Core\Imaging\IconFactory;
 use TYPO3\CMS\Core\Localization\LanguageService;
 use TYPO3\CMS\Core\Messaging\AbstractMessage;
 use TYPO3\CMS\Core\Page\PageRenderer;
-use TYPO3\CMS\Core\Resource\ResourceFactory;
 use TYPO3\CMS\Core\Utility\ArrayUtility;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 use TYPO3\CMS\Extbase\Mvc\View\JsonView;
 use TYPO3\CMS\Form\Exception as FormException;
 use TYPO3\CMS\Form\Mvc\Persistence\Exception\PersistenceManagerException;
+use TYPO3\CMS\Form\Service\DatabaseService;
 use TYPO3\CMS\Form\Service\TranslationService;
 
 /**
@@ -43,6 +42,20 @@ class FormManagerController extends AbstractBackendController
 {
 
     /**
+     * @var DatabaseService
+     */
+    protected $databaseService;
+
+    /**
+     * @param \TYPO3\CMS\Form\Service\DatabaseService $databaseService
+     * @internal
+     */
+    public function injectDatabaseService(\TYPO3\CMS\Form\Service\DatabaseService $databaseService)
+    {
+        $this->databaseService = $databaseService;
+    }
+
+    /**
      * Default View Container
      *
      * @var BackendTemplateView
@@ -237,7 +250,7 @@ class FormManagerController extends AbstractBackendController
      */
     public function deleteAction(string $formPersistenceIdentifier)
     {
-        if (empty($this->getReferences($formPersistenceIdentifier))) {
+        if (empty($this->databaseService->getReferencesByPersistenceIdentifier($formPersistenceIdentifier))) {
             foreach ($GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['ext/form']['beforeFormDelete'] ?? [] as $className) {
                 $hookObj = GeneralUtility::makeInstance($className);
                 if (method_exists($hookObj, 'beforeFormDelete')) {
@@ -335,12 +348,25 @@ class FormManagerController extends AbstractBackendController
      */
     protected function getAvailableFormDefinitions(): array
     {
+        $allReferencesForFileUid = $this->databaseService->getAllReferencesForFileUid();
+        $allReferencesForPersistenceIdentifier = $this->databaseService->getAllReferencesForPersistenceIdentifier();
+
         $availableFormDefinitions = [];
         foreach ($this->formPersistenceManager->listForms() as $formDefinition) {
-            $referenceCount = count($this->getReferences($formDefinition['persistenceIdentifier']));
+            $referenceCount  = 0;
+            if (
+                isset($formDefinition['fileUid'])
+                && array_key_exists($formDefinition['fileUid'], $allReferencesForFileUid)
+            ) {
+                $referenceCount = $allReferencesForFileUid[$formDefinition['fileUid']];
+            } elseif (array_key_exists($formDefinition['persistenceIdentifier'], $allReferencesForPersistenceIdentifier)) {
+                $referenceCount = $allReferencesForPersistenceIdentifier[$formDefinition['persistenceIdentifier']];
+            }
+
             $formDefinition['referenceCount'] = $referenceCount;
             $availableFormDefinitions[] = $formDefinition;
         }
+
         return $availableFormDefinitions;
     }
 
@@ -361,7 +387,7 @@ class FormManagerController extends AbstractBackendController
         $references = [];
         $iconFactory = GeneralUtility::makeInstance(IconFactory::class);
 
-        $referenceRows = $this->getReferences($persistenceIdentifier);
+        $referenceRows = $this->databaseService->getReferencesByPersistenceIdentifier($persistenceIdentifier);
         foreach ($referenceRows as &$referenceRow) {
             $record = $this->getRecord($referenceRow['tablename'], $referenceRow['recuid']);
             if (!$record) {
@@ -389,41 +415,6 @@ class FormManagerController extends AbstractBackendController
     }
 
     /**
-     * Returns an array with all sys_refindex database rows which be
-     * connected to a formDefinition identified by $persistenceIdentifier
-     *
-     * @param string $persistenceIdentifier
-     * @return array
-     * @throws \InvalidArgumentException
-     */
-    protected function getReferences(string $persistenceIdentifier): array
-    {
-        if (empty($persistenceIdentifier)) {
-            throw new \InvalidArgumentException('$persistenceIdentifier must not be empty.', 1472238493);
-        }
-
-        $resourceFactory = GeneralUtility::makeInstance(ResourceFactory::class);
-        $file = $resourceFactory->retrieveFileOrFolderObject($persistenceIdentifier);
-
-        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('sys_refindex');
-        $referenceRows = $queryBuilder
-            ->select('*')
-            ->from('sys_refindex')
-            ->where(
-                $queryBuilder->expr()->eq('deleted', 0),
-                $queryBuilder->expr()->eq('softref_key', $queryBuilder->createNamedParameter('formPersistenceIdentifier', \PDO::PARAM_STR)),
-                $queryBuilder->expr()->orX(
-                    $queryBuilder->expr()->eq('ref_string', $queryBuilder->createNamedParameter($persistenceIdentifier, \PDO::PARAM_STR)),
-                    $queryBuilder->expr()->eq('ref_uid', $queryBuilder->createNamedParameter($file->getUid(), \PDO::PARAM_INT))
-                ),
-                $queryBuilder->expr()->eq('tablename', $queryBuilder->createNamedParameter('tt_content', \PDO::PARAM_STR))
-            )
-            ->execute()
-            ->fetchAll();
-        return $referenceRows;
-    }
-
-    /**
      * Check if a given $templatePath for a given $prototypeName is valid
      * and accessible.
      *
index 2eab469..bf4dffe 100644 (file)
@@ -29,7 +29,6 @@ use TYPO3\CMS\Core\Utility\GeneralUtility;
 use TYPO3\CMS\Core\Utility\PathUtility;
 use TYPO3\CMS\Core\Utility\StringUtility;
 use TYPO3\CMS\Extbase\Object\ObjectManager;
-use TYPO3\CMS\Form\Mvc\Configuration\YamlSource;
 use TYPO3\CMS\Form\Mvc\Persistence\FormPersistenceManager;
 use TYPO3\CMS\Form\Slot\FilePersistenceSlot;
 use TYPO3\CMS\Install\Updates\ChattyInterface;
@@ -53,11 +52,6 @@ class FormFileExtensionUpdate implements ChattyInterface, UpgradeWizardInterface
     protected $persistenceManager;
 
     /**
-     * @var YamlSource
-     */
-    protected $yamlSource;
-
-    /**
      * @var ResourceFactory
      */
     protected $resourceFactory;
@@ -139,7 +133,6 @@ class FormFileExtensionUpdate implements ChattyInterface, UpgradeWizardInterface
         $updateNeeded = false;
 
         $this->persistenceManager = $this->getObjectManager()->get(FormPersistenceManager::class);
-        $this->yamlSource = $this->getObjectManager()->get(YamlSource::class);
         $this->resourceFactory = GeneralUtility::makeInstance(ResourceFactory::class);
 
         foreach ($this->getFormDefinitionsInformation() as $formDefinitionInformation) {
@@ -212,7 +205,6 @@ class FormFileExtensionUpdate implements ChattyInterface, UpgradeWizardInterface
 
         $this->connection = $connectionPool->getConnectionForTable('tt_content');
         $this->persistenceManager = $this->getObjectManager()->get(FormPersistenceManager::class);
-        $this->yamlSource = $this->getObjectManager()->get(YamlSource::class);
         $this->resourceFactory = GeneralUtility::makeInstance(ResourceFactory::class);
         $this->referenceIndex = GeneralUtility::makeInstance(ReferenceIndex::class);
         $this->flexFormTools = GeneralUtility::makeInstance(FlexFormTools::class);
@@ -728,7 +720,8 @@ class FormFileExtensionUpdate implements ChattyInterface, UpgradeWizardInterface
     protected function getFormDefinition(File $file): array
     {
         try {
-            $formDefinition = $this->yamlSource->load([$file]);
+            $rawYamlContent = $file->getContents();
+            $formDefinition = $this->extractMetaDataFromCouldBeFormDefinition($rawYamlContent);
 
             if (!$this->looksLikeAFormDefinition($formDefinition)) {
                 $formDefinition = [];
@@ -741,6 +734,35 @@ class FormFileExtensionUpdate implements ChattyInterface, UpgradeWizardInterface
     }
 
     /**
+     * @param string $maybeRawFormDefinition
+     * @return array
+     */
+    protected function extractMetaDataFromCouldBeFormDefinition(string $maybeRawFormDefinition): array
+    {
+        $metaDataProperties = ['identifier', 'type', 'label', 'prototypeName'];
+        $metaData = [];
+        foreach (explode("\n", $maybeRawFormDefinition) as $line) {
+            if (empty($line) || $line[0] === ' ') {
+                continue;
+            }
+
+            [$key, $value] = explode(':', $line);
+            if (
+                empty($key)
+                || empty($value)
+                || !in_array($key, $metaDataProperties)
+            ) {
+                continue;
+            }
+
+            $value = trim($value, ' \'"');
+            $metaData[$key] = $value;
+        }
+
+        return $metaData;
+    }
+
+    /**
      * @return array
      */
     protected function getAllFlexformFieldsFromFormPlugins(): array
index 936ebd3..aed5ef7 100644 (file)
@@ -17,11 +17,14 @@ namespace TYPO3\CMS\Form\Mvc\Persistence;
  * The TYPO3 project - inspiring people to share!
  */
 
+use TYPO3\CMS\Core\Cache\CacheManager;
+use TYPO3\CMS\Core\Cache\Frontend\FrontendInterface;
 use TYPO3\CMS\Core\Resource\Exception\FolderDoesNotExistException;
 use TYPO3\CMS\Core\Resource\Exception\InsufficientFolderAccessPermissionsException;
 use TYPO3\CMS\Core\Resource\File;
 use TYPO3\CMS\Core\Resource\Filter\FileExtensionFilter;
 use TYPO3\CMS\Core\Resource\Folder;
+use TYPO3\CMS\Core\Resource\ResourceFactory;
 use TYPO3\CMS\Core\Resource\ResourceStorage;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 use TYPO3\CMS\Core\Utility\PathUtility;
@@ -29,6 +32,7 @@ use TYPO3\CMS\Core\Utility\StringUtility;
 use TYPO3\CMS\Extbase\Object\ObjectManager;
 use TYPO3\CMS\Form\Mvc\Configuration\ConfigurationManagerInterface;
 use TYPO3\CMS\Form\Mvc\Configuration\Exception\FileWriteException;
+use TYPO3\CMS\Form\Mvc\Configuration\Exception\NoSuchFileException;
 use TYPO3\CMS\Form\Mvc\Persistence\Exception\NoUniqueIdentifierException;
 use TYPO3\CMS\Form\Mvc\Persistence\Exception\NoUniquePersistenceIdentifierException;
 use TYPO3\CMS\Form\Mvc\Persistence\Exception\PersistenceManagerException;
@@ -64,6 +68,16 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
     protected $filePersistenceSlot;
 
     /**
+     * @var FrontendInterface
+     */
+    protected $runtimeCache;
+
+    /**
+     * @var ResourceFactory
+     */
+    protected $resourceFactory;
+
+    /**
      * @param \TYPO3\CMS\Form\Mvc\Configuration\YamlSource $yamlSource
      * @internal
      */
@@ -82,14 +96,22 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
     }
 
     /**
-     * @param FilePersistenceSlot $filePersistenceSlot
+     * @param \TYPO3\CMS\Form\Slot\FilePersistenceSlot $filePersistenceSlot
      */
-    public function injectFilePersistenceSlot(FilePersistenceSlot $filePersistenceSlot)
+    public function injectFilePersistenceSlot(\TYPO3\CMS\Form\Slot\FilePersistenceSlot $filePersistenceSlot)
     {
         $this->filePersistenceSlot = $filePersistenceSlot;
     }
 
     /**
+     * @param \TYPO3\CMS\Core\Resource\ResourceFactory $resourceFactory
+     */
+    public function injectResourceFactory(\TYPO3\CMS\Core\Resource\ResourceFactory $resourceFactory)
+    {
+        $this->resourceFactory = $resourceFactory;
+    }
+
+    /**
      * @internal
      */
     public function initializeObject()
@@ -97,6 +119,7 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
         $this->formSettings = GeneralUtility::makeInstance(ObjectManager::class)
             ->get(ConfigurationManagerInterface::class)
             ->getConfiguration(ConfigurationManagerInterface::CONFIGURATION_TYPE_YAML_SETTINGS, 'form');
+        $this->runtimeCache = GeneralUtility::makeInstance(CacheManager::class)->getCache('cache_runtime');
     }
 
     /**
@@ -107,43 +130,22 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
      *
      * @param string $persistenceIdentifier
      * @return array
-     * @throws PersistenceManagerException
      * @internal
      */
     public function load(string $persistenceIdentifier): array
     {
-        if (pathinfo($persistenceIdentifier, PATHINFO_EXTENSION) !== 'yaml') {
-            throw new PersistenceManagerException(sprintf('The file "%s" could not be loaded.', $persistenceIdentifier), 1477679819);
-        }
+        $cacheKey = 'formLoad' . md5($persistenceIdentifier);
 
-        if (strpos($persistenceIdentifier, 'EXT:') === 0) {
-            if (!array_key_exists(pathinfo($persistenceIdentifier, PATHINFO_DIRNAME) . '/', $this->getAccessibleExtensionFolders())) {
-                throw new PersistenceManagerException(sprintf('The file "%s" could not be loaded.', $persistenceIdentifier), 1484071985);
-            }
-            $file = $persistenceIdentifier;
-        } else {
-            $file = $this->getFileByIdentifier($persistenceIdentifier);
+        $yaml = $this->runtimeCache->get($cacheKey);
+        if ($yaml !== false) {
+            return $yaml;
         }
 
+        $file = $this->retrieveFileByPersistenceIdentifier($persistenceIdentifier);
+
         try {
             $yaml = $this->yamlSource->load([$file]);
-
-            if (isset($yaml['identifier'], $yaml['type']) && $yaml['type'] === 'Form') {
-                if (
-                    !$this->hasValidFileExtension($persistenceIdentifier)
-                    && strpos($persistenceIdentifier, 'EXT:') === 0
-                ) {
-                    trigger_error(
-                        'Form definition file name ("' . $persistenceIdentifier . '") which does not end with ".form.yaml" has been deprecated in v9 and will not be supported in v10.',
-                        E_USER_DEPRECATED
-                    );
-                } elseif (
-                    !$this->hasValidFileExtension($persistenceIdentifier)
-                    && strpos($persistenceIdentifier, 'EXT:') !== 0
-                ) {
-                    throw new PersistenceManagerException(sprintf('Form definition "%s" does not end with ".form.yaml".', $persistenceIdentifier), 1531160649);
-                }
-            }
+            $this->generateErrorsIfFormDefinitionIsValidButHasInvalidFileExtension($yaml, $persistenceIdentifier);
         } catch (\Exception $e) {
             $yaml = [
                 'type' => 'Form',
@@ -152,6 +154,7 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
                 'invalid' => true,
             ];
         }
+        $this->runtimeCache->set($cacheKey, $yaml);
 
         return $yaml;
     }
@@ -179,8 +182,9 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
             if (!$this->formSettings['persistenceManager']['allowSaveToExtensionPaths']) {
                 throw new PersistenceManagerException('Save to extension paths is not allowed.', 1477680881);
             }
-            if (!array_key_exists(pathinfo($persistenceIdentifier, PATHINFO_DIRNAME) . '/', $this->getAccessibleExtensionFolders())) {
-                throw new PersistenceManagerException(sprintf('The file "%s" could not be saved.', $persistenceIdentifier), 1484073571);
+            if (!$this->isFileWithinAccessibleExtensionFolders($persistenceIdentifier)) {
+                $message = sprintf('The file "%s" could not be saved. Please check your configuration option "persistenceManager.allowedExtensionPaths"', $persistenceIdentifier);
+                throw new PersistenceManagerException($message, 1484073571);
             }
             $fileToSave = GeneralUtility::getFileAbsFileName($persistenceIdentifier);
         } else {
@@ -218,8 +222,9 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
             if (!$this->formSettings['persistenceManager']['allowDeleteFromExtensionPaths']) {
                 throw new PersistenceManagerException(sprintf('The file "%s" could not be removed.', $persistenceIdentifier), 1472239536);
             }
-            if (!array_key_exists(pathinfo($persistenceIdentifier, PATHINFO_DIRNAME) . '/', $this->getAccessibleExtensionFolders())) {
-                throw new PersistenceManagerException(sprintf('The file "%s" could not be removed.', $persistenceIdentifier), 1484073878);
+            if (!$this->isFileWithinAccessibleExtensionFolders($persistenceIdentifier)) {
+                $message = sprintf('The file "%s" could not be removed. Please check your configuration option "persistenceManager.allowedExtensionPaths"', $persistenceIdentifier);
+                throw new PersistenceManagerException($message, 1484073878);
             }
             $fileToDelete = GeneralUtility::getFileAbsFileName($persistenceIdentifier);
             unlink($fileToDelete);
@@ -246,7 +251,7 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
         $exists = false;
         if ($this->hasValidFileExtension($persistenceIdentifier)) {
             if (strpos($persistenceIdentifier, 'EXT:') === 0) {
-                if (array_key_exists(pathinfo($persistenceIdentifier, PATHINFO_DIRNAME) . '/', $this->getAccessibleExtensionFolders())) {
+                if ($this->isFileWithinAccessibleExtensionFolders($persistenceIdentifier)) {
                     $exists = file_exists(GeneralUtility::getFileAbsFileName($persistenceIdentifier));
                 }
             } else {
@@ -279,13 +284,13 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
             // TODO: deprecated since TYPO3 v9, will be removed in TYPO3 v10
             $formReadOnly = $folder->getCombinedIdentifier() === '1:/user_upload/';
 
-            $persistenceIdentifier = $file->getCombinedIdentifier();
+            $form = $this->loadMetaData($file);
 
-            $form = $this->load($persistenceIdentifier);
-            if (empty($form['identifier']) || ($form['type'] ?? null) !== 'Form') {
+            if (!$this->looksLikeAFormDefinition($form)) {
                 continue;
             }
 
+            $persistenceIdentifier = $file->getCombinedIdentifier();
             if ($this->hasValidFileExtension($persistenceIdentifier)) {
                 $forms[] = [
                     'identifier' => $form['identifier'],
@@ -296,6 +301,7 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
                     'location' => 'storage',
                     'duplicateIdentifier' => false,
                     'invalid' => $form['invalid'],
+                    'fileUid' => $form['fileUid'],
                 ];
                 $identifiers[$form['identifier']]++;
             } else {
@@ -309,13 +315,15 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
                     'duplicateIdentifier' => false,
                     'invalid' => false,
                     'deprecatedFileExtension' => true,
+                    'fileUid' => $form['fileUid'],
                 ];
             }
         }
 
         foreach ($this->retrieveYamlFilesFromExtensionFolders() as $fullPath => $fileName) {
-            $form = $this->load($fullPath);
-            if (isset($form['identifier'], $form['type']) && $form['type'] === 'Form') {
+            $form = $this->loadMetaData($fullPath);
+
+            if ($this->looksLikeAFormDefinition($form)) {
                 if ($this->hasValidFileExtension($fileName)) {
                     $forms[] = [
                         'identifier' => $form['identifier'],
@@ -326,6 +334,7 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
                         'location' => 'extension',
                         'duplicateIdentifier' => false,
                         'invalid' => $form['invalid'],
+                        'fileUid' => $form['fileUid'],
                     ];
                     $identifiers[$form['identifier']]++;
                 } else {
@@ -339,6 +348,7 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
                         'duplicateIdentifier' => false,
                         'invalid' => false,
                         'deprecatedFileExtension' => true,
+                        'fileUid' => $form['fileUid'],
                     ];
                 }
             }
@@ -471,12 +481,19 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
      */
     public function getAccessibleExtensionFolders(): array
     {
+        $extensionFolders = $this->runtimeCache->get('formAccessibleExtensionFolders');
+
+        if ($extensionFolders !== false) {
+            return $extensionFolders;
+        }
+
         $extensionFolders = [];
         if (
             !isset($this->formSettings['persistenceManager']['allowedExtensionPaths'])
             || !is_array($this->formSettings['persistenceManager']['allowedExtensionPaths'])
             || empty($this->formSettings['persistenceManager']['allowedExtensionPaths'])
         ) {
+            $this->runtimeCache->set('formAccessibleExtensionFolders', $extensionFolders);
             return $extensionFolders;
         }
 
@@ -492,6 +509,8 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
             $allowedExtensionPath = rtrim($allowedExtensionPath, '/') . '/';
             $extensionFolders[$allowedExtensionPath] = $allowedExtensionFullPath;
         }
+
+        $this->runtimeCache->set('formAccessibleExtensionFolders', $extensionFolders);
         return $extensionFolders;
     }
 
@@ -581,24 +600,6 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
     }
 
     /**
-     * Returns a File object for a given $persistenceIdentifier
-     *
-     * @param string $persistenceIdentifier
-     * @return File
-     * @throws PersistenceManagerException
-     */
-    protected function getFileByIdentifier(string $persistenceIdentifier): File
-    {
-        list($storageUid, $fileIdentifier) = explode(':', $persistenceIdentifier, 2);
-        $storage = $this->getStorageByUid((int)$storageUid);
-        $file = $storage->getFile($fileIdentifier);
-        if (!$storage->checkFileActionPermission('read', $file)) {
-            throw new PersistenceManagerException(sprintf('No read access to file "%s".', $persistenceIdentifier), 1471630578);
-        }
-        return $file;
-    }
-
-    /**
      * Returns a File object for a given $persistenceIdentifier.
      * If no file for this identifier exists a new object will be
      * created.
@@ -659,6 +660,131 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
     }
 
     /**
+     * @param string|File $persistenceIdentifier
+     * @return array
+     * @throws NoSuchFileException
+     */
+    protected function loadMetaData($persistenceIdentifier): array
+    {
+        if ($persistenceIdentifier instanceof File) {
+            $file = $persistenceIdentifier;
+            $persistenceIdentifier = $file->getCombinedIdentifier();
+        } else {
+            $file = $this->retrieveFileByPersistenceIdentifier($persistenceIdentifier);
+        }
+
+        try {
+            $rawYamlContent = $file->getContents();
+
+            if ($rawYamlContent === false) {
+                throw new NoSuchFileException(sprintf('YAML file "%s" could not be loaded', $persistenceIdentifier), 1524684462);
+            }
+
+            $yaml = $this->extractMetaDataFromCouldBeFormDefinition($rawYamlContent);
+            $this->generateErrorsIfFormDefinitionIsValidButHasInvalidFileExtension($yaml, $persistenceIdentifier);
+            $yaml['fileUid'] = $file->getUid();
+        } catch (\Exception $e) {
+            $yaml = [
+                'type' => 'Form',
+                'identifier' => $persistenceIdentifier,
+                'label' => $e->getMessage(),
+                'invalid' => true,
+            ];
+        }
+
+        return $yaml;
+    }
+
+    /**
+     * @param string $maybeRawFormDefinition
+     * @return array
+     */
+    protected function extractMetaDataFromCouldBeFormDefinition(string $maybeRawFormDefinition): array
+    {
+        $metaDataProperties = ['identifier', 'type', 'label', 'prototypeName'];
+        $metaData = [];
+        foreach (explode(LF, $maybeRawFormDefinition) as $line) {
+            if (empty($line) || $line[0] === ' ') {
+                continue;
+            }
+
+            [$key, $value] = explode(':', $line);
+            if (
+                empty($key)
+                || empty($value)
+                || !in_array($key, $metaDataProperties, true)
+            ) {
+                continue;
+            }
+
+            $value = trim($value, ' \'"');
+            $metaData[$key] = $value;
+        }
+
+        return $metaData;
+    }
+
+    /**
+     * @param array $formDefinition
+     * @param string $persistenceIdentifier
+     * @throws PersistenceManagerException
+     */
+    protected function generateErrorsIfFormDefinitionIsValidButHasInvalidFileExtension(array $formDefinition, string $persistenceIdentifier): void
+    {
+        if (
+            $this->looksLikeAFormDefinition($formDefinition)
+            && !$this->hasValidFileExtension($persistenceIdentifier)
+        ) {
+            if (strpos($persistenceIdentifier, 'EXT:') === 0) {
+                trigger_error(
+                    'Form definition file name ("' . $persistenceIdentifier . '") which does not end with ".form.yaml" has been deprecated in v9 and will not be supported in v10.',
+                    E_USER_DEPRECATED
+                );
+            } elseif (strpos($persistenceIdentifier, 'EXT:') !== 0) {
+                throw new PersistenceManagerException(sprintf('Form definition "%s" does not end with ".form.yaml".', $persistenceIdentifier), 1531160649);
+            }
+        }
+    }
+
+    /**
+     * @param string $persistenceIdentifier
+     * @return File
+     * @throws PersistenceManagerException
+     * @throws NoSuchFileException
+     */
+    protected function retrieveFileByPersistenceIdentifier(string $persistenceIdentifier): File
+    {
+        if (pathinfo($persistenceIdentifier, PATHINFO_EXTENSION) !== 'yaml') {
+            throw new PersistenceManagerException(sprintf('The file "%s" could not be loaded.', $persistenceIdentifier), 1477679819);
+        }
+
+        if (
+            strpos($persistenceIdentifier, 'EXT:') === 0
+            && !$this->isFileWithinAccessibleExtensionFolders($persistenceIdentifier)
+        ) {
+            $message = sprintf('The file "%s" could not be loaded. Please check your configuration option "persistenceManager.allowedExtensionPaths"', $persistenceIdentifier);
+            throw new PersistenceManagerException($message, 1484071985);
+        }
+
+        try {
+            $file = $this->resourceFactory->retrieveFileOrFolderObject($persistenceIdentifier);
+        } catch (\Exception $e) {
+            // Top level catch to ensure useful following exception handling, because FAL throws top level exceptions.
+            $file = null;
+        }
+
+        if ($file === null) {
+            throw new NoSuchFileException(sprintf('YAML file "%s" could not be loaded', $persistenceIdentifier), 1524684442);
+        }
+
+        if (!$file->getStorage()->checkFileActionPermission('read', $file)) {
+            throw new PersistenceManagerException(sprintf('No read access to file "%s".', $persistenceIdentifier), 1471630578);
+        }
+
+        return $file;
+    }
+
+    /**
      * @param string $fileName
      * @return bool
      */
@@ -666,4 +792,23 @@ class FormPersistenceManager implements FormPersistenceManagerInterface
     {
         return StringUtility::endsWith($fileName, self::FORM_DEFINITION_FILE_EXTENSION);
     }
+
+    /**
+     * @param string $fileName
+     * @return bool
+     */
+    protected function isFileWithinAccessibleExtensionFolders(string $fileName): bool
+    {
+        $dirName = rtrim(PathUtility::pathinfo($fileName, PATHINFO_DIRNAME), '/') . '/';
+        return array_key_exists($dirName, $this->getAccessibleExtensionFolders());
+    }
+
+    /**
+     * @param array $data
+     * @return bool
+     */
+    protected function looksLikeAFormDefinition(array $data): bool
+    {
+        return isset($data['identifier'], $data['type']) && !empty($data['identifier']) && $data['type'] === 'Form';
+    }
 }
diff --git a/typo3/sysext/form/Classes/Service/DatabaseService.php b/typo3/sysext/form/Classes/Service/DatabaseService.php
new file mode 100644 (file)
index 0000000..d4c4f40
--- /dev/null
@@ -0,0 +1,134 @@
+<?php
+declare(strict_types = 1);
+namespace TYPO3\CMS\Form\Service;
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+use TYPO3\CMS\Core\Database\ConnectionPool;
+use TYPO3\CMS\Core\Resource\ResourceFactory;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
+/**
+ * This class is subjected to change.
+ * **Do NOT subclass**
+ *
+ * Scope: frontend / backend
+ * @internal
+ */
+class DatabaseService
+{
+
+    /**
+     * Returns an array with all sys_refindex database rows which be
+     * connected to a formDefinition identified by $persistenceIdentifier
+     *
+     * @param string $persistenceIdentifier
+     * @return array
+     * @throws \InvalidArgumentException
+     * @internal
+     */
+    public function getReferencesByPersistenceIdentifier(string $persistenceIdentifier): array
+    {
+        if (empty($persistenceIdentifier)) {
+            throw new \InvalidArgumentException('$persistenceIdentifier must not be empty.', 1472238493);
+        }
+
+        $resourceFactory = GeneralUtility::makeInstance(ResourceFactory::class);
+        $file = $resourceFactory->retrieveFileOrFolderObject($persistenceIdentifier);
+
+        if ($file === null) {
+            return [];
+        }
+
+        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('sys_refindex');
+
+        return $queryBuilder
+            ->select('*')
+            ->from('sys_refindex')
+            ->where(
+                $queryBuilder->expr()->eq('deleted', 0),
+                $queryBuilder->expr()->eq('softref_key', $queryBuilder->createNamedParameter('formPersistenceIdentifier', \PDO::PARAM_STR)),
+                $queryBuilder->expr()->orX(
+                    $queryBuilder->expr()->eq('ref_string', $queryBuilder->createNamedParameter($persistenceIdentifier, \PDO::PARAM_STR)),
+                    $queryBuilder->expr()->eq('ref_uid', $queryBuilder->createNamedParameter($file->getUid(), \PDO::PARAM_INT))
+                ),
+                $queryBuilder->expr()->eq('tablename', $queryBuilder->createNamedParameter('tt_content', \PDO::PARAM_STR))
+            )
+            ->execute()
+            ->fetchAll();
+    }
+
+    /**
+     * Returns an array with all form definition persistenceIdentifiers
+     * as keys and their reference counts as values.
+     *
+     * @return array
+     * @internal
+     */
+    public function getAllReferencesForPersistenceIdentifier(): array
+    {
+        $items = [];
+        foreach ($this->getAllReferences('ref_string') as $item) {
+            $items[$item['identifier']] = $item['items'];
+        }
+        return $items;
+    }
+
+    /**
+     * Returns an array with all form definition file uids as keys
+     * and their reference counts as values.
+     *
+     * @return array
+     * @internal
+     */
+    public function getAllReferencesForFileUid(): array
+    {
+        $items = [];
+        foreach ($this->getAllReferences('ref_uid') as $item) {
+            $items[$item['identifier']] = $item['items'];
+        }
+        return $items;
+    }
+
+    /**
+     * @param string $column
+     * @return array
+     * @throws \InvalidArgumentException
+     */
+    protected function getAllReferences(string $column): array
+    {
+        if ($column !== 'ref_string' && $column !== 'ref_uid') {
+            throw new \InvalidArgumentException('$column must not be "ref_string" or "ref_uid".', 1535406600);
+        }
+
+        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('sys_refindex');
+
+        $constraints = [$queryBuilder->expr()->eq('softref_key', $queryBuilder->createNamedParameter('formPersistenceIdentifier', \PDO::PARAM_STR))];
+
+        if ($column === 'ref_string') {
+            $constraints[] = $queryBuilder->expr()->neq('ref_string', $queryBuilder->createNamedParameter('', \PDO::PARAM_STR));
+        } else {
+            $constraints[] = $queryBuilder->expr()->gt('ref_uid', $queryBuilder->createNamedParameter(0, \PDO::PARAM_INT));
+        }
+
+        return $queryBuilder
+            ->select($column . ' AS identifier')
+            ->addSelectLiteral('COUNT(' . $queryBuilder->quoteIdentifier($column) . ') AS ' . $queryBuilder->quoteIdentifier('items'))
+            ->from('sys_refindex')
+            ->where(...$constraints)
+            ->groupBy($column)
+            ->execute()
+            ->fetchAll();
+    }
+}
index d4ee020..ce57de4 100644 (file)
@@ -26,6 +26,7 @@ use TYPO3\CMS\Extbase\Mvc\Web\Routing\UriBuilder;
 use TYPO3\CMS\Extbase\Object\ObjectManager;
 use TYPO3\CMS\Form\Controller\FormManagerController;
 use TYPO3\CMS\Form\Mvc\Persistence\FormPersistenceManager;
+use TYPO3\CMS\Form\Service\DatabaseService;
 use TYPO3\CMS\Form\Service\TranslationService;
 use TYPO3\TestingFramework\Core\Unit\UnitTestCase;
 
@@ -175,12 +176,15 @@ class FormManagerControllerTest extends UnitTestCase
     public function getAvailableFormDefinitionsReturnsProcessedArray(): void
     {
         $mockController = $this->getAccessibleMock(FormManagerController::class, [
-            'getReferences'
+            'dummy'
         ], [], '', false);
 
         $formPersistenceManagerProphecy = $this->prophesize(FormPersistenceManager::class);
         $mockController->_set('formPersistenceManager', $formPersistenceManagerProphecy->reveal());
 
+        $databaseService = $this->prophesize(DatabaseService::class);
+        $mockController->_set('databaseService', $databaseService->reveal());
+
         $formPersistenceManagerProphecy->listForms(Argument::cetera())->willReturn([
             0 => [
                 'identifier' => 'ext-form-identifier',
@@ -193,13 +197,13 @@ class FormManagerControllerTest extends UnitTestCase
             ],
         ]);
 
-        $mockController
-            ->expects($this->any())
-            ->method('getReferences')
-            ->willReturn([
-                'someRow',
-                'anotherRow',
-            ]);
+        $databaseService->getAllReferencesForFileUid(Argument::cetera())->willReturn([
+            0 => 0,
+        ]);
+
+        $databaseService->getAllReferencesForPersistenceIdentifier(Argument::cetera())->willReturn([
+            '1:/user_uploads/someFormName.yaml' => 2,
+        ]);
 
         $expected = [
             0 => [
@@ -247,9 +251,18 @@ class FormManagerControllerTest extends UnitTestCase
             'getModuleUrl',
             'getRecord',
             'getRecordTitle',
-            'getReferences',
         ], [], '', false);
 
+        $databaseService = $this->prophesize(DatabaseService::class);
+        $mockController->_set('databaseService', $databaseService->reveal());
+
+        $databaseService->getReferencesByPersistenceIdentifier(Argument::cetera())->willReturn([
+            0 => [
+                'tablename' => 'tt_content',
+                'recuid' => -1,
+            ],
+        ]);
+
         $mockController
             ->expects($this->any())
             ->method('getModuleUrl')
@@ -265,16 +278,6 @@ class FormManagerControllerTest extends UnitTestCase
             ->method('getRecordTitle')
             ->willReturn('record title');
 
-        $mockController
-            ->expects($this->any())
-            ->method('getReferences')
-            ->willReturn([
-                0 => [
-                    'tablename' => 'tt_content',
-                    'recuid' => -1,
-                ],
-            ]);
-
         $expected = [
             0 => [
                 'recordPageTitle' => 'record title',
index 0826601..18054f2 100644 (file)
@@ -15,7 +15,9 @@ namespace TYPO3\CMS\Form\Tests\Unit\Mvc\Persistence;
  * The TYPO3 project - inspiring people to share!
  */
 
+use TYPO3\CMS\Core\Cache\Frontend\VariableFrontend;
 use TYPO3\CMS\Core\Resource\File;
+use TYPO3\CMS\Core\Resource\ResourceFactory;
 use TYPO3\CMS\Core\Resource\ResourceStorage;
 use TYPO3\CMS\Core\Resource\StorageRepository;
 use TYPO3\CMS\Form\Mvc\Persistence\Exception\NoUniqueIdentifierException;
@@ -40,6 +42,18 @@ class FormPersistenceManagerTest extends UnitTestCase
             'dummy'
         ], [], '', false);
 
+        $runtimeCache= $this->getMockBuilder(VariableFrontend::class)
+            ->setMethods(['get', 'set'])
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $runtimeCache
+            ->expects($this->any())
+            ->method('get')
+            ->willReturn(false);
+
+        $mockFormPersistenceManager->_set('runtimeCache', $runtimeCache);
+
         $input = '-1:/user_uploads/_example.php';
         $mockFormPersistenceManager->_call('load', $input);
     }
@@ -56,6 +70,18 @@ class FormPersistenceManagerTest extends UnitTestCase
             'dummy'
         ], [], '', false);
 
+        $runtimeCache= $this->getMockBuilder(VariableFrontend::class)
+            ->setMethods(['get', 'set'])
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $runtimeCache
+            ->expects($this->any())
+            ->method('get')
+            ->willReturn(false);
+
+        $mockFormPersistenceManager->_set('runtimeCache', $runtimeCache);
+
         $mockFormPersistenceManager->_set('formSettings', [
             'persistenceManager' => [
                 'allowedExtensionPaths' => [],
@@ -116,6 +142,18 @@ class FormPersistenceManagerTest extends UnitTestCase
             'dummy'
         ], [], '', false);
 
+        $runtimeCache= $this->getMockBuilder(VariableFrontend::class)
+            ->setMethods(['get', 'set'])
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $runtimeCache
+            ->expects($this->any())
+            ->method('get')
+            ->willReturn(false);
+
+        $mockFormPersistenceManager->_set('runtimeCache', $runtimeCache);
+
         $mockFormPersistenceManager->_set('formSettings', [
             'persistenceManager' => [
                 'allowSaveToExtensionPaths' => true,
@@ -203,6 +241,18 @@ class FormPersistenceManagerTest extends UnitTestCase
             'exists'
         ], [], '', false);
 
+        $runtimeCache= $this->getMockBuilder(VariableFrontend::class)
+            ->setMethods(['get', 'set'])
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $runtimeCache
+            ->expects($this->any())
+            ->method('get')
+            ->willReturn(false);
+
+        $mockFormPersistenceManager->_set('runtimeCache', $runtimeCache);
+
         $mockFormPersistenceManager
             ->expects($this->any())
             ->method('exists')
@@ -270,6 +320,18 @@ class FormPersistenceManagerTest extends UnitTestCase
             'dummy'
         ], [], '', false);
 
+        $runtimeCache= $this->getMockBuilder(VariableFrontend::class)
+            ->setMethods(['get', 'set'])
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $runtimeCache
+            ->expects($this->any())
+            ->method('get')
+            ->willReturn(false);
+
+        $mockFormPersistenceManager->_set('runtimeCache', $runtimeCache);
+
         $mockFormPersistenceManager->_set('formSettings', [
             'persistenceManager' => [
                 'allowedExtensionPaths' => [
@@ -558,37 +620,39 @@ class FormPersistenceManagerTest extends UnitTestCase
     /**
      * @test
      */
-    public function getFileByIdentifierThrowsExceptionIfReadFromStorageIsNotAllowed(): void
+    public function retrieveFileByPersistenceIdentifierThrowsExceptionIfReadFromStorageIsNotAllowed(): void
     {
         $this->expectException(PersistenceManagerException::class);
         $this->expectExceptionCode(1471630578);
 
         $mockFormPersistenceManager = $this->getAccessibleMock(FormPersistenceManager::class, [
-            'getStorageByUid',
+            'dummy',
         ], [], '', false);
 
-        $mockStorage = $this->getMockBuilder(ResourceStorage::class)
+        $storage = $this->getMockBuilder(ResourceStorage::class)
             ->disableOriginalConstructor()
             ->getMock();
 
-        $mockStorage
+        $storage
             ->expects($this->any())
             ->method('checkFileActionPermission')
             ->willReturn(false);
 
-        $file = new File(['name' => 'foo', 'identifier' => '', 'mime_type' => ''], $mockStorage);
-        $mockStorage
+        $file = new File(['name' => 'foo', 'identifier' => '', 'mime_type' => ''], $storage);
+
+        $resourceFactory = $this->getMockBuilder(ResourceFactory::class)
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $resourceFactory
             ->expects($this->any())
-            ->method('getFile')
+            ->method('retrieveFileOrFolderObject')
             ->willReturn($file);
 
-        $mockFormPersistenceManager
-            ->expects($this->any())
-            ->method('getStorageByUid')
-            ->willReturn($mockStorage);
+        $mockFormPersistenceManager->_set('resourceFactory', $resourceFactory);
 
         $input = '-1:/user_uploads/example.yaml';
-        $mockFormPersistenceManager->_call('getFileByIdentifier', $input);
+        $mockFormPersistenceManager->_call('retrieveFileByPersistenceIdentifier', $input);
     }
 
     /**