[!!!][TASK] Migrate backend_layout.icon to FAL 30/54830/8
authorBenni Mack <benni@typo3.org>
Tue, 28 Nov 2017 11:57:38 +0000 (12:57 +0100)
committerChristian Kuhn <lolli@schwarzbu.ch>
Wed, 29 Nov 2017 17:31:13 +0000 (18:31 +0100)
The last place in TYPO3 Core to use internal_type=file (backend_layout.icon)
is now moved to FAL with sys_file_reference.

An upgrade wizard is in place to migrate existing icons from uploads/media
to sys_file.

FormEngine is migrated to also allow to use sys_file_references for the
TCA ctrl property "selicon_field".

With this change, a followup could be integrated to deprecate "old-style"
internal_type=file code, and also to not create the uploads/ folder at all anymore
within TYPO3 installations.

Additionally, the functionality of "selicon_field_path" should be deprecated
in the future.

Resolves: #83153
Releases: master
Change-Id: I578fd68b7e1f7bc6a1991b90e7750b903d3ec28b
Reviewed-on: https://review.typo3.org/54830
Tested-by: TYPO3com <no-reply@typo3.com>
Reviewed-by: Sebastian Hofer <sebastian.hofer@marit.ag>
Tested-by: Sebastian Hofer <sebastian.hofer@marit.ag>
Reviewed-by: Pawel Cieslik <p.cieslik@macopedia.pl>
Tested-by: Pawel Cieslik <p.cieslik@macopedia.pl>
Reviewed-by: Christian Kuhn <lolli@schwarzbu.ch>
Tested-by: Christian Kuhn <lolli@schwarzbu.ch>
typo3/sysext/backend/Classes/Form/FormDataProvider/AbstractItemProvider.php
typo3/sysext/backend/Classes/View/BackendLayout/DefaultDataProvider.php
typo3/sysext/core/Documentation/Changelog/master/Breaking-83153-MigratedBackendLayoutIconToFileAbstractionLayer.rst [new file with mode: 0644]
typo3/sysext/frontend/Configuration/TCA/backend_layout.php
typo3/sysext/frontend/ext_tables.sql
typo3/sysext/install/Classes/Updates/BackendLayoutIconUpdateWizard.php [new file with mode: 0644]
typo3/sysext/install/ext_localconf.php

index cce602d..8601df5 100644 (file)
@@ -32,6 +32,7 @@ use TYPO3\CMS\Core\Localization\LanguageService;
 use TYPO3\CMS\Core\Messaging\FlashMessage;
 use TYPO3\CMS\Core\Messaging\FlashMessageQueue;
 use TYPO3\CMS\Core\Messaging\FlashMessageService;
+use TYPO3\CMS\Core\Resource\FileRepository;
 use TYPO3\CMS\Core\Utility\ArrayUtility;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 use TYPO3\CMS\Core\Utility\MathUtility;
@@ -479,6 +480,7 @@ abstract class AbstractItemProvider
         }
 
         $iconFactory = GeneralUtility::makeInstance(IconFactory::class);
+        $fileRepository = GeneralUtility::makeInstance(FileRepository::class);
 
         while ($foreignRow = $queryResult->fetch()) {
             BackendUtility::workspaceOL($foreignTable, $foreignRow);
@@ -486,20 +488,34 @@ abstract class AbstractItemProvider
                 // If the foreign table sets selicon_field, this field can contain an image
                 // that represents this specific row.
                 $iconFieldName = '';
+                $isReferenceField = false;
                 if (!empty($GLOBALS['TCA'][$foreignTable]['ctrl']['selicon_field'])) {
                     $iconFieldName = $GLOBALS['TCA'][$foreignTable]['ctrl']['selicon_field'];
+                    if ($GLOBALS['TCA'][$foreignTable]['columns'][$iconFieldName]['config']['type'] === 'inline'
+                        && $GLOBALS['TCA'][$foreignTable]['columns'][$iconFieldName]['config']['foreign_table'] === 'sys_file_reference') {
+                        $isReferenceField = true;
+                    }
                 }
-                $iconPath = '';
-                if (!empty($GLOBALS['TCA'][$foreignTable]['ctrl']['selicon_field_path'])) {
-                    $iconPath = $GLOBALS['TCA'][$foreignTable]['ctrl']['selicon_field_path'];
-                }
-                if ($iconFieldName && $iconPath && $foreignRow[$iconFieldName]) {
-                    // Prepare the row icon if available
-                    $iParts = GeneralUtility::trimExplode(',', $foreignRow[$iconFieldName], true);
-                    $icon = $iconPath . '/' . trim($iParts[0]);
+                $icon = '';
+                if ($isReferenceField) {
+                    $references = $fileRepository->findByRelation($foreignTable, $iconFieldName, $foreignRow['uid']);
+                    if (is_array($references) && !empty($references)) {
+                        $icon = reset($references);
+                        $icon = $icon->getPublicUrl();
+                    }
                 } else {
-                    // Else, determine icon based on record type, or a generic fallback
-                    $icon = $iconFactory->mapRecordTypeToIconIdentifier($foreignTable, $foreignRow);
+                    $iconPath = '';
+                    if (!empty($GLOBALS['TCA'][$foreignTable]['ctrl']['selicon_field_path'])) {
+                        $iconPath = $GLOBALS['TCA'][$foreignTable]['ctrl']['selicon_field_path'];
+                    }
+                    if ($iconFieldName && $iconPath && $foreignRow[$iconFieldName]) {
+                        // Prepare the row icon if available
+                        $iParts = GeneralUtility::trimExplode(',', $foreignRow[$iconFieldName], true);
+                        $icon = $iconPath . '/' . trim($iParts[0]);
+                    } else {
+                        // Else, determine icon based on record type, or a generic fallback
+                        $icon = $iconFactory->mapRecordTypeToIconIdentifier($foreignTable, $foreignRow);
+                    }
                 }
                 // Add the item
                 $items[] = [
index eb26963..9aece71 100644 (file)
@@ -16,6 +16,7 @@ namespace TYPO3\CMS\Backend\View\BackendLayout;
 
 use Doctrine\Common\Collections\Expr\Comparison;
 use TYPO3\CMS\Core\Database\ConnectionPool;
+use TYPO3\CMS\Core\Resource\FileRepository;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 
 /**
@@ -108,27 +109,26 @@ class DefaultDataProvider implements DataProviderInterface
     protected function createBackendLayout(array $data)
     {
         $backendLayout = BackendLayout::create($data['uid'], $data['title'], $data['config']);
-        $backendLayout->setIconPath($this->getIconPath($data['icon']));
+        $backendLayout->setIconPath($this->getIconPath($data));
         $backendLayout->setData($data);
         return $backendLayout;
     }
 
     /**
-     * Gets and sanitizes the icon path.
+     * Resolves the icon from the database record
      *
-     * @param string $icon Name of the icon file
+     * @param array $icon
      * @return string
      */
-    protected function getIconPath($icon)
+    protected function getIconPath(array $icon)
     {
-        $iconPath = '';
-
-        if (!empty($icon)) {
-            $path = rtrim($GLOBALS['TCA']['backend_layout']['ctrl']['selicon_field_path'], '/') . '/';
-            $iconPath = $path . $icon;
+        $fileRepository = GeneralUtility::makeInstance(FileRepository::class);
+        $references = $fileRepository->findByRelation($this->tableName, 'icon', $icon['uid']);
+        if (!empty($references)) {
+            $icon = reset($references);
+            return $icon->getPublicUrl();
         }
-
-        return $iconPath;
+        return '';
     }
 
     /**
diff --git a/typo3/sysext/core/Documentation/Changelog/master/Breaking-83153-MigratedBackendLayoutIconToFileAbstractionLayer.rst b/typo3/sysext/core/Documentation/Changelog/master/Breaking-83153-MigratedBackendLayoutIconToFileAbstractionLayer.rst
new file mode 100644 (file)
index 0000000..2e8e072
--- /dev/null
@@ -0,0 +1,39 @@
+.. include:: ../../Includes.txt
+
+======================================================
+Breaking: #83153 - Migrated backend_layout.icon to FAL
+======================================================
+
+See :issue:`83153`
+
+Description
+===========
+
+The existing database field "icon" for Backend Layouts put into the database, was previously a file upload field,
+putting all icons under `uploads/media`. The field is migrated to the File Abstraction Layer (FAL), having
+proper file relations like all other parts of TYPO3 core.
+
+
+Impact
+======
+
+When working with the TCA for the backend_layout.icon field, sys_file_reference relations are now expected.
+When querying the database table directly, icon only contains the number of references of this backend layout.
+
+
+Affected Installations
+======================
+
+Installations with custom backend layout icons and, more specifically extensions dealing with the database
+table directly.
+
+
+Migration
+=========
+
+An upgrade wizard in the TYPO3 install tool moves all existing icons of backend_layouts from `uploads/media` to
+`fileadmin/_migrated/backend_layouts/`.
+
+For extensions directly working on the database table, the database access needs to be modified.
+
+.. index:: Database, NotScanned
\ No newline at end of file
index a461c9b..37a2298 100644 (file)
@@ -17,8 +17,7 @@ return [
         'typeicon_classes' => [
             'default' => 'mimetypes-x-backend_layout'
         ],
-        'selicon_field' => 'icon',
-        'selicon_field_path' => 'uploads/media'
+        'selicon_field' => 'icon'
     ],
     'interface' => [
         'showRecordFieldList' => 'title,config,description,hidden,icon'
@@ -59,14 +58,16 @@ return [
         'icon' => [
             'label' => 'LLL:EXT:frontend/Resources/Private/Language/locallang_tca.xlf:backend_layout.icon',
             'exclude' => true,
-            'config' => [
-                'type' => 'group',
-                'internal_type' => 'file',
-                'allowed' => 'jpg,gif,png',
-                'uploadfolder' => 'uploads/media',
-                'size' => 1,
-                'maxitems' => 1
-            ]
+            'config' => \TYPO3\CMS\Core\Utility\ExtensionManagementUtility::getFileFieldTCAConfig(
+                'icon',
+                [
+                    'maxitems' => 1,
+                    'appearance' => [
+                        'createNewRelationLinkTitle' => 'LLL:EXT:frontend/Resources/Private/Language/locallang_ttc.xlf:images.addFileReference'
+                    ],
+                ],
+                $GLOBALS['TYPO3_CONF_VARS']['GFX']['imagefile_ext']
+            )
         ]
     ],
     'types' => [
index 23a2737..9c65091 100644 (file)
@@ -316,7 +316,7 @@ CREATE TABLE backend_layout (
        title varchar(255) DEFAULT '' NOT NULL,
        description text,
        config text NOT NULL,
-       icon text NOT NULL,
+       icon text,
 
        PRIMARY KEY (uid),
        KEY parent (pid),
diff --git a/typo3/sysext/install/Classes/Updates/BackendLayoutIconUpdateWizard.php b/typo3/sysext/install/Classes/Updates/BackendLayoutIconUpdateWizard.php
new file mode 100644 (file)
index 0000000..1486f44
--- /dev/null
@@ -0,0 +1,310 @@
+<?php
+declare(strict_types=1);
+namespace TYPO3\CMS\Install\Updates;
+
+/**
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+use Doctrine\DBAL\DBALException;
+use TYPO3\CMS\Core\Database\ConnectionPool;
+use TYPO3\CMS\Core\Database\Query\Expression\ExpressionBuilder;
+use TYPO3\CMS\Core\Log\Logger;
+use TYPO3\CMS\Core\Log\LogManager;
+use TYPO3\CMS\Core\Resource\File;
+use TYPO3\CMS\Core\Resource\ResourceStorage;
+use TYPO3\CMS\Core\Resource\StorageRepository;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
+/**
+ * Upgrade wizard which goes through all files referenced in backend_layout.icon
+ * and creates sys_file records as well as sys_file_reference records for each hit.
+ */
+class BackendLayoutIconUpdateWizard extends AbstractUpdate
+{
+    /**
+     * @var string
+     */
+    protected $title = 'Migrate all file relations from backend_layout.icon to sys_file_references';
+
+    /**
+     * @var ResourceStorage
+     */
+    protected $storage;
+
+    /**
+     * @var Logger
+     */
+    protected $logger;
+
+    /**
+     * Table to migrate records from
+     *
+     * @var string
+     */
+    protected $table = 'backend_layout';
+
+    /**
+     * Table field holding the migration to be
+     *
+     * @var string
+     */
+    protected $fieldToMigrate = 'icon';
+
+    /**
+     * the source file resides here
+     *
+     * @var string
+     */
+    protected $sourcePath = 'uploads/media/';
+
+    /**
+     * target folder after migration
+     * Relative to fileadmin
+     *
+     * @var string
+     */
+    protected $targetPath = '_migrated/backend_layouts/';
+
+    /**
+     * Constructor
+     */
+    public function __construct()
+    {
+        $this->logger = GeneralUtility::makeInstance(LogManager::class)->getLogger(__CLASS__);
+    }
+
+    /**
+     * Checks if an update is needed
+     *
+     * @param string &$description The description for the update
+     *
+     * @return bool TRUE if an update is needed, FALSE otherwise
+     */
+    public function checkForUpdate(&$description)
+    {
+        if ($this->isWizardDone()) {
+            return false;
+        }
+
+        // If there are no valid records, the wizard can be marked as done directly
+        $dbQueries = [];
+        $records = $this->getRecordsFromTable($dbQueries);
+        if (empty($records)) {
+            $this->markWizardAsDone();
+            return false;
+        }
+
+        $description = 'This update wizard goes through all files that are referenced in the backend_layout.icon field'
+            . ' and adds the files to the FAL File Index.<br />'
+            . 'It also moves the files from uploads/ to the fileadmin/_migrated/ path.';
+
+        return true;
+    }
+
+    /**
+     * Performs the database update.
+     *
+     * @param array &$dbQueries Queries done in this update
+     * @param string &$customMessage Custom message
+     * @return bool TRUE on success, FALSE on error
+     */
+    public function performUpdate(array &$dbQueries, &$customMessage)
+    {
+        $customMessage = '';
+        try {
+            $storages = GeneralUtility::makeInstance(StorageRepository::class)->findAll();
+            $this->storage = $storages[0];
+
+            $records = $this->getRecordsFromTable($dbQueries);
+            foreach ($records as $record) {
+                $this->migrateField($record, $customMessage, $dbQueries);
+            }
+
+            $this->markWizardAsDone();
+        } catch (\Exception $e) {
+            $customMessage .= PHP_EOL . $e->getMessage();
+        }
+
+        return empty($customMessage);
+    }
+
+    /**
+     * Get records from table where the field to migrate is not empty (NOT NULL and != '')
+     * and also not numeric (which means that it is migrated)
+     *
+     * @param array $dbQueries
+     *
+     * @return array
+     * @throws \RuntimeException
+     */
+    protected function getRecordsFromTable(&$dbQueries)
+    {
+        $connectionPool = GeneralUtility::makeInstance(ConnectionPool::class);
+        $queryBuilder = $connectionPool->getQueryBuilderForTable($this->table);
+        $queryBuilder->getRestrictions()->removeAll();
+
+        try {
+            $result = $queryBuilder
+                ->select('uid', 'pid', $this->fieldToMigrate)
+                ->from($this->table)
+                ->where(
+                    $queryBuilder->expr()->isNotNull($this->fieldToMigrate),
+                    $queryBuilder->expr()->neq(
+                        $this->fieldToMigrate,
+                        $queryBuilder->createNamedParameter('', \PDO::PARAM_STR)
+                    ),
+                    $queryBuilder->expr()->comparison(
+                        'CAST(CAST(' . $queryBuilder->quoteIdentifier($this->fieldToMigrate) . ' AS DECIMAL) AS CHAR)',
+                        ExpressionBuilder::NEQ,
+                        'CAST(' . $queryBuilder->quoteIdentifier($this->fieldToMigrate) . ' AS CHAR)'
+                    )
+                )
+                ->orderBy('uid')
+                ->execute();
+
+            $dbQueries[] = $queryBuilder->getSQL();
+
+            return $result->fetchAll();
+        } catch (DBALException $e) {
+            throw new \RuntimeException(
+                'Database query failed. Error was: ' . $e->getPrevious()->getMessage(),
+                1511950673
+            );
+        }
+    }
+
+    /**
+     * Migrates a single field.
+     *
+     * @param array $row
+     * @param string $customMessage
+     * @param array $dbQueries
+     *
+     * @throws \Exception
+     */
+    protected function migrateField($row, &$customMessage, &$dbQueries)
+    {
+        $fieldItems = GeneralUtility::trimExplode(',', $row[$this->fieldToMigrate], true);
+        if (empty($fieldItems) || is_numeric($row[$this->fieldToMigrate])) {
+            return;
+        }
+        $fileadminDirectory = rtrim($GLOBALS['TYPO3_CONF_VARS']['BE']['fileadminDir'], '/') . '/';
+        $i = 0;
+
+        $storageUid = (int)$this->storage->getUid();
+        $connectionPool = GeneralUtility::makeInstance(ConnectionPool::class);
+
+        foreach ($fieldItems as $item) {
+            $fileUid = null;
+            $sourcePath = PATH_site . $this->sourcePath . $item;
+            $targetDirectory = PATH_site . $fileadminDirectory . $this->targetPath;
+            $targetPath = $targetDirectory . basename($item);
+
+            // maybe the file was already moved, so check if the original file still exists
+            if (file_exists($sourcePath)) {
+                if (!is_dir($targetDirectory)) {
+                    GeneralUtility::mkdir_deep($targetDirectory);
+                }
+
+                // see if the file already exists in the storage
+                $fileSha1 = sha1_file($sourcePath);
+
+                $queryBuilder = $connectionPool->getQueryBuilderForTable('sys_file');
+                $queryBuilder->getRestrictions()->removeAll();
+                $existingFileRecord = $queryBuilder->select('uid')->from('sys_file')->where(
+                    $queryBuilder->expr()->eq(
+                        'sha1',
+                        $queryBuilder->createNamedParameter($fileSha1, \PDO::PARAM_STR)
+                    ),
+                    $queryBuilder->expr()->eq(
+                        'storage',
+                        $queryBuilder->createNamedParameter($storageUid, \PDO::PARAM_INT)
+                    )
+                )->execute()->fetch();
+
+                // the file exists, the file does not have to be moved again
+                if (is_array($existingFileRecord)) {
+                    $fileUid = $existingFileRecord['uid'];
+                } else {
+                    // just move the file (no duplicate)
+                    rename($sourcePath, $targetPath);
+                }
+            }
+
+            if ($fileUid === null) {
+                // get the File object if it hasn't been fetched before
+                try {
+                    // if the source file does not exist, we should just continue, but leave a message in the docs;
+                    // ideally, the user would be informed after the update as well.
+                    /** @var File $file */
+                    $file = $this->storage->getFile($this->targetPath . $item);
+                    $fileUid = $file->getUid();
+                } catch (\InvalidArgumentException $e) {
+
+                    // no file found, no reference can be set
+                    $this->logger->notice(
+                        'File ' . $this->sourcePath . $item . ' does not exist. Reference was not migrated.',
+                        [
+                            'table' => $this->table,
+                            'record' => $row,
+                            'field' => $this->fieldToMigrate,
+                        ]
+                    );
+
+                    $format = 'File \'%s\' does not exist. Referencing field: %s.%d.%s. The reference was not migrated.';
+                    $message = sprintf(
+                        $format,
+                        $this->sourcePath . $item,
+                        $this->table,
+                        $row['uid'],
+                        $this->fieldToMigrate
+                    );
+                    $customMessage .= PHP_EOL . $message;
+                    continue;
+                }
+            }
+
+            if ($fileUid > 0) {
+                $fields = [
+                    'fieldname' => $this->fieldToMigrate,
+                    'table_local' => 'sys_file',
+                    'pid' => ($this->table === 'pages' ? $row['uid'] : $row['pid']),
+                    'uid_foreign' => $row['uid'],
+                    'uid_local' => $fileUid,
+                    'tablenames' => $this->table,
+                    'crdate' => time(),
+                    'tstamp' => time(),
+                    'sorting' => ($i + 256),
+                    'sorting_foreign' => $i,
+                ];
+
+                $queryBuilder = $connectionPool->getQueryBuilderForTable('sys_file_reference');
+                $queryBuilder->insert('sys_file_reference')->values($fields)->execute();
+                $dbQueries[] = str_replace(LF, ' ', $queryBuilder->getSQL());
+                ++$i;
+            }
+        }
+
+        // Update referencing table's original field to now contain the count of references,
+        // but only if all new references could be set
+        if ($i === count($fieldItems)) {
+            $queryBuilder = $connectionPool->getQueryBuilderForTable($this->table);
+            $queryBuilder->update($this->table)->where(
+                $queryBuilder->expr()->eq(
+                    'uid',
+                    $queryBuilder->createNamedParameter($row['uid'], \PDO::PARAM_INT)
+                )
+            )->set($this->fieldToMigrate, $i)->execute();
+            $dbQueries[] = str_replace(LF, ' ', $queryBuilder->getSQL());
+        }
+    }
+}
index 765af30..6742cde 100644 (file)
@@ -54,6 +54,8 @@ $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['ext/install']['update']['pagesLanguag
     = \TYPO3\CMS\Install\Updates\MigratePagesLanguageOverlayUpdate::class;
 $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['ext/install']['update']['pagesLanguageOverlayBeGroupsAccessRights']
     = \TYPO3\CMS\Install\Updates\MigratePagesLanguageOverlayBeGroupsAccessRights::class;
+$GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['ext/install']['update']['backendLayoutIcons']
+    = \TYPO3\CMS\Install\Updates\BackendLayoutIconUpdateWizard::class;
 
 $iconRegistry = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Core\Imaging\IconRegistry::class);
 $icons = [