[!!!][TASK] Migrate DB field fe_users.image to FAL 39/47139/13
authorpythondetective <pythondetective@gmail.com>
Sun, 6 Mar 2016 13:00:27 +0000 (14:00 +0100)
committerBenni Mack <benni@typo3.org>
Wed, 3 Aug 2016 14:32:32 +0000 (16:32 +0200)
Using the File Abstraction Layer for this field as well,
like it is handled within tt_content.image and pages.media
already.

Also, the patch adds a migration wizard in the install
tool to move existing images to the File Abstraction Layer.

Resolves: #74375
Releases: master
Change-Id: I157d619dec18ef24e7d1e2d8694ed150304549d7
Reviewed-on: https://review.typo3.org/47139
Tested-by: Bamboo TYPO3com <info@typo3.com>
Reviewed-by: Anja Leichsenring <aleichsenring@ab-softlab.de>
Tested-by: Anja Leichsenring <aleichsenring@ab-softlab.de>
Reviewed-by: Benni Mack <benni@typo3.org>
Tested-by: Benni Mack <benni@typo3.org>
typo3/sysext/core/Documentation/Changelog/master/Breaking-74375-Fe_usersimageMigratedToFAL.rst [new file with mode: 0644]
typo3/sysext/frontend/Configuration/TCA/fe_users.php
typo3/sysext/install/Classes/Updates/FrontendUserImageUpdateWizard.php [new file with mode: 0644]
typo3/sysext/install/ext_localconf.php

diff --git a/typo3/sysext/core/Documentation/Changelog/master/Breaking-74375-Fe_usersimageMigratedToFAL.rst b/typo3/sysext/core/Documentation/Changelog/master/Breaking-74375-Fe_usersimageMigratedToFAL.rst
new file mode 100644 (file)
index 0000000..ed788b6
--- /dev/null
@@ -0,0 +1,32 @@
+=================================================
+Breaking: #74375 - fe_users.image migrated to FAL
+=================================================
+
+Description
+===========
+
+The Frontend User field "image" was previously handled via images located under uploads/pics/, as simple file references
+not able to handle duplicate images etc.
+
+The field is now set up as adding references from the File Abstraction Layer avoiding the need to copy all images to uploads/pics/.
+
+
+Impact
+======
+
+Using the ``fe_users.image`` field in the frontend or backend will result in unexpected behaviour.
+
+
+Affected Installations
+======================
+
+Any TYPO3 installation using the field "image" within the database table "fe_users", common in third-party extensions using
+the field for storing images for frontend users (like mm_forum).
+
+
+Migration
+=========
+
+Use the File Abstraction Layer for outputting and dealing with rendering or changing images for frontend users.
+
+Use the migration wizard provided in the install tool to migrate existing code to proper file references.
\ No newline at end of file
index 231333c..ff2c469 100644 (file)
@@ -209,15 +209,13 @@ return array(
         'image' => array(
             'exclude' => true,
             'label' => 'LLL:EXT:lang/locallang_general.xlf:LGL.image',
-            'config' => array(
-                'type' => 'group',
-                'internal_type' => 'file',
-                'allowed' => $GLOBALS['TYPO3_CONF_VARS']['GFX']['imagefile_ext'],
-                'uploadfolder' => 'uploads/pics',
-                'show_thumbs' => true,
-                'size' => 3,
-                'maxitems' => 6,
-                'minitems' => 0
+            'config' => \TYPO3\CMS\Core\Utility\ExtensionManagementUtility::getFileFieldTCAConfig(
+                'image',
+                array(
+                    'maxitems' => 6,
+                    'minitems'=> 0
+                ),
+                $GLOBALS['TYPO3_CONF_VARS']['GFX']['imagefile_ext']
             )
         ),
         'disable' => array(
diff --git a/typo3/sysext/install/Classes/Updates/FrontendUserImageUpdateWizard.php b/typo3/sysext/install/Classes/Updates/FrontendUserImageUpdateWizard.php
new file mode 100644 (file)
index 0000000..1b477cb
--- /dev/null
@@ -0,0 +1,332 @@
+<?php
+namespace TYPO3\CMS\Install\Updates;
+
+/**
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+use TYPO3\CMS\Core\Database\ConnectionPool;
+use TYPO3\CMS\Core\Database\Query\Expression\ExpressionBuilder;
+use TYPO3\CMS\Core\Database\Query\Restriction\DeletedRestriction;
+use TYPO3\CMS\Core\Log\Logger;
+use TYPO3\CMS\Core\Log\LogManager;
+use TYPO3\CMS\Core\Registry;
+use TYPO3\CMS\Core\Resource\ResourceStorage;
+use TYPO3\CMS\Core\Resource\StorageRepository;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
+/**
+ * Upgrade wizard which goes through all files referenced in fe_users::image
+ * and creates sys_file records as well as sys_file_reference records for each hit.
+ */
+class FrontendUserImageUpdateWizard extends AbstractUpdate
+{
+
+    /**
+     * Number of records fetched per database query
+     * Used to prevent memory overflows for huge databases
+     */
+    const RECORDS_PER_QUERY = 1000;
+
+    /**
+     * @var string
+     */
+    protected $title = 'Migrate all file relations from fe_users.image to sys_file_references';
+
+    /**
+     * @var ResourceStorage
+     */
+    protected $storage;
+
+    /**
+     * @var Logger
+     */
+    protected $logger;
+
+    /**
+     * Table to migrate records from
+     *
+     * @var string
+     */
+    protected $table = 'fe_users';
+
+    /**
+     * Table field holding the migration to be
+     *
+     * @var string
+     */
+    protected $fieldToMigrate = 'image';
+
+    /**
+     * the source file resides here
+     *
+     * @var string
+     */
+    protected $sourcePath = 'uploads/pics/';
+
+    /**
+     * target folder after migration
+     * Relative to fileadmin
+     *
+     * @var string
+     */
+    protected $targetPath = '_migrated/frontend_users/';
+
+    /**
+     * @var Registry
+     */
+    protected $registry;
+
+    /**
+     * @var string
+     */
+    protected $registryNamespace = 'FrontendUserImageUpdateWizard';
+
+    /**
+     * @var array
+     */
+    protected $recordOffset = array();
+
+    /**
+     * Constructor
+     */
+    public function __construct()
+    {
+        $this->logger = GeneralUtility::makeInstance(LogManager::class)->getLogger(__CLASS__);
+    }
+
+    /**
+     * Initialize the storage repository.
+     */
+    public function init()
+    {
+        $storages = GeneralUtility::makeInstance(StorageRepository::class)->findAll();
+        $this->storage = $storages[0];
+        $this->registry = GeneralUtility::makeInstance(Registry::class);
+        $this->recordOffset = $this->registry->get($this->registryNamespace, 'recordOffset', []);
+    }
+
+    /**
+     * Checks if an update is needed
+     *
+     * @param string &$description The description for the update
+     *
+     * @return bool TRUE if an update is needed, FALSE otherwise
+     */
+    public function checkForUpdate(&$description)
+    {
+        $description = 'This update wizard goes through all files that are referenced in the fe_users.image field'
+            . ' and adds the files to the FAL File Index.<br />'
+            . 'It also moves the files from uploads/ to the fileadmin/_migrated/ path.';
+
+        $this->init();
+
+        return !$this->isWizardDone() || $this->recordOffset !== [];
+    }
+
+    /**
+     * Performs the database update.
+     *
+     * @param array &$dbQueries Queries done in this update
+     * @param mixed &$customMessages Custom messages
+     *
+     * @return bool TRUE on success, FALSE on error
+     */
+    public function performUpdate(array &$dbQueries, &$customMessages)
+    {
+        try {
+            $this->init();
+
+            if (!isset($this->recordOffset[$this->table])) {
+                $this->recordOffset[$this->table] = 0;
+            }
+
+            do {
+                $limit = $this->recordOffset[$this->table] . ',' . self::RECORDS_PER_QUERY;
+                $records = $this->getRecordsFromTable($limit, $dbQueries);
+                foreach ($records as $record) {
+                    $this->migrateField($record, $customMessages, $dbQueries);
+                }
+                $this->registry->set($this->registryNamespace, 'recordOffset', $this->recordOffset);
+            } while (count($records) === self::RECORDS_PER_QUERY);
+
+            $this->markWizardAsDone();
+            $this->registry->remove($this->registryNamespace, 'recordOffset');
+        } catch (\Exception $e) {
+            $customMessages .= PHP_EOL . $e->getMessage();
+        }
+
+        return empty($customMessages);
+    }
+
+    /**
+     * Get records from table where the field to migrate is not empty (NOT NULL and != '')
+     * and also not numeric (which means that it is migrated)
+     *
+     * @param int $limit Maximum number records to select
+     * @param array $dbQueries
+     *
+     * @return array
+     * @throws \RuntimeException
+     */
+    protected function getRecordsFromTable($limit, &$dbQueries)
+    {
+        $connectionPool = GeneralUtility::makeInstance(ConnectionPool::class);
+        $queryBuilder = $connectionPool->getQueryBuilderForTable($this->table);
+
+        $queryBuilder->getRestrictions()
+            ->removeAll()
+            ->add(GeneralUtility::makeInstance(DeletedRestriction::class));
+
+        $stmt = $queryBuilder
+            ->select('uid', 'pid', $this->fieldToMigrate)
+            ->from($this->table)
+            ->where(
+                $queryBuilder->expr()->isNotNull($this->fieldToMigrate),
+                $queryBuilder->expr()->neq($this->fieldToMigrate,
+                    $queryBuilder->createNamedParameter('')),
+                $queryBuilder->expr()->comparison('CAST(CAST(' . $this->fieldToMigrate . ' AS DECIMAL) AS CHAR)',
+                    ExpressionBuilder::NEQ, 'CAST(' . $this->fieldToMigrate . ' AS CHAR)')
+            )
+            ->orderBy('uid')
+            ->setFirstResult($limit)
+            ->execute();
+
+        $dbQueries[] = $queryBuilder->getSQL();
+
+        if ($stmt->errorCode() > 0) {
+            throw new \RuntimeException('Database query failed. Error was: ' . implode(CRLF,
+                    $stmt->errorInfo()));
+        }
+
+        return $stmt->fetchAll();
+    }
+
+    /**
+     * Migrates a single field.
+     *
+     * @param array $row
+     * @param string $customMessages
+     * @param array $dbQueries
+     *
+     * @throws \Exception
+     */
+    protected function migrateField($row, &$customMessages, &$dbQueries)
+    {
+        $fieldItems = GeneralUtility::trimExplode(',', $row[$this->fieldToMigrate], true);
+        if (empty($fieldItems) || is_numeric($row[$this->fieldToMigrate])) {
+            return;
+        }
+        $fileadminDirectory = rtrim($GLOBALS['TYPO3_CONF_VARS']['BE']['fileadminDir'], '/') . '/';
+        $i = 0;
+
+        if (!PATH_site) {
+            throw new \Exception('PATH_site was undefined.');
+        }
+
+        $storageUid = (int)$this->storage->getUid();
+
+        $connectionPool = GeneralUtility::makeInstance(ConnectionPool::class);
+
+        foreach ($fieldItems as $item) {
+            $fileUid = null;
+            $sourcePath = PATH_site . $this->sourcePath . $item;
+            $targetDirectory = PATH_site . $fileadminDirectory . $this->targetPath;
+            $targetPath = $targetDirectory . basename($item);
+
+            // maybe the file was already moved, so check if the original file still exists
+            if (file_exists($sourcePath)) {
+                if (!is_dir($targetDirectory)) {
+                    GeneralUtility::mkdir_deep($targetDirectory);
+                }
+
+                // see if the file already exists in the storage
+                $fileSha1 = sha1_file($sourcePath);
+
+                $queryBuilder = $connectionPool->getQueryBuilderForTable('sys_file');
+                $queryBuilder->getRestrictions()->removeAll();
+                $existingFileRecord = $queryBuilder->select('uid')->from('sys_file')->where(
+                    $queryBuilder->expr()->eq('sha1',
+                        $queryBuilder->createNamedParameter($fileSha1)),
+                    $queryBuilder->expr()->eq('storage',
+                        $queryBuilder->createNamedParameter($storageUid))
+                )->execute()->fetch();
+
+                // the file exists, the file does not have to be moved again
+                if (is_array($existingFileRecord)) {
+                    $fileUid = $existingFileRecord['uid'];
+                } else {
+                    // just move the file (no duplicate)
+                    rename($sourcePath, $targetPath);
+                }
+            }
+
+            if ($fileUid === null) {
+                // get the File object if it hasn't been fetched before
+                try {
+                    // if the source file does not exist, we should just continue, but leave a message in the docs;
+                    // ideally, the user would be informed after the update as well.
+                    $file = $this->storage->getFile($this->targetPath . $item);
+                    $fileUid = $file->getUid();
+                } catch (\InvalidArgumentException $e) {
+
+                    // no file found, no reference can be set
+                    $this->logger->notice(
+                        'File ' . $this->sourcePath . $item . ' does not exist. Reference was not migrated.',
+                        [
+                            'table' => $this->table,
+                            'record' => $row,
+                            'field' => $this->fieldToMigrate,
+                        ]
+                    );
+
+                    $format = 'File \'%s\' does not exist. Referencing field: %s.%d.%s. The reference was not migrated.';
+                    $message = sprintf($format, $this->sourcePath . $item, $this->table,
+                        $row['uid'], $this->fieldToMigrate);
+                    $customMessages .= PHP_EOL . $message;
+                    continue;
+                }
+            }
+
+            if ($fileUid > 0) {
+                $fields = [
+                    'fieldname' => $this->fieldToMigrate,
+                    'table_local' => 'sys_file',
+                    'pid' => ($this->table === 'pages' ? $row['uid'] : $row['pid']),
+                    'uid_foreign' => $row['uid'],
+                    'uid_local' => $fileUid,
+                    'tablenames' => $this->table,
+                    'crdate' => time(),
+                    'tstamp' => time(),
+                    'sorting' => ($i + 256),
+                    'sorting_foreign' => $i,
+                ];
+
+                $queryBuilder = $connectionPool->getQueryBuilderForTable('sys_file_reference');
+                $queryBuilder->insert('sys_file_reference')->values($fields)->execute();
+                $dbQueries[] = str_replace(LF, ' ', $queryBuilder->getSQL());
+                ++$i;
+            }
+        }
+
+        // Update referencing table's original field to now contain the count of references,
+        // but only if all new references could be set
+        if ($i === count($fieldItems)) {
+            $queryBuilder = $connectionPool->getQueryBuilderForTable($this->table);
+            $queryBuilder->update($this->table)->where(
+                $queryBuilder->expr()->eq('uid', $row['uid'])
+            )->set($this->fieldToMigrate, $i)->execute();
+            $dbQueries[] = str_replace(LF, ' ', $queryBuilder->getSQL());
+        } else {
+            $this->recordOffset[$this->table]++;
+        }
+    }
+}
index 5773186..837b1af 100644 (file)
@@ -42,3 +42,5 @@ $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['ext/install']['update'][\TYPO3\CMS\In
     = \TYPO3\CMS\Install\Updates\WizardDoneToRegistry::class;
 $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['ext/install']['update'][\TYPO3\CMS\Install\Updates\BackendUserStartModuleUpdateAboutModule::class]
     = \TYPO3\CMS\Install\Updates\BackendUserStartModuleUpdateAboutModule::class;
+$GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['ext/install']['update'][\TYPO3\CMS\Install\Updates\FrontendUserImageUpdateWizard::class]
+    = \TYPO3\CMS\Install\Updates\FrontendUserImageUpdateWizard::class;