[BUGFIX] Avoid unnecessary generation of processed files 44/39644/14
authorMarkus Klein <markus.klein@typo3.org>
Thu, 21 May 2015 00:36:46 +0000 (02:36 +0200)
committerAlexander Opitz <opitz.alexander@googlemail.com>
Wed, 10 Jun 2015 07:40:20 +0000 (09:40 +0200)
Removing [GFX] off the checksum data used to identify a processed file
ensures that data type inconsistencies within [GFX] do not cause a
regeneration of the processed file.

A new upgrade wizard for the Install Tool is added to migrate existing
processed files without the need to regenerate those.

Resolves: #66614
Releases: master, 6.2
Change-Id: Id90ce923b26ee726d15cf3d0cfcafc4ec4d7405d
Reviewed-on: http://review.typo3.org/39644
Reviewed-by: Andreas Fernandez <typo3@scripting-base.de>
Reviewed-by: Xavier Perseguers <xavier@typo3.org>
Tested-by: Xavier Perseguers <xavier@typo3.org>
Reviewed-by: Stefan Neufeind <typo3.neufeind@speedpartner.de>
Reviewed-by: Alexander Opitz <opitz.alexander@googlemail.com>
Tested-by: Alexander Opitz <opitz.alexander@googlemail.com>
typo3/sysext/core/Classes/Resource/Processing/AbstractGraphicalTask.php
typo3/sysext/core/Documentation/Changelog/master/Important-66614-ChecksumForProcessedFilesChanged.rst [new file with mode: 0644]
typo3/sysext/install/Classes/Updates/ProcessedFileChecksumUpdate.php [new file with mode: 0644]
typo3/sysext/install/ext_localconf.php

index a4fed74..5c632d4 100644 (file)
@@ -32,20 +32,6 @@ abstract class AbstractGraphicalTask extends AbstractTask {
        protected $targetFileExtension;
 
        /**
-        * Sets parameters needed in the checksum. Can be overridden to add additional parameters to the checksum.
-        * This should include all parameters that could possibly vary between different task instances, e.g. the
-        * TYPO3 image configuration in TYPO3_CONF_VARS[GFX] for graphic processing tasks.
-        *
-        * @return array
-        */
-       protected function getChecksumData() {
-               return array_merge(
-                       parent::getChecksumData(),
-                       array(serialize($GLOBALS['TYPO3_CONF_VARS']['GFX']))
-               );
-       }
-
-       /**
         * Returns the name the processed file should have
         * in the filesystem.
         *
diff --git a/typo3/sysext/core/Documentation/Changelog/master/Important-66614-ChecksumForProcessedFilesChanged.rst b/typo3/sysext/core/Documentation/Changelog/master/Important-66614-ChecksumForProcessedFilesChanged.rst
new file mode 100644 (file)
index 0000000..793e5f6
--- /dev/null
@@ -0,0 +1,28 @@
+===================================================================
+Important: #66614 - Checksums for processed files have been changed
+===================================================================
+
+Description
+===========
+
+The base data used for the checksum calculation of processed files have been changed.
+The checksum is used to identify changes, which require regeneration of processed files.
+
+Formerly the ``GFX`` section of the ``TYPO3_CONF_VARS`` was included in this base data,
+which caused weird problems in some cases.
+
+With TYPO3 CMS 7.3 (and 6.2.13) this has been changed. In case you are adjusting ``GFX`` settings and you want
+processed files to be regenerated, you need to manually clean the existing processed files by using
+the Clean up utility in the Install Tool.
+
+Since the base data are different now, the Core would not recognize the existing processed files as
+valid files and would delete those and build a new version.
+In case you are having a large installation, you might want to avoid this costly operation.
+The Install Tool provides a dedicated Upgrade Wizard for you, which avoids the expensive
+regeneration of processed files by updating the checksum of all existing processed files.
+
+.. note::
+
+       The Upgrade Wizard is only relevant for you if you're upgrading from any TYPO3 CMS version below 7.3 or 6.2.13.
+       Any upgrade from 7.3 or later or from 6.2.13 or later to a newer version does **not** require to run the wizard.
+
diff --git a/typo3/sysext/install/Classes/Updates/ProcessedFileChecksumUpdate.php b/typo3/sysext/install/Classes/Updates/ProcessedFileChecksumUpdate.php
new file mode 100644 (file)
index 0000000..575be1c
--- /dev/null
@@ -0,0 +1,138 @@
+<?php
+namespace TYPO3\CMS\Install\Updates;
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+use TYPO3\CMS\Core\Resource\ProcessedFile;
+use TYPO3\CMS\Core\Resource\ResourceFactory;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
+/**
+ * Updates the checksum of sys_file_processedfile records to avoid regeneration of the thumbnails
+ */
+class ProcessedFileChecksumUpdate extends AbstractUpdate {
+
+       /**
+        * @var string
+        */
+       protected $title = '[Optional] Update sys_file_processedfile records to match new checksum calculation.';
+
+       /**
+        * Checks if an update is needed
+        *
+        * @param string &$description The description for the update
+        * @return bool Whether an update is needed (TRUE) or not (FALSE)
+        */
+       public function checkForUpdate(&$description) {
+               if ($this->isWizardDone()) {
+                       return FALSE;
+               }
+
+               $join = 'sys_file_processedfile LEFT JOIN sys_registry ON entry_key = sys_file_processedfile.uid AND entry_namespace = \'ProcessedFileChecksumUpdate\'';
+               $count = $this->getDatabaseConnection()->exec_SELECTcountRows('*', $join, '(entry_key IS NULL AND sys_file_processedfile.identifier <> \'\') OR sys_file_processedfile.width IS NULL');
+               if (!$count) {
+                       return FALSE;
+               }
+
+               $description = 'The checksum calculation for processed files (image thumbnails) has been changed with TYPO3 CMS 7.3 and 6.2.13.
+This means that your processed files need to be updated, if you update from versions <strong>below TYPO3 CMS 7.3 or 6.2.13</strong>.<br />
+This can either happen on demand, when the processed file is first needed, or by executing this wizard, which updates all processed images at once.<br />
+<strong>Important:</strong> If you have lots of processed files, you should prefer using this wizard, otherwise this might cause a lot of work for your server.';
+
+               return TRUE;
+       }
+
+       /**
+        * Performs the update
+        *
+        * @param array &$databaseQueries Queries done in this update
+        * @param mixed &$customMessages Custom messages
+        * @return bool
+        */
+       public function performUpdate(array &$databaseQueries, &$customMessages) {
+               $db = $this->getDatabaseConnection();
+
+               // remove all invalid records which hold NULL values
+               $db->exec_DELETEquery('sys_file_processedfile', 'width IS NULL or height IS NULL');
+
+               $factory = GeneralUtility::makeInstance(ResourceFactory::class);
+
+               $join = 'sys_file_processedfile LEFT JOIN sys_registry ON entry_key = sys_file_processedfile.uid AND entry_namespace = \'ProcessedFileChecksumUpdate\'';
+               $res = $db->exec_SELECTquery('sys_file_processedfile.*', $join, 'entry_key IS NULL AND sys_file_processedfile.identifier <> \'\'');
+               while ($processedFileRow = $db->sql_fetch_assoc($res)) {
+                       $storage = $factory->getStorageObject($processedFileRow['storage']);
+                       if (!$storage) {
+                               // invalid storage, delete record, we can't take care of the associated file
+                               $db->exec_DELETEquery('sys_file_processedfile', 'uid=' . $processedFileRow['uid']);
+                               continue;
+                       }
+
+                       if ($storage->getDriverType() !== 'Local') {
+                               // non-local storage, we can't treat this, skip the record and mark it done
+                               $db->exec_INSERTquery('sys_registry', array('entry_namespace' => 'ProcessedFileChecksumUpdate', 'entry_key' => $processedFileRow['uid']));
+                               continue;
+                       }
+
+                       $configuration = $storage->getConfiguration();
+                       if ($configuration['pathType'] === 'relative') {
+                               $absoluteBasePath = PATH_site . $configuration['basePath'];
+                       } else {
+                               $absoluteBasePath = $configuration['basePath'];
+                       }
+                       $filePath = rtrim($absoluteBasePath, '/') . '/' . ltrim($processedFileRow['identifier'], '/');
+
+                       try {
+                               $originalFile = $factory->getFileObject($processedFileRow['original']);
+                       } catch (\Exception $e) {
+                               // no original file there anymore, delete local file
+                               @unlink($filePath);
+                               $db->exec_DELETEquery('sys_file_processedfile', 'uid=' . $processedFileRow['uid']);
+                               continue;
+                       }
+
+                       $processedFileObject = new ProcessedFile($originalFile, '', array(), $processedFileRow);
+
+                       // calculate new checksum and name
+                       $newChecksum = $processedFileObject->calculateChecksum();
+
+                       // if the checksum already matches, there is nothing to do
+                       if ($newChecksum !== $processedFileRow['checksum']) {
+                               $newName = str_replace($processedFileRow['checksum'], $newChecksum, $processedFileRow['name']);
+                               $newIdentifier = str_replace($processedFileRow['checksum'], $newChecksum, $processedFileRow['identifier']);
+                               $newFilePath = str_replace($processedFileRow['checksum'], $newChecksum, $filePath);
+
+                               // rename file
+                               if (@rename($filePath, $newFilePath)) {
+                                       // save result back into database
+                                       $fields = array(
+                                               'tstamp' => time(),
+                                               'identifier' => $newIdentifier,
+                                               'name' => $newName,
+                                               'checksum' => $newChecksum
+                                       );
+                                       $db->exec_UPDATEquery('sys_file_processedfile', 'uid=' . $processedFileRow['uid'], $fields);
+                               }
+                               // if the rename of the file failed, keep the record, but do not bother with it again
+                       }
+
+                       // remember we finished this record
+                       $db->exec_INSERTquery('sys_registry', array('entry_namespace' => 'ProcessedFileChecksumUpdate', 'entry_key' => $processedFileRow['uid']));
+               }
+
+               $db->exec_DELETEquery('sys_registry', 'entry_namespace = \'ProcessedFileChecksumUpdate\'');
+               $this->markWizardAsDone();
+               return TRUE;
+       }
+
+}
index f346b80..d70b906 100644 (file)
@@ -6,6 +6,7 @@ $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['ext/install']['update']['backendUserS
 $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['ext/install']['update']['languageIsoCode'] = \TYPO3\CMS\Install\Updates\LanguageIsoCodeUpdate::class;
 $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['ext/install']['update']['PageShortcutParent'] = \TYPO3\CMS\Install\Updates\PageShortcutParentUpdate::class;
 $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['ext/install']['update']['backendShortcuts'] = \TYPO3\CMS\Install\Updates\MigrateShortcutUrlsUpdate::class;
+$GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['ext/install']['update']['processedFilesChecksum'] = \TYPO3\CMS\Install\Updates\ProcessedFileChecksumUpdate::class;
 
 $signalSlotDispatcher = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Extbase\SignalSlot\Dispatcher::class);
 $signalSlotDispatcher->connect(