[TASK] Split sys_file table to extract metadata 03/24503/9
authorSteffen Ritter <info@rs-websystems.de>
Sun, 15 Sep 2013 15:11:51 +0000 (17:11 +0200)
committerSteffen Ritter <info@rs-websystems.de>
Sun, 13 Oct 2013 11:07:37 +0000 (13:07 +0200)
Parts of the sys_file table are internal cache data
for the FAL Api to work. Other parts are a meta-data
storage for the FAL frontend output.
On the one hand this mixes up several concerns, on
the other hand this makes it difficult to translate
and version metadata since this would result in
duplicate FAL index entries.

As a result those two concerns are split in separate
tables. The user does not "see" the sys_file table
anymore. Metadata is edited only in sys_file_metadata.

Change-Id: Ia6e3664ad5602affcba4e02cf415aa39436d4352
Resolves: #52726
Releases: 6.2
Reviewed-on: https://review.typo3.org/24503
Reviewed-by: Ernesto Baschny
Tested-by: Ernesto Baschny
Reviewed-by: Frans Saris
Tested-by: Frans Saris
Reviewed-by: Fabien Udriot
Tested-by: Fabien Udriot
Reviewed-by: Steffen Ritter
Tested-by: Steffen Ritter
16 files changed:
typo3/sysext/backend/Classes/Controller/ContentElement/ElementInformationController.php
typo3/sysext/core/Classes/Resource/File.php
typo3/sysext/core/Classes/Resource/Hook/FileInfoHook.php
typo3/sysext/core/Classes/Resource/Index/FileIndexRepository.php
typo3/sysext/core/Classes/Resource/Index/MetaDataRepository.php [new file with mode: 0644]
typo3/sysext/core/Configuration/DefaultConfiguration.php
typo3/sysext/core/Configuration/TCA/sys_file.php
typo3/sysext/core/Configuration/TCA/sys_file_metadata.php [new file with mode: 0644]
typo3/sysext/core/Configuration/TCA/sys_file_reference.php
typo3/sysext/core/Tests/Unit/Resource/FileTest.php
typo3/sysext/core/ext_tables.sql
typo3/sysext/filelist/Classes/FileList.php
typo3/sysext/install/Classes/Service/SilentConfigurationUpgradeService.php
typo3/sysext/install/Classes/Updates/FileTableSplittingUpdate.php [new file with mode: 0644]
typo3/sysext/install/ext_localconf.php
typo3/sysext/lang/locallang_tca.xlf

index df07a65..4020ce1 100644 (file)
@@ -179,10 +179,11 @@ class ElementInformationController {
                        $this->fileObject = $fileOrFolderObject;
                        $this->access = $this->fileObject->checkActionPermission('read');
                        $this->type = 'file';
-                       $this->table = 'sys_file';
+                       $this->table = 'sys_file_metadata';
 
                        try {
-                               $this->row = BackendUtility::getRecordWSOL($this->table, $this->fileObject->getUid());
+                               $metaData = $fileOrFolderObject->_getMetaData();
+                               $this->row = BackendUtility::getRecordWSOL($this->table, $metaData['uid']);
                        } catch (\Exception $e) {
                                $this->row = array();
                        }
index e803b1a..77b7632 100644 (file)
@@ -26,6 +26,10 @@ namespace TYPO3\CMS\Core\Resource;
  *
  *  This copyright notice MUST APPEAR in all copies of the script!
  ***************************************************************/
+
+use TYPO3\CMS\Core\Resource\Index\MetaDataRepository;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
 /**
  * File representation in the file abstraction layer.
  *
@@ -50,6 +54,11 @@ class File extends AbstractFile {
        protected $indexable = TRUE;
 
        /**
+        * @var array
+        */
+       protected $metaDataProperties = array();
+
+       /**
         * Set to TRUE while this file is being indexed - used to prevent some endless loops
         *
         * @var boolean
@@ -65,16 +74,6 @@ class File extends AbstractFile {
        protected $updatedProperties = array();
 
        /**
-        * @var Service\IndexerService
-        */
-       protected $indexerService = NULL;
-
-       /*********************************************
-        * GENERIC FILE TYPES
-        * these are generic filetypes or -groups,
-        * don't mix it up with mime types
-        *********************************************/
-       /**
         * Constructor for a file object. Should normally not be used directly, use
         * the corresponding factory methods instead.
         *
@@ -82,13 +81,15 @@ class File extends AbstractFile {
         * @param ResourceStorage $storage
         */
        public function __construct(array $fileData, ResourceStorage $storage) {
-               if (isset($fileData['uid']) && intval($fileData['uid']) > 0) {
-                       $this->indexed = TRUE;
-               }
                $this->identifier = $fileData['identifier'];
                $this->name = $fileData['name'];
                $this->properties = $fileData;
                $this->storage = $storage;
+
+               if (isset($fileData['uid']) && intval($fileData['uid']) > 0) {
+                       $this->indexed = TRUE;
+                       $this->loadMetaData();
+               }
        }
 
        /*******************************
@@ -104,7 +105,11 @@ class File extends AbstractFile {
                if ($this->indexed === NULL) {
                        $this->loadIndexRecord();
                }
-               return parent::getProperty($key);
+               if (parent::hasProperty($key)) {
+                       return parent::getProperty($key);
+               } else {
+                       return array_key_exists($key, $this->metaDataProperties) ? $this->metaDataProperties[$key] : NULL;
+               }
        }
 
        /**
@@ -116,7 +121,17 @@ class File extends AbstractFile {
                if ($this->indexed === NULL) {
                        $this->loadIndexRecord();
                }
-               return parent::getProperties();
+               return array_merge(parent::getProperties(), array_diff_key((array)$this->metaDataProperties, parent::getProperties()));
+       }
+
+       /**
+        * Returns the MetaData
+        *
+        * @return array|null
+        * @internal
+        */
+       public function _getMetaData() {
+               return $this->metaDataProperties;
        }
 
        /******************
@@ -164,23 +179,33 @@ class File extends AbstractFile {
         * @return void
         */
        protected function loadIndexRecord($indexIfNotIndexed = TRUE) {
-               if ($this->indexed !== NULL || !$this->indexable) {
+               if ($this->indexed !== NULL || !$this->indexable || $this->indexingInProgress) {
                        return;
                }
+               $this->indexingInProgress = TRUE;
 
-               $indexRecord = Index\FileIndexRepository::getInstance()->findOneByFileObject($this);
+               $indexRecord = $this->getFileIndexRepository()->findOneByCombinedIdentifier($this->getCombinedIdentifier());
                if ($indexRecord === FALSE && $indexIfNotIndexed) {
-                       $this->indexingInProgress = TRUE;
+                       // the IndexerService is not used at this place since, its not about additional MetaData anymore
                        $indexRecord = $this->getIndexerService()->indexFile($this, FALSE);
                        $this->mergeIndexRecord($indexRecord);
                        $this->indexed = TRUE;
-                       $this->indexingInProgress = FALSE;
+                       $this->loadMetaData();
                } elseif ($indexRecord !== FALSE) {
                        $this->mergeIndexRecord($indexRecord);
                        $this->indexed = TRUE;
+                       $this->loadMetaData();
                } else {
                        throw new \RuntimeException('Could not load index record for "' . $this->getIdentifier() . '"', 1321288316);
                }
+               $this->indexingInProgress = FALSE;
+       }
+
+       /**
+        * Loads MetaData from Repository
+        */
+       protected function loadMetaData() {
+               $this->metaDataProperties = $this->getMetaDataRepository()->findByFile($this);
        }
 
        /**
@@ -375,6 +400,20 @@ class File extends AbstractFile {
        }
 
        /**
+        * @return \TYPO3\CMS\Core\Resource\Index\MetaDataRepository
+        */
+       protected function getMetaDataRepository() {
+               return GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\Resource\\Index\\MetaDataRepository');
+       }
+
+       /**
+        * @return \TYPO3\CMS\Core\Resource\Index\FileIndexRepository
+        */
+       protected function getFileIndexRepository() {
+               return GeneralUtility::makeInstance('TYPO3\\CMS\\Core\\Resource\\Index\\FileIndexRepository');
+       }
+
+       /**
         * Internal function to retrieve the indexer service,
         * if it does not exist, an instance will be created
         *
index 0e0a252..9f58d75 100644 (file)
@@ -26,6 +26,10 @@ namespace TYPO3\CMS\Core\Resource\Hook;
  *
  *  This copyright notice MUST APPEAR in all copies of the script!
  ***************************************************************/
+
+use \TYPO3\CMS\Backend\Utility\BackendUtility;
+use \TYPO3\CMS\Core\Resource\ResourceFactory;
+
 /**
  * Utility class to render TCEforms information about a sys_file record
  *
@@ -36,32 +40,66 @@ class FileInfoHook {
        /**
         * User function for sys_file (element)
         *
-        * @param array $PA the array with additional configuration options.
+        * @param array $propertyArray the array with additional configuration options.
         * @param \TYPO3\CMS\Backend\Form\FormEngine $tceformsObj the TCEforms parent object
         * @return string The HTML code for the TCEform field
         */
-       public function renderFileInfo(array $PA, \TYPO3\CMS\Backend\Form\FormEngine $tceformsObj) {
-               $fileRecord = $PA['row'];
+       public function renderFileInfo(array $propertyArray, \TYPO3\CMS\Backend\Form\FormEngine $tceformsObj) {
+               $fileRecord = $propertyArray['row'];
+               $fileObject = NULL;
                if ($fileRecord['uid'] > 0) {
-                       $fileObject = \TYPO3\CMS\Core\Resource\ResourceFactory::getInstance()->getFileObject($fileRecord['uid']);
-                       $processedFile = $fileObject->process(\TYPO3\CMS\Core\Resource\ProcessedFile::CONTEXT_IMAGEPREVIEW, array('width' => 150, 'height' => 150));
+                       $fileObject = ResourceFactory::getInstance()->getFileObject(intval($fileRecord['uid']));
+
+               }
+               return $this->renderFileInformationContent($fileObject);
+       }
+
+       /**
+        * User function for sys_file_meta (element)
+        *
+        * @param array $propertyArray the array with additional configuration options.
+        * @param \TYPO3\CMS\Backend\Form\FormEngine $tceformsObj the TCEforms parent object
+        * @return string The HTML code for the TCEform field
+        */
+       public function renderFileMetadataInfo(array $propertyArray, \TYPO3\CMS\Backend\Form\FormEngine $tceformsObj) {
+               $fileMedadataRecord = $propertyArray['row'];
+               $fileObject = NULL;
+               if ($fileMedadataRecord['file'] > 0) {
+                       $fileObject = ResourceFactory::getInstance()->getFileObject(intval($fileMedadataRecord['file']));
+               }
+
+               return $this->renderFileInformationContent($fileObject);
+       }
+
+
+       /**
+        * Renders a HTML Block with file information
+        *
+        * @param \TYPO3\CMS\Core\Resource\File $file
+        * @return string
+        */
+       protected function renderFileInformationContent(\TYPO3\CMS\Core\Resource\File $file = NULL) {
+               if ($file !== NULL) {
+                       $processedFile = $file->process(\TYPO3\CMS\Core\Resource\ProcessedFile::CONTEXT_IMAGEPREVIEW, array('width' => 150, 'height' => 150));
                        $previewImage = $processedFile->getPublicUrl(TRUE);
                        $content = '';
-                       if ($fileObject->isMissing()) {
-                               $flashMessage = \TYPO3\CMS\Core\Resource\Utility\BackendUtility::getFlashMessageForMissingFile($fileObject);
+                       if ($file->isMissing()) {
+                               $flashMessage = \TYPO3\CMS\Core\Resource\Utility\BackendUtility::getFlashMessageForMissingFile($file);
                                $content .= $flashMessage->render();
                        }
                        if ($previewImage) {
                                $content .= '<img src="' . htmlspecialchars($previewImage) . '" alt="" class="t3-tceforms-sysfile-imagepreview" />';
                        }
-                       $content .= '<strong>' . htmlspecialchars($fileObject->getName()) . '</strong> (' . htmlspecialchars(\TYPO3\CMS\Core\Utility\GeneralUtility::formatSize($fileObject->getSize())) . ')<br />';
-                       $content .= \TYPO3\CMS\Backend\Utility\BackendUtility::getProcessedValue($PA['table'], 'type', $fileObject->getType()) . ' (' . $fileObject->getMimeType() . ')<br />';
-                       $content .= $GLOBALS['LANG']->sL('LLL:EXT:lang/locallang_misc.xlf:fileMetaDataLocation', TRUE) . ': ' . htmlspecialchars($fileObject->getStorage()->getName()) . ' - ' . htmlspecialchars($fileObject->getIdentifier()) . '<br />';
+                       $content .= '<strong>' . htmlspecialchars($file->getName()) . '</strong>';
+                       $content .= '(' . htmlspecialchars(\TYPO3\CMS\Core\Utility\GeneralUtility::formatSize($file->getSize())) . 'bytes)<br />';
+                       $content .= BackendUtility::getProcessedValue('sys_file', 'type', $file->getType()) . ' (' . $file->getMimeType() . ')<br />';
+                       $content .= $GLOBALS['LANG']->sL('LLL:EXT:lang/locallang_misc.xlf:fileMetaDataLocation', TRUE) . ': ';
+                       $content .= htmlspecialchars($file->getStorage()->getName()) . ' - ' . htmlspecialchars($file->getIdentifier()) . '<br />';
                        $content .= '<br />';
                } else {
                        $content = '<h2>' . $GLOBALS['LANG']->sL('LLL:EXT:lang/locallang_misc.xlf:fileMetaErrorInvalidRecord', TRUE) . '</h2>';
                }
+
                return $content;
        }
-
 }
index 0245956..b820541 100644 (file)
@@ -53,9 +53,8 @@ class FileIndexRepository implements SingletonInterface {
         * @var array
         */
        protected $fields = array(
-               'uid', 'pid',   'missing', 'type', 'storage', 'identifier',
-               'extension', 'mime_type', 'name', 'title', 'sha1', 'size', 'creation_date',
-               'modification_date', 'width', 'height', 'description', 'alternative'
+               'uid', 'pid', 'missing', 'type', 'storage', 'identifier', 'extension',
+               'mime_type', 'name', 'sha1', 'size', 'creation_date', 'modification_date',
        );
 
        /**
@@ -95,7 +94,7 @@ class FileIndexRepository implements SingletonInterface {
         */
        public function findOneByUid($fileUid) {
                $row = $this->getDatabase()->exec_SELECTgetSingleRow(
-                       '*',
+                       implode(',', $this->fields),
                        $this->table,
                        'uid=' . intval($fileUid)
                );
@@ -113,7 +112,7 @@ class FileIndexRepository implements SingletonInterface {
         */
        public function findOneByStorageUidAndIdentifier($storageUid, $identifier) {
                $row = $this->getDatabase()->exec_SELECTgetSingleRow(
-                       '*',
+                       implode(',', $this->fields),
                        $this->table,
                        sprintf('storage=%u AND identifier=%s', intval($storageUid), $this->getDatabase()->fullQuoteStr($identifier, $this->table))
                );
@@ -146,7 +145,7 @@ class FileIndexRepository implements SingletonInterface {
                        return array();
                }
                $resultRows = $this->getDatabase()->exec_SELECTgetRows(
-                       '*',
+                       implode(',', $this->fields),
                        $this->table,
                        'sha1=' . $this->getDatabase()->fullQuoteStr($hash, $this->table)
                );
@@ -204,4 +203,4 @@ class FileIndexRepository implements SingletonInterface {
                        $this->getDatabase()->exec_UPDATEquery($this->table, 'uid=' . intval($file->getUid()), $updateRow);
                }
        }
-}
+}
\ No newline at end of file
diff --git a/typo3/sysext/core/Classes/Resource/Index/MetaDataRepository.php b/typo3/sysext/core/Classes/Resource/Index/MetaDataRepository.php
new file mode 100644 (file)
index 0000000..2604430
--- /dev/null
@@ -0,0 +1,116 @@
+<?php
+
+namespace TYPO3\CMS\Core\Resource\Index;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2013 Steffen Ritter <steffen.ritter@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *  A copy is found in the textfile GPL.txt and important notices to the license
+ *  from the author is found in LICENSE.txt distributed with these scripts.
+ *
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+use TYPO3\CMS\Core\Resource\File;
+use TYPO3\CMS\Core\SingletonInterface;
+
+
+/**
+ * Repository Class as an abstraction layer to sys_file_metadata
+ *
+ * Every access to table sys_file_metadata which is not handled by TCEmain
+ * has to use this Repository class
+ */
+class MetaDataRepository implements SingletonInterface {
+
+       /**
+        * @var string
+        */
+       protected $tableName = 'sys_file_metadata';
+
+       /**
+        * Wrapper method for getting DatabaseConnection
+        *
+        * @return \TYPO3\CMS\Core\Database\DatabaseConnection
+        */
+       protected function getDatabase() {
+               return $GLOBALS['TYPO3_DB'];
+       }
+
+       /**
+        * Returns array of meta-data properties
+        *
+        * @param File $file
+        * @return array
+        */
+       public function findByFile(File $file) {
+               return $this->findByFileUid($file->getUid());
+       }
+
+       /**
+        * Retrieves metadata for file
+        *
+        * @param int $uid
+        * @return array
+        * @throws \RuntimeException
+        */
+       public function findByFileUid($uid) {
+               $uid = intval($uid);
+               if ($uid <= 0) {
+                       throw new \RuntimeException('Metadata can only be retrieved for indexed files.', 1381590731);
+               }
+               $record = $this->getDatabase()->exec_SELECTgetSingleRow('*', $this->tableName, 'file = ' . $uid);
+
+               if ($record === FALSE) {
+                       $record = $this->createMetaDataRecord($uid);
+               }
+
+               return $record;
+       }
+
+       /**
+        * General Where-Clause which is needed to fetch only language 0 and live record.
+        *
+        * @return string
+        */
+       protected function getGeneralWhereClause() {
+               return ' AND 1=1';
+       }
+       /**
+        * Create empty
+        *
+        * @param int $fileUid
+        * @return array
+        */
+       protected function createMetaDataRecord($fileUid) {
+               $emptyRecord =  array(
+                       'file' => intval($fileUid),
+                       'pid' => 0,
+                       'crdate' => $GLOBALS['EXEC_TIME'],
+                       'tstamp' => $GLOBALS['EXEC_TIME'],
+                       'cruser_id' => TYPO3_MODE == 'BE' ? $GLOBALS['BE_USER']->user['uid'] : 0
+               );
+               $this->getDatabase()->exec_INSERTquery($this->tableName, $emptyRecord);
+               $record = $emptyRecord;
+               $record['uid'] = $this->getDatabase()->sql_insert_id();
+
+               return $record;
+       }
+}
\ No newline at end of file
index 4a62938..87ca77d 100644 (file)
@@ -188,7 +188,7 @@ return array(
                                ),
                        ),
                ),
-               'defaultCategorizedTables' => 'pages,tt_content,sys_file', // List of comma separated tables that are categorizable by default.
+               'defaultCategorizedTables' => 'pages,tt_content,sys_file_metadata', // List of comma separated tables that are categorizable by default.
                'displayErrors' => -1,          // <p>Integer (-1, 0, 1, 2). Configures whether PHP errors should be displayed.</p><dl><dt>0</dt><dd>Do not display any PHP error messages. Overrides the value of "exceptionalErrors" and sets it to 0 (= no errors are turned into exceptions), the configured "productionExceptionHandler" is used as exception handler</dd><dt>1</dt><dd>Display error messages with the registered errorhandler. The configured "debugExceptionHandler" is used as exception handler</dd><dt>2</dt><dd>Display errors only if client matches <a href="#SYS-devIPmask">[SYS][devIPmask]</a>. If devIPmask matches the users IP address  the configured "debugExceptionHandler" is used  for exceptions, if not "productionExceptionHandler" will be used</dd><dt>-1</dt><dd>Default setting. With this option, you can override the PHP setting "display_errors". If devIPmask matches the users IP address  the configured "debugExceptionHandler" is used  for exceptions, if not "productionExceptionHandler" will be used.</dd></dl>
                'productionExceptionHandler' => 'TYPO3\\CMS\\Core\\Error\\ProductionExceptionHandler',          // String: Classname to handle exceptions that might happen in the TYPO3-code. Leave empty to disable exception handling. Default: "TYPO3\\CMS\\Core\\Error\\ProductionExceptionHandler". This exception handler displays a nice error message when something went wrong. The error message is logged to the configured logs. Note: The configured "productionExceptionHandler" is used if displayErrors is set to "0" or to "-1" and devIPmask doesn't match the users IP.
                'debugExceptionHandler' => 'TYPO3\\CMS\\Core\\Error\\DebugExceptionHandler',            // String: Classname to handle exceptions that might happen in the TYPO3-code. Leave empty to disable exception handling. Default: "TYPO3\\CMS\\Core\\Error\\DebugExceptionHandler". This exception handler displays the complete stack trace of any encountered exception. The error message and the stack trace  is logged to the configured logs. Note: The configured "debugExceptionHandler" is used if displayErrors is set to "1" and if displayErrors is "-1"  or "2" and the devIPmask matches the users IP.
index 786ea24..8e4a0d7 100644 (file)
@@ -9,9 +9,7 @@ return array(
                'type' => 'type',
                'hideTable' => TRUE,
                'rootLevel' => TRUE,
-               'versioningWS' => TRUE,
-               'origUid' => 't3_origuid',
-               'default_sortby' => 'ORDER BY crdate DESC',
+               'default_sortby' => 'ORDER BY name ASC',
                'dividers2tabs' => TRUE,
                'typeicon_column' => 'type',
                'typeicon_classes' => array(
@@ -28,17 +26,9 @@ return array(
                ),
        ),
        'interface' => array(
-               'showRecordFieldList' => 'storage, name, description, alternative, type, mime_type, size, sha1, missing'
+               'showRecordFieldList' => 'storage, name, type, mime_type, size, sha1, missing'
        ),
        'columns' => array(
-               't3ver_label' => array(
-                       'label' => 'LLL:EXT:lang/locallang_general.xlf:LGL.versionLabel',
-                       'config' => array(
-                               'type' => 'input',
-                               'size' => '30',
-                               'max' => '30'
-                       )
-               ),
                'fileinfo' => array(
                        'config' => array(
                                'type' => 'user',
@@ -74,37 +64,10 @@ return array(
                        'exclude' => 0,
                        'label' => 'LLL:EXT:lang/locallang_tca.xlf:sys_file.name',
                        'config' => array(
+                               'readOnly' => 1,
                                'type' => 'input',
                                'size' => '30',
                                'eval' => 'required',
-                               'readOnly' => TRUE
-                       )
-               ),
-               'title' => array(
-                       'exclude' => 1,
-                       'label' => 'LLL:EXT:lang/locallang_tca.xlf:sys_file.title',
-                       'config' => array(
-                               'type' => 'input',
-                               'size' => '30',
-                               'placeholder' => '__row|name'
-                       )
-               ),
-               'description' => array(
-                       'exclude' => 0,
-                       'label' => 'LLL:EXT:lang/locallang_tca.xlf:sys_file.description',
-                       'config' => array(
-                               'type' => 'text',
-                               'cols' => '40',
-                               'rows' => '3'
-                       )
-               ),
-               'alternative' => array(
-                       'exclude' => 0,
-                       'label' => 'LLL:EXT:lang/locallang_tca.xlf:sys_file.alternative',
-                       'config' => array(
-                               'type' => 'text',
-                               'cols' => '40',
-                               'rows' => '3'
                        )
                ),
                'type' => array(
@@ -160,12 +123,24 @@ return array(
                        'config' => array(
                                'readOnly' => 1,
                                'type' => 'check',
-                               'default' => '0'
+                               'default' => 0
+                       )
+               ),
+               'metadata' => array(
+                       'exclude' => 0,
+                       'label' => 'LLL:EXT:lang/locallang_tca.xlf:sys_file.metadata',
+                       'config' => array(
+                               'readOnly' => 1,
+                               'type' => 'select',
+                               'foreign_table' => 'sys_file_metadata',
+                               'foreign_field' => 'file',
+                               'size' => 1,
+                               'minitems' => 1
                        )
                )
        ),
        'types' => array(
-               '1' => array('showitem' => 'fileinfo, name, title, description, alternative, storage')
+               '1' => array('showitem' => 'fileinfo, storage, missing')
        ),
        'palettes' => array()
 );
diff --git a/typo3/sysext/core/Configuration/TCA/sys_file_metadata.php b/typo3/sysext/core/Configuration/TCA/sys_file_metadata.php
new file mode 100644 (file)
index 0000000..edd66a2
--- /dev/null
@@ -0,0 +1,92 @@
+<?php
+return array(
+       'ctrl' => array(
+               'title' => 'LLL:EXT:lang/locallang_tca.xlf:sys_file',
+               'label' => 'file',
+               'tstamp' => 'tstamp',
+               'crdate' => 'crdate',
+               'cruser_id' => 'cruser_id',
+               'type' => 'type',
+               'hideTable' => TRUE,
+               'rootLevel' => TRUE,
+               'versioningWS' => TRUE,
+               'origUid' => 't3_origuid',
+               'default_sortby' => 'ORDER BY crdate DESC',
+               'dividers2tabs' => TRUE,
+               'typeicon_column' => '__row|file|type',
+               'typeicon_classes' => array(
+                       '1' => 'mimetypes-text-text',
+                       '2' => 'mimetypes-media-image',
+                       '3' => 'mimetypes-media-audio',
+                       '4' => 'mimetypes-media-video',
+                       '5' => 'mimetypes-application',
+                       'default' => 'mimetypes-other-other'
+               ),
+               'security' => array(
+                       'ignoreWebMountRestriction' => TRUE,
+                       'ignoreRootLevelRestriction' => TRUE,
+               ),
+       ),
+       'interface' => array(
+               'showRecordFieldList' => 'file, title, description, alternative'
+       ),
+       'columns' => array(
+               't3ver_label' => array(
+                       'label' => 'LLL:EXT:lang/locallang_general.xlf:LGL.versionLabel',
+                       'config' => array(
+                               'type' => 'input',
+                               'size' => '30',
+                               'max' => '30'
+                       )
+               ),
+               'fileinfo' => array(
+                       'config' => array(
+                               'type' => 'user',
+                               'userFunc' => 'typo3/sysext/core/Classes/Resource/Hook/FileInfoHook.php:TYPO3\CMS\Core\Resource\Hook\FileInfoHook->renderFileMetadataInfo'
+                       )
+               ),
+               'file' => array(
+                       'exclude' => 0,
+                       'label' => 'LLL:EXT:lang/locallang_tca.xlf:sys_file',
+                       'config' => array(
+                               'readOnly' => 1,
+                               'type' => 'select',
+                               'foreign_table' => 'sys_file',
+                               'minitems' => 1,
+                               'maxitems' => 1,
+                               'size' => 1,
+                       )
+               ),
+               'title' => array(
+                       'exclude' => 1,
+                       'label' => 'LLL:EXT:lang/locallang_tca.xlf:sys_file.title',
+                       'config' => array(
+                               'type' => 'input',
+                               'size' => '30',
+                               'placeholder' => '__row|file|name'
+                       )
+               ),
+               'description' => array(
+                       'exclude' => 0,
+                       'label' => 'LLL:EXT:lang/locallang_tca.xlf:sys_file.description',
+                       'config' => array(
+                               'type' => 'text',
+                               'cols' => '40',
+                               'rows' => '3'
+                       )
+               ),
+               'alternative' => array(
+                       'exclude' => 0,
+                       'label' => 'LLL:EXT:lang/locallang_tca.xlf:sys_file.alternative',
+                       'config' => array(
+                               'type' => 'text',
+                               'cols' => '40',
+                               'rows' => '3'
+                       )
+               ),
+       ),
+       'types' => array(
+               '1' => array('showitem' => 'fileinfo, title, description, alternative')
+       ),
+       'palettes' => array()
+);
index a8a09fc..98298ac 100644 (file)
@@ -161,7 +161,7 @@ return array(
                                'type' => 'input',
                                'size' => '20',
                                'eval' => 'null',
-                               'placeholder' => '__row|uid_local|title',
+                               'placeholder' => '__row|uid_local|metadata|title',
                                'mode' => 'useOrOverridePlaceholder',
                        )
                ),
@@ -194,7 +194,7 @@ return array(
                                'cols' => '20',
                                'rows' => '5',
                                'eval' => 'null',
-                               'placeholder' => '__row|uid_local|description',
+                               'placeholder' => '__row|uid_local|metadata|description',
                                'mode' => 'useOrOverridePlaceholder',
                        )
                ),
@@ -206,7 +206,7 @@ return array(
                                'type' => 'input',
                                'size' => '20',
                                'eval' => 'null',
-                               'placeholder' => '__row|uid_local|alternative',
+                               'placeholder' => '__row|uid_local|metadata|alternative',
                                'mode' => 'useOrOverridePlaceholder',
                        ),
                ),
index e3a8eeb..495e43c 100644 (file)
@@ -107,8 +107,10 @@ class FileTest extends \TYPO3\CMS\Core\Tests\UnitTestCase {
         */
        public function fileAsksRepositoryForIndexStatus() {
                $mockedRepository = $this->getMock('TYPO3\\CMS\\Core\\Resource\\Index\\FileIndexRepository');
-               $mockedRepository->expects($this->once())->method('findOneByFileObject')->will($this->returnValue(array()));
+               $mockedRepository->expects($this->once())->method('findOneByCombinedIdentifier')->will($this->returnValue(array('uid' => 1)));
+               $mockedRepository->expects($this->exactly(0))->method('add');
                \TYPO3\CMS\Core\Utility\GeneralUtility::setSingletonInstance('TYPO3\\CMS\\Core\\Resource\\Index\\FileIndexRepository', $mockedRepository);
+
                $fixture = new \TYPO3\CMS\Core\Resource\File(array(), $this->storageMock);
                $this->assertTrue($fixture->isIndexed());
        }
@@ -200,7 +202,7 @@ class FileTest extends \TYPO3\CMS\Core\Tests\UnitTestCase {
         */
        public function fetchingIndexedPropertyCausesFileObjectToLoadIndexRecord() {
                $mockedRepository = $this->getMock('TYPO3\\CMS\\Core\\Resource\\Index\\FileIndexRepository');
-               $mockedRepository->expects($this->once())->method('findOneByFileObject')->will($this->returnValue(array('uid' => 10)));
+               $mockedRepository->expects($this->once())->method('findOneByCombinedIdentifier')->will($this->returnValue(array('uid' => 10)));
                \TYPO3\CMS\Core\Utility\GeneralUtility::setSingletonInstance('TYPO3\\CMS\\Core\\Resource\\Index\\FileIndexRepository', $mockedRepository);
 
                $fixture = new \TYPO3\CMS\Core\Resource\File(array('identifier' => '/test', 'storage' => 5), $this->storageMock);
@@ -211,15 +213,12 @@ class FileTest extends \TYPO3\CMS\Core\Tests\UnitTestCase {
         * @test
         */
        public function isIndexedTriggersIndexingIfFileIsNotIndexedAlready() {
-               $fixture = new \TYPO3\CMS\Core\Resource\File(array('identifier' => '/test', 'storage' => 5), $this->storageMock);
+               $fixture = $this->getMock('TYPO3\\CMS\\Core\\Resource\\File', array('loadMetadata'), array(array('identifier' => '/test', 'storage' => 5), $this->storageMock));
+
                $mockedRepository = $this->getMock('TYPO3\\CMS\\Core\\Resource\\Index\\FileIndexRepository');
-               $mockedRepository->expects($this->once())->method('findOneByFileObject')->will($this->returnValue(FALSE));
+               $mockedRepository->expects($this->once())->method('findOneByCombinedIdentifier')->will($this->returnValue(FALSE));
                \TYPO3\CMS\Core\Utility\GeneralUtility::setSingletonInstance('TYPO3\\CMS\\Core\\Resource\\Index\\FileIndexRepository', $mockedRepository);
 
-               $indexerMock = $this->getMock('TYPO3\\CMS\\Core\\Resource\\Service\\IndexerService', array(), array(), '', FALSE);
-               $indexerMock->expects($this->once())->method('indexFile')->will($this->returnValue(array()));
-               \TYPO3\CMS\Core\Utility\GeneralUtility::setSingletonInstance('TYPO3\\CMS\\Core\\Resource\\Service\\IndexerService', $indexerMock);
-
 
                $fixture->isIndexed();
        }
@@ -228,17 +227,14 @@ class FileTest extends \TYPO3\CMS\Core\Tests\UnitTestCase {
         * @test
         */
        public function fileIsAutomaticallyIndexedOnPropertyAccessIfNotAlreadyIndexed() {
-               $fixture = new \TYPO3\CMS\Core\Resource\File(array('identifier' => '/test', 'storage' => 5), $this->storageMock);
+               $fixture = $this->getMock('TYPO3\\CMS\\Core\\Resource\\File', array('loadMetadata'), array(array('identifier' => '/test', 'storage' => 5), $this->storageMock));
+
                $mockedRepository = $this->getMock('TYPO3\\CMS\\Core\\Resource\\Index\\FileIndexRepository');
-               $mockedRepository->expects($this->once())->method('findOneByFileObject')->will($this->returnValue(FALSE));
+               $mockedRepository->expects($this->once())->method('findOneByCombinedIdentifier')->will($this->returnValue(FALSE));
+               $mockedRepository->expects($this->once())->method('add');
                \TYPO3\CMS\Core\Utility\GeneralUtility::setSingletonInstance('TYPO3\\CMS\\Core\\Resource\\Index\\FileIndexRepository', $mockedRepository);
 
-               $indexerMock = $this->getMock('TYPO3\\CMS\\Core\\Resource\\Service\\IndexerService', array(), array(), '', FALSE);
-               $indexerMock->expects($this->once())->method('indexFile')->will($this->returnValue(array('uid' => 10)));
-               \TYPO3\CMS\Core\Utility\GeneralUtility::setSingletonInstance('TYPO3\\CMS\\Core\\Resource\\Service\\IndexerService', $indexerMock);
-
-
-               $this->assertEquals(10, $fixture->getProperty('uid'));
+               $fixture->getProperty('uid');
        }
 
        /**
index 7191b54..1128035 100644 (file)
@@ -282,9 +282,37 @@ CREATE TABLE sys_file (
        pid int(11) DEFAULT '0' NOT NULL,
        # update timestamp of the database record, not the file!
        tstamp int(11) DEFAULT '0' NOT NULL,
+
+       # management information
+       missing tinyint(4) DEFAULT '0' NOT NULL,
+       storage int(11) DEFAULT '0' NOT NULL,
+       type varchar(10) DEFAULT '' NOT NULL,
+       metadata int(11) DEFAULT '0' NOT NULL,
+
+       # file info data
+       identifier varchar(200) DEFAULT '' NOT NULL,
+       extension varchar(255) DEFAULT '' NOT NULL,
+       mime_type varchar(255) DEFAULT '' NOT NULL,
+       name tinytext,
+       sha1 tinytext,
+       size int(11) DEFAULT '0' NOT NULL,
+       creation_date int(11) DEFAULT '0' NOT NULL,
+       modification_date int(11) DEFAULT '0' NOT NULL,
+
+       PRIMARY KEY (uid),
+       KEY sel01 (storage,identifier(20)),
+       KEY sha1 (sha1(40))
+);
+
+#
+# Table structure for table 'sys_file_metadata'
+#
+CREATE TABLE sys_file_metadata (
+       uid int(11) NOT NULL auto_increment,
+       pid int(11) DEFAULT '0' NOT NULL,
+       tstamp int(11) DEFAULT '0' NOT NULL,
        crdate int(11) DEFAULT '0' NOT NULL,
        cruser_id int(11) DEFAULT '0' NOT NULL,
-       missing tinyint(4) DEFAULT '0' NOT NULL,
 
        # Versioning fields
        t3ver_oid int(11) DEFAULT '0' NOT NULL,
@@ -298,30 +326,19 @@ CREATE TABLE sys_file (
        t3ver_move_id int(11) DEFAULT '0' NOT NULL,
        t3_origuid int(11) DEFAULT '0' NOT NULL,
 
-       type varchar(10) DEFAULT '' NOT NULL,
-       storage int(11) DEFAULT '0' NOT NULL,
-       identifier varchar(512) DEFAULT '' NOT NULL,
-       extension varchar(255) DEFAULT '' NOT NULL,
-       mime_type varchar(255) DEFAULT '' NOT NULL,
-       name tinytext,
+       file int(11) DEFAULT '0' NOT NULL,
        title tinytext,
-       sha1 tinytext,
-       size int(11) DEFAULT '0' NOT NULL,
-       # creation/modification date of the file (not the record!)
-       creation_date int(11) DEFAULT '0' NOT NULL,
-       modification_date int(11) DEFAULT '0' NOT NULL,
        width int(11) DEFAULT '0' NOT NULL,
        height int(11) DEFAULT '0' NOT NULL,
        description text,
        alternative text,
 
        PRIMARY KEY (uid),
-       KEY parent (pid),
-       KEY t3ver_oid (t3ver_oid,t3ver_wsid),
-       KEY sel01 (storage,identifier(20)),
-       KEY sha1 (sha1(40))
+       KEY file (file),
+       KEY t3ver_oid (t3ver_oid,t3ver_wsid)
 );
 
+
 #
 # Table structure for table 'sys_file_processedfile'.
 # which is a "temporary" file, like an image preview
index 0732ec0..1cfc2dd 100644 (file)
@@ -726,8 +726,9 @@ class FileList extends \TYPO3\CMS\Backend\RecordList\AbstractRecordList {
                // Edit metadata of file
                try {
                        if (is_a($fileOrFolderObject, 'TYPO3\\CMS\\Core\\Resource\\File') && $fileOrFolderObject->isIndexed() && $fileOrFolderObject->checkActionPermission('write')) {
+                               $metaData = $fileOrFolderObject->_getMetaData();
                                $data = array(
-                                       'sys_file' => array($fileOrFolderObject->getUid() => 'edit')
+                                       'sys_file_metadata' => array($metaData['uid'] => 'edit')
                                );
                                $editOnClick = BackendUtility::editOnClick(GeneralUtility::implodeArrayForUrl('edit', $data), $GLOBALS['BACK_PATH'], $this->listUrl());
                                $cells['editmetadata'] = '<a href="#" onclick="' . $editOnClick . '" title="Edit Metadata of this file">' . IconUtility::getSpriteIcon('actions-document-open') . '</a>';
index 2bcfe10..0a4f6be 100644 (file)
@@ -407,8 +407,8 @@ class SilentConfigurationUpgradeService {
 
        /**
         * Make sure file table is categorized as of TYPO3 6.2. To enable DAM Migration
-        * sys_file table is included in DefaultConfiguration.
-        * If the setting already has been modified but does not contain sys_file: add it
+        * sys_file_metadata table is included in DefaultConfiguration.
+        * If the setting already has been modified but does not contain sys_file_metadata: add it
         *
         * @return void
         */
@@ -424,8 +424,8 @@ class SilentConfigurationUpgradeService {
                }
 
                $tables =  GeneralUtility::trimExplode(',', $actual);
-               if ($actual !== '' && $actual !== $default && !in_array('sys_file', $tables)) {
-                       $tables[] = 'sys_file';
+               if ($actual !== '' && $actual !== $default && !in_array('sys_file_metadata', $tables)) {
+                       $tables[] = 'sys_file_metadata';
                        $configurationManager->setLocalConfigurationValueByPath('SYS/defaultCategorizedTables', implode(',', $tables));
                        $this->throwRedirectException();
                }
diff --git a/typo3/sysext/install/Classes/Updates/FileTableSplittingUpdate.php b/typo3/sysext/install/Classes/Updates/FileTableSplittingUpdate.php
new file mode 100644 (file)
index 0000000..0be243b
--- /dev/null
@@ -0,0 +1,124 @@
+<?php
+namespace TYPO3\CMS\Install\Updates;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2010-2013 Steffen Ritter <steffen.ritter@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *  A copy is found in the textfile GPL.txt and important notices to the license
+ *  from the author is found in LICENSE.txt distributed with these scripts.
+ *
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+/**
+ * Migrate metadata from sys_file table to sys_filemetadata.
+ * Also takes care of custom TCA fields if they have been created beforehand.
+ *
+ * @author Steffen Ritter <steffen.ritter@typo3.org>
+ */
+class FileTableSplittingUpdate extends AbstractUpdate {
+
+       /**
+        * The table the metadata is to be stored in
+        * @var string
+        */
+       protected $metaDataTable = 'sys_file_metadata';
+
+       /**
+        * @var string
+        */
+       protected $title = 'Migrate file metadata from sys_file to an external metadata table';
+
+       /**
+        * Checks if an update is needed
+        *
+        * @param string &$description The description for the update
+        * @return boolean Whether an update is needed (TRUE) or not (FALSE)
+        */
+       public function checkForUpdate(&$description) {
+               $result = FALSE;
+
+               $description = 'In TYPO3 CMS 6.2 LTS the metadata has been split off to an external table. This wizard will migrate the data. If you have extended the sys_file table manually your custom data will be included, too, if you create TCA and columns in sys_file_metadata before running this wizard.';
+
+               if (!array_key_exists($this->metaDataTable, $GLOBALS['TYPO3_DB']->admin_get_tables())) {
+                       $result = TRUE;
+               } else {
+                       $sysFileCount = $GLOBALS['TYPO3_DB']->exec_SELECTcountRows('uid', 'sys_file');
+                       $sysFileMetaDataCount = $GLOBALS['TYPO3_DB']->exec_SELECTcountRows('uid', $this->metaDataTable);
+                       $result = $sysFileCount > $sysFileMetaDataCount;
+               }
+
+               return $result;
+       }
+
+       /**
+        * Performs the database update. Won't run if the table is not present.
+        * Will stop if the table does not exist to give users the possibility to
+        * migrate custom fields to and therefore move their TCA and sql upfront.
+        *
+        * @param array &$dbQueries Queries done in this update
+        * @param mixed &$customMessages Custom messages
+        * @return boolean Whether it worked (TRUE) or not (FALSE)
+        */
+       public function performUpdate(array &$dbQueries, &$customMessages) {
+
+               if (!array_key_exists($this->metaDataTable, $GLOBALS['TYPO3_DB']->admin_get_tables())) {
+                       $customMessages = 'ERROR! Make sure you created the table before. If you added custom metadata to sys_file table add TCA configuration as well as sql definitions to sys_file_metadata, too.';
+                       return FALSE;
+               }
+
+               $filesToMigrateRows = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows(
+                       'uid',
+                       'sys_file',
+                       'uid NOT IN (' . $GLOBALS['TYPO3_DB']->SELECTquery('file', $this->metaDataTable, '') . ')'
+               );
+               $filesToMigrateUids = array();
+               foreach ($filesToMigrateRows as $row) {
+                       $filesToMigrateUids[] = intval($row['uid']);
+               }
+               $filesToMigrateUids = array_unique($filesToMigrateUids);
+               $dataToMove = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows(
+                       implode(',', $this->detectFieldsToMigrate()) . ', uid AS file',
+                       'sys_file',
+                       'uid IN (' . implode(',', $filesToMigrateUids) . ')'
+               );
+
+               $resultObject = $GLOBALS['TYPO3_DB']->exec_INSERTmultipleRows($this->metaDataTable, array_keys(current($dataToMove)), $dataToMove);
+               return $resultObject !== FALSE;
+       }
+
+       /**
+        * Looks at the table sql definitions and checks which fields are present in both tables.
+        * ignories some Management field
+        *
+        * @return array
+        */
+       protected function detectFieldsToMigrate() {
+               $fieldsBlackListed = array('uid', 'deleted', 'sys_language_uid');
+               $fieldsInSysFile = array_keys($GLOBALS['TYPO3_DB']->admin_get_fields('sys_file'));
+               $fieldsInSysFileMetaData = array_keys($GLOBALS['TYPO3_DB']->admin_get_fields($this->metaDataTable));
+
+               $commonFields = array_intersect($fieldsInSysFileMetaData, $fieldsInSysFile);
+               $commonFields = array_diff($commonFields, $fieldsBlackListed);
+
+               return $commonFields;
+
+       }
+}
index 8008d6b..8c8b961 100644 (file)
@@ -38,6 +38,7 @@ $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['ext/install']['update']['imagelink']
 $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['ext/install']['update']['sysext_file_init'] = 'TYPO3\\CMS\\Install\\Updates\\InitUpdateWizard';
 $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['ext/install']['update']['sysext_file_images'] = 'TYPO3\\CMS\\Install\\Updates\\TceformsUpdateWizard';
 $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['ext/install']['update']['sysext_file_uploads'] = 'TYPO3\\CMS\\Install\\Updates\\TtContentUploadsUpdateWizard';
+$GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['ext/install']['update']['sysext_file_splitMetaData'] = 'TYPO3\\CMS\\Install\\Updates\\FileTableSplittingUpdate';
 $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['ext/install']['update']['referenceIntegrity'] = 'TYPO3\\CMS\\Install\\Updates\\ReferenceIntegrityUpdateWizard';
 
 $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['ext/install']['update']['sysext_file_filemounts'] = 'TYPO3\\CMS\\Install\\Updates\\FilemountUpdateWizard';
index 8df1740..0be1338 100644 (file)
                        <trans-unit id="sys_file.missing" xml:space="preserve">
                                <source>Marked as missing</source>
                        </trans-unit>
+                       <trans-unit id="sys_file.metadata" xml:space="preserve">
+                               <source>Metadata records</source>
+                       </trans-unit>
                        <trans-unit id="sys_file_reference" xml:space="preserve">
                                <source>File Reference</source>
                        </trans-unit>