Commit 99de17ab authored by Frans Saris's avatar Frans Saris Committed by Georg Ringer
Browse files

[FEATURE] Trigger metadata extraction after file upload

Releases: master
Resolves: #56726
Change-Id: I8f08403aca72bc9ca3f37dec6f98bf016c79a9ee
Reviewed-on: http://review.typo3.org/43059


Reviewed-by: Wouter Wolters's avatarWouter Wolters <typo3@wouterwolters.nl>
Tested-by: Wouter Wolters's avatarWouter Wolters <typo3@wouterwolters.nl>
Reviewed-by: Georg Ringer's avatarGeorg Ringer <georg.ringer@gmail.com>
Tested-by: Georg Ringer's avatarGeorg Ringer <georg.ringer@gmail.com>
parent 3aaa86f1
...@@ -39,6 +39,11 @@ class Indexer { ...@@ -39,6 +39,11 @@ class Indexer {
*/ */
protected $storage = NULL; protected $storage = NULL;
/**
* @var ExtractorInterface[]
*/
protected $extractionServices = NULL;
/** /**
* @param ResourceStorage $storage * @param ResourceStorage $storage
*/ */
...@@ -95,28 +100,46 @@ class Indexer { ...@@ -95,28 +100,46 @@ class Indexer {
*/ */
public function runMetaDataExtraction($maximumFileCount = -1) { public function runMetaDataExtraction($maximumFileCount = -1) {
$fileIndexRecords = $this->getFileIndexRepository()->findInStorageWithIndexOutstanding($this->storage, $maximumFileCount); $fileIndexRecords = $this->getFileIndexRepository()->findInStorageWithIndexOutstanding($this->storage, $maximumFileCount);
$extractionServices = $this->getExtractorRegistry()->getExtractorsWithDriverSupport($this->storage->getDriverType());
foreach ($fileIndexRecords as $indexRecord) { foreach ($fileIndexRecords as $indexRecord) {
$fileObject = $this->getResourceFactory()->getFileObject($indexRecord['uid'], $indexRecord); $fileObject = $this->getResourceFactory()->getFileObject($indexRecord['uid'], $indexRecord);
$this->extractMetaData($fileObject);
}
}
$newMetaData = array( /**
0 => $fileObject->_getMetaData() * Extract metadata for given fileObject
); *
foreach ($extractionServices as $service) { * @param File $fileObject
if ($service->canProcess($fileObject)) { */
$newMetaData[$service->getPriority()] = $service->extractMetaData($fileObject, $newMetaData); public function extractMetaData(File $fileObject) {
} $newMetaData = array(
} 0 => $fileObject->_getMetaData()
ksort($newMetaData); );
$metaData = array(); foreach ($this->getExtractionServices() as $service) {
foreach ($newMetaData as $data) { if ($service->canProcess($fileObject)) {
$metaData = array_merge($metaData, $data); $newMetaData[$service->getPriority()] = $service->extractMetaData($fileObject, $newMetaData);
} }
$fileObject->_updateMetaDataProperties($metaData);
$this->getMetaDataRepository()->update($fileObject->getUid(), $metaData);
$this->getFileIndexRepository()->updateIndexingTime($fileObject->getUid());
} }
ksort($newMetaData);
$metaData = array();
foreach ($newMetaData as $data) {
$metaData = array_merge($metaData, $data);
}
$fileObject->_updateMetaDataProperties($metaData);
$this->getMetaDataRepository()->update($fileObject->getUid(), $metaData);
$this->getFileIndexRepository()->updateIndexingTime($fileObject->getUid());
}
/**
* Get available extraction services
*
* @return ExtractorInterface[]
*/
protected function getExtractionServices() {
if ($this->extractionServices === NULL) {
$this->extractionServices = $this->getExtractorRegistry()->getExtractorsWithDriverSupport($this->storage->getDriverType());
}
return $this->extractionServices;
} }
/** /**
......
...@@ -16,6 +16,7 @@ namespace TYPO3\CMS\Core\Resource; ...@@ -16,6 +16,7 @@ namespace TYPO3\CMS\Core\Resource;
use TYPO3\CMS\Core\Resource\Exception\InvalidTargetFolderException; use TYPO3\CMS\Core\Resource\Exception\InvalidTargetFolderException;
use TYPO3\CMS\Core\Resource\Index\FileIndexRepository; use TYPO3\CMS\Core\Resource\Index\FileIndexRepository;
use TYPO3\CMS\Core\Resource\Index\Indexer;
use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Core\Utility\PathUtility; use TYPO3\CMS\Core\Utility\PathUtility;
...@@ -349,6 +350,15 @@ class ResourceStorage implements ResourceStorageInterface { ...@@ -349,6 +350,15 @@ class ResourceStorage implements ResourceStorageInterface {
return $this->isOnline; return $this->isOnline;
} }
/**
* Returns TRUE if auto extracting of metadata is enabled
*
* @return bool
*/
public function autoExtractMetadataEnabled() {
return !empty($this->storageRecord['auto_extract_metadata']);
}
/** /**
* Blows the "fuse" and marks the storage as offline. * Blows the "fuse" and marks the storage as offline.
* *
...@@ -1116,6 +1126,11 @@ class ResourceStorage implements ResourceStorageInterface { ...@@ -1116,6 +1126,11 @@ class ResourceStorage implements ResourceStorageInterface {
$fileIdentifier = $this->driver->addFile($localFilePath, $targetFolder->getIdentifier(), $targetFileName); $fileIdentifier = $this->driver->addFile($localFilePath, $targetFolder->getIdentifier(), $targetFileName);
$file = ResourceFactory::getInstance()->getFileObjectByStorageAndIdentifier($this->getUid(), $fileIdentifier); $file = ResourceFactory::getInstance()->getFileObjectByStorageAndIdentifier($this->getUid(), $fileIdentifier);
if ($this->autoExtractMetadataEnabled()) {
$indexer = GeneralUtility::makeInstance(Indexer::class, $this);
$indexer->extractMetaData($file);
}
$this->emitPostFileAddSignal($file, $targetFolder); $this->emitPostFileAddSignal($file, $targetFolder);
return $file; return $file;
...@@ -1199,6 +1214,7 @@ class ResourceStorage implements ResourceStorageInterface { ...@@ -1199,6 +1214,7 @@ class ResourceStorage implements ResourceStorageInterface {
if ($this->isOnline()) { if ($this->isOnline()) {
// Pre-process the public URL by an accordant slot // Pre-process the public URL by an accordant slot
$this->emitPreGeneratePublicUrlSignal($resourceObject, $relativeToCurrentScript, array('publicUrl' => &$publicUrl)); $this->emitPreGeneratePublicUrlSignal($resourceObject, $relativeToCurrentScript, array('publicUrl' => &$publicUrl));
// If slot did not handle the signal, use the default way to determine public URL // If slot did not handle the signal, use the default way to determine public URL
if ($publicUrl === NULL) { if ($publicUrl === NULL) {
...@@ -1792,7 +1808,12 @@ class ResourceStorage implements ResourceStorageInterface { ...@@ -1792,7 +1808,12 @@ class ResourceStorage implements ResourceStorageInterface {
if ($file instanceof File) { if ($file instanceof File) {
$this->getIndexer()->updateIndexEntry($file); $this->getIndexer()->updateIndexEntry($file);
} }
if ($this->autoExtractMetadataEnabled()) {
$indexer = GeneralUtility::makeInstance(Indexer::class, $this);
$indexer->extractMetaData($file);
}
$this->emitPostFileReplaceSignal($file, $localFilePath); $this->emitPostFileReplaceSignal($file, $localFilePath);
return $file; return $file;
} }
......
...@@ -16,7 +16,7 @@ return array( ...@@ -16,7 +16,7 @@ return array(
'searchFields' => 'name,description' 'searchFields' => 'name,description'
), ),
'interface' => array( 'interface' => array(
'showRecordFieldList' => 'name,description,driver,processingfolder,configuration' 'showRecordFieldList' => 'name,description,driver,processingfolder,configuration,auto_extract_metadata'
), ),
'columns' => array( 'columns' => array(
'name' => array( 'name' => array(
...@@ -82,6 +82,14 @@ return array( ...@@ -82,6 +82,14 @@ return array(
'default' => 1 'default' => 1
) )
), ),
'auto_extract_metadata' => array(
'exclude' => 0,
'label' => 'LLL:EXT:lang/locallang_tca.xlf:sys_file_storage.auto_extract_metadata',
'config' => array(
'type' => 'check',
'default' => 1
)
),
'processingfolder' => array( 'processingfolder' => array(
'exclude' => 0, 'exclude' => 0,
'label' => 'LLL:EXT:lang/locallang_tca.xlf:sys_file_storage.processingfolder', 'label' => 'LLL:EXT:lang/locallang_tca.xlf:sys_file_storage.processingfolder',
...@@ -112,7 +120,7 @@ return array( ...@@ -112,7 +120,7 @@ return array(
) )
), ),
'types' => array( 'types' => array(
'0' => array('showitem' => 'name, description, --div--;Configuration, driver, configuration, is_default, processingfolder, --div--;Access, --palette--;Capabilities;capabilities, is_online') '0' => array('showitem' => 'name, description, --div--;Configuration, driver, configuration, is_default, auto_extract_metadata, processingfolder, --div--;Access, --palette--;Capabilities;capabilities, is_online')
), ),
'palettes' => array( 'palettes' => array(
'capabilities' => array( 'capabilities' => array(
......
===============================================================
Feature: #56726 - Trigger metadata extraction after file upload
===============================================================
Description
===========
Before #56726 the metadata extraction was only called through the extract metadata scheduler task.
So when a editor uploaded a new file he had to wait until the scheduler task is triggered again and extracted the metadata.
Now the metadata extraction is by default triggered after adding/uploading a file in the BE. Or when the FAL API is used ``ResourceStorage::addFile()``, ``ResourceStorage::replaceFile()`` and ``ResourceStorage::addUploadedFile()``.
In some special situations it isn't desired to have metadata extraction direct after file upload/adding a file to the storage.
For these cases the automatic extraction can be disabled in File Storage configuration.
Impact
======
The flag is by default set for all existing and new storages. When you have some special use-case where automatic extraction of metadata is not desired the flag needs to be disabled in File Storage configuration.
\ No newline at end of file
...@@ -259,6 +259,7 @@ CREATE TABLE sys_file_storage ( ...@@ -259,6 +259,7 @@ CREATE TABLE sys_file_storage (
is_public tinyint(4) DEFAULT '0' NOT NULL, is_public tinyint(4) DEFAULT '0' NOT NULL,
is_writable tinyint(4) DEFAULT '0' NOT NULL, is_writable tinyint(4) DEFAULT '0' NOT NULL,
is_online tinyint(4) DEFAULT '1' NOT NULL, is_online tinyint(4) DEFAULT '1' NOT NULL,
auto_extract_metadata tinyint(4) DEFAULT '1' NOT NULL,
processingfolder tinytext, processingfolder tinytext,
PRIMARY KEY (uid), PRIMARY KEY (uid),
......
...@@ -429,6 +429,9 @@ ...@@ -429,6 +429,9 @@
<trans-unit id="sys_file_storage.is_writable"> <trans-unit id="sys_file_storage.is_writable">
<source>Is writable?</source> <source>Is writable?</source>
</trans-unit> </trans-unit>
<trans-unit id="sys_file_storage.auto_extract_metadata">
<source>Automatically extract metadata after upload</source>
</trans-unit>
<trans-unit id="sys_file_storage.capabilities"> <trans-unit id="sys_file_storage.capabilities">
<source>Capabilities</source> <source>Capabilities</source>
</trans-unit> </trans-unit>
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment