Commit d66d7f68 authored by Christian Kuhn's avatar Christian Kuhn Committed by Benni Mack
Browse files

[TASK] Cleanup ReferenceIndex class

* Since introduction of class ReferenceIndexUpdater each
  record is handled only once when updating the reference
  index. The record runtime cache is obsolete and can be
  dropped.

* The $excludedTables and $excludedColumns properties are
  non-static now: In normal operation only one instance
  of ReferenceIndex class is created and used for many
  records, there is no point in creating hard to evict
  static state.

* A runtime cache has been used for information derived
  from TCA (no db calls). This is dropped in favor of a
  class property.

Change-Id: I5cf16e38ec8f36dfa838cdbc6591b59b463be3f9
Resolves: #93038
Releases: master
Reviewed-on: https://review.typo3.org/c/Packages/TYPO3.CMS/+/67076


Tested-by: Oliver Bartsch's avatarOliver Bartsch <bo@cedev.de>
Tested-by: default avatarTYPO3com <noreply@typo3.com>
Tested-by: Benni Mack's avatarBenni Mack <benni@typo3.org>
Reviewed-by: Oliver Bartsch's avatarOliver Bartsch <bo@cedev.de>
Reviewed-by: Benni Mack's avatarBenni Mack <benni@typo3.org>
parent 960273fc
......@@ -61,7 +61,6 @@ class ReferenceIndexUpdateCommand extends Command
$progressListener = GeneralUtility::makeInstance(ReferenceIndexProgressListener::class);
$progressListener->initialize($io);
$referenceIndex = GeneralUtility::makeInstance(ReferenceIndex::class);
$referenceIndex->enableRuntimeCache();
if ($isTestOnly) {
$io->section('Reference Index being TESTED (nothing written, remove the "--check" argument)');
} else {
......
......@@ -196,7 +196,6 @@ class ReferenceIndexUpdater
// Perform reference index updates
$referenceIndex = GeneralUtility::makeInstance(ReferenceIndex::class);
$referenceIndex->enableRuntimeCache();
foreach ($this->updateRegistry as $workspace => $tableArray) {
$referenceIndex->setWorkspaceId($workspace);
foreach ($tableArray as $table => $uidArray) {
......
......@@ -22,7 +22,6 @@ use Psr\Log\LoggerAwareTrait;
use Psr\Log\LogLevel;
use TYPO3\CMS\Backend\Utility\BackendUtility;
use TYPO3\CMS\Backend\View\ProgressListenerInterface;
use TYPO3\CMS\Core\Cache\CacheManager;
use TYPO3\CMS\Core\Configuration\FlexForm\FlexFormTools;
use TYPO3\CMS\Core\Database\Platform\PlatformInformation;
use TYPO3\CMS\Core\DataHandling\DataHandler;
......@@ -60,7 +59,7 @@ class ReferenceIndex implements LoggerAwareInterface
* @see updateRefIndexTable()
* @see shouldExcludeTableFromReferenceIndex()
*/
protected static $excludedTables = [
protected array $excludedTables = [
'sys_log' => true,
'tx_extensionmanager_domain_model_extension' => true
];
......@@ -75,7 +74,7 @@ class ReferenceIndex implements LoggerAwareInterface
* @see fetchTableRelationFields()
* @see shouldExcludeTableColumnFromReferenceIndex()
*/
protected static $excludedColumns = [
protected array $excludedColumns = [
'uid' => true,
'perms_userid' => true,
'perms_groupid' => true,
......@@ -85,14 +84,6 @@ class ReferenceIndex implements LoggerAwareInterface
'pid' => true
];
/**
* Fields of tables that could contain relations are cached per table. This is the prefix for the cache entries since
* the runtimeCache has a global scope.
*
* @var string
*/
protected static $cachePrefixTableRelationFields = 'core-refidx-tblRelFields-';
/**
* This array holds the FlexForm references of a record
*
......@@ -110,14 +101,6 @@ class ReferenceIndex implements LoggerAwareInterface
*/
protected $relations = [];
/**
* A cache to avoid that identical rows are refetched from the database
*
* @var array
* @see getRecordRawCached()
*/
protected $recordCache = [];
/**
* Number which we can increase if a change in the code means we will have to force a re-generation of the index.
*
......@@ -128,36 +111,21 @@ class ReferenceIndex implements LoggerAwareInterface
/**
* Current workspace id
*
* @var int
*/
protected $workspaceId = 0;
/**
* Runtime Cache to store and retrieve data computed for a single request
*
* @var \TYPO3\CMS\Core\Cache\Frontend\FrontendInterface
*/
protected $runtimeCache;
protected int $workspaceId = 0;
/**
* Enables $runtimeCache and $recordCache
* @var bool
* A list of fields that may contain relations per TCA table.
* This is either ['*'] or an array of single field names. The list
* depends on TCA and is built when a first table row is handled.
*/
protected $useRuntimeCache = false;
protected array $tableRelationFieldCache = [];
/**
* @var EventDispatcherInterface
*/
protected $eventDispatcher;
protected EventDispatcherInterface $eventDispatcher;
/**
* @param EventDispatcherInterface $eventDispatcher
*/
public function __construct(EventDispatcherInterface $eventDispatcher = null)
{
$this->eventDispatcher = $eventDispatcher ?? GeneralUtility::getContainer()->get(EventDispatcherInterface::class);
$this->runtimeCache = GeneralUtility::makeInstance(CacheManager::class)->getCache('runtime');
}
/**
......@@ -212,15 +180,7 @@ class ReferenceIndex implements LoggerAwareInterface
return $result;
}
// Fetch tableRelationFields and save them in cache if not there yet
$cacheId = static::$cachePrefixTableRelationFields . $tableName;
$tableRelationFields = $this->useRuntimeCache ? $this->runtimeCache->get($cacheId) : false;
if ($tableRelationFields === false) {
$tableRelationFields = $this->fetchTableRelationFields($tableName);
if ($this->useRuntimeCache) {
$this->runtimeCache->set($cacheId, $tableRelationFields);
}
}
$tableRelationFields = $this->fetchTableRelationFields($tableName);
$connectionPool = GeneralUtility::makeInstance(ConnectionPool::class);
$connection = $connectionPool->getConnectionForTable('sys_refindex');
......@@ -243,8 +203,8 @@ class ReferenceIndex implements LoggerAwareInterface
}
// If the table has fields which could contain relations and the record does exist
if ($tableRelationFields !== '') {
$existingRecord = $this->getRecordRawCached($tableName, $uid);
if ($tableRelationFields !== []) {
$existingRecord = $this->getRecord($tableName, $uid);
if ($existingRecord) {
// Table has relation fields and record exists - get relations
$this->relations = [];
......@@ -866,7 +826,7 @@ class ReferenceIndex implements LoggerAwareInterface
* @param array $configuration Config array for TCA/columns field
* @return bool TRUE if DB reference field (group/db or select with foreign-table)
*/
protected function isDbReferenceField(array $configuration)
protected function isDbReferenceField(array $configuration): bool
{
return
($configuration['type'] === 'group' && $configuration['internal_type'] === 'db')
......@@ -883,7 +843,7 @@ class ReferenceIndex implements LoggerAwareInterface
* @param array $configuration Config array for TCA/columns field
* @return bool TRUE if reference field
*/
protected function isReferenceField(array $configuration)
protected function isReferenceField(array $configuration): bool
{
return
$this->isDbReferenceField($configuration)
......@@ -900,23 +860,25 @@ class ReferenceIndex implements LoggerAwareInterface
* Returns all fields of a table which could contain a relation
*
* @param string $tableName Name of the table
* @return string Fields which could contain a relation
* @return array Fields which may contain relations
*/
protected function fetchTableRelationFields($tableName)
protected function fetchTableRelationFields(string $tableName): array
{
if (!empty($this->tableRelationFieldCache[$tableName])) {
return $this->tableRelationFieldCache[$tableName];
}
if (!isset($GLOBALS['TCA'][$tableName]['columns'])) {
return '';
return [];
}
$fields = [];
foreach ($GLOBALS['TCA'][$tableName]['columns'] as $field => $fieldDefinition) {
if (is_array($fieldDefinition['config'])) {
// Check for flex field
if (isset($fieldDefinition['config']['type']) && $fieldDefinition['config']['type'] === 'flex') {
// Fetch all fields if the is a field of type flex in the table definition because the complete row is passed to
// FlexFormTools->getDataStructureIdentifier() in the end and might be needed in ds_pointerField or a hook
return '*';
$this->tableRelationFieldCache[$tableName] = ['*'];
return ['*'];
}
// Only fetch this field if it can contain a reference
if ($this->isReferenceField($fieldDefinition['config'])) {
......@@ -924,8 +886,8 @@ class ReferenceIndex implements LoggerAwareInterface
}
}
}
return implode(',', $fields);
$this->tableRelationFieldCache[$tableName] = $fields;
return $fields;
}
/**
......@@ -1119,83 +1081,52 @@ class ReferenceIndex implements LoggerAwareInterface
}
/**
* Gets one record from database and stores it in an internal cache (which expires along with object lifecycle) for faster retrieval
*
* Assumption:
*
* - This method is only used from within delegate methods and so only caches queries generated based on the record being indexed; the query
* to select origin side record is uncached
* - Origin side records do not change in database while updating the reference index
* - Origin record does not get removed while updating index
* - Relations may change during indexing, which is why only the origin record is cached and all relations are re-process even when repeating
* indexing of the same origin record
*
* Please note that the cache is disabled by default but can be enabled using $this->enableRuntimeCaches()
* due to possible side-effects while handling references that were changed during one single
* request.
* Get one record from database.
*
* @param string $tableName
* @param int $uid
* @return array|false
*/
protected function getRecordRawCached(string $tableName, int $uid)
protected function getRecord(string $tableName, int $uid)
{
$recordCacheId = $tableName . ':' . $uid;
if (!$this->useRuntimeCache || !isset($this->recordCache[$recordCacheId])) {
// Fetch fields of the table which might contain relations
$cacheId = static::$cachePrefixTableRelationFields . $tableName;
$tableRelationFields = $this->useRuntimeCache ? $this->runtimeCache->get($cacheId) : false;
if ($tableRelationFields === false) {
$tableRelationFields = $this->fetchTableRelationFields($tableName);
if ($this->useRuntimeCache) {
$this->runtimeCache->set($cacheId, $tableRelationFields);
}
}
// Fetch fields of the table which might contain relations
$tableRelationFields = $this->fetchTableRelationFields($tableName);
if ($tableRelationFields === []) {
// Return if there are no fields which could contain relations
if ($tableRelationFields === '') {
return $this->relations;
}
if ($tableRelationFields === '*') {
// If one field of a record is of type flex, all fields have to be fetched to be passed to FlexFormTools->getDataStructureIdentifier()
$selectFields = '*';
} else {
// otherwise only fields that might contain relations are fetched
$selectFields = 'uid,' . $tableRelationFields;
if (BackendUtility::isTableWorkspaceEnabled($tableName)) {
$selectFields .= ',t3ver_wsid,t3ver_state';
}
}
$queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
->getQueryBuilderForTable($tableName);
$queryBuilder->getRestrictions()->removeAll();
$queryBuilder
->select(...GeneralUtility::trimExplode(',', $selectFields, true))
->from($tableName)
->where(
$queryBuilder->expr()->eq(
'uid',
$queryBuilder->createNamedParameter($uid, \PDO::PARAM_INT)
)
);
// Do not fetch soft deleted records
$deleteField = $GLOBALS['TCA'][$tableName]['ctrl']['delete'] ?? false;
if ($deleteField) {
$queryBuilder->andWhere(
$queryBuilder->expr()->eq(
$deleteField,
$queryBuilder->createNamedParameter(0, \PDO::PARAM_INT)
)
);
return $this->relations;
}
if ($tableRelationFields !== ['*']) {
// Only fields that might contain relations are fetched
$tableRelationFields[] = 'uid';
if (BackendUtility::isTableWorkspaceEnabled($tableName)) {
$tableRelationFields = array_merge($tableRelationFields, ['t3ver_wsid', 't3ver_state']);
}
$row = $queryBuilder->execute()->fetch();
}
$this->recordCache[$recordCacheId] = $row;
$queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
->getQueryBuilderForTable($tableName);
$queryBuilder->getRestrictions()->removeAll();
$queryBuilder
->select(...array_unique($tableRelationFields))
->from($tableName)
->where(
$queryBuilder->expr()->eq(
'uid',
$queryBuilder->createNamedParameter($uid, \PDO::PARAM_INT)
)
);
// Do not fetch soft deleted records
$deleteField = (string)($GLOBALS['TCA'][$tableName]['ctrl']['delete'] ?? '');
if ($deleteField !== '') {
$queryBuilder->andWhere(
$queryBuilder->expr()->eq(
$deleteField,
$queryBuilder->createNamedParameter(0, \PDO::PARAM_INT)
)
);
}
return $this->recordCache[$recordCacheId];
return $queryBuilder->execute()->fetch();
}
/**
......@@ -1204,19 +1135,17 @@ class ReferenceIndex implements LoggerAwareInterface
* @param string $tableName Name of the table
* @return bool true if it should be excluded
*/
protected function shouldExcludeTableFromReferenceIndex($tableName)
protected function shouldExcludeTableFromReferenceIndex(string $tableName): bool
{
if (isset(static::$excludedTables[$tableName])) {
return static::$excludedTables[$tableName];
if (isset($this->excludedTables[$tableName])) {
return $this->excludedTables[$tableName];
}
// Only exclude tables from ReferenceIndex which do not contain any relations and never
// did since existing references won't be deleted!
$event = new IsTableExcludedFromReferenceIndexEvent($tableName);
$event = $this->eventDispatcher->dispatch($event);
static::$excludedTables[$tableName] = $event->isTableExcluded();
return static::$excludedTables[$tableName];
$this->excludedTables[$tableName] = $event->isTableExcluded();
return $this->excludedTables[$tableName];
}
/**
......@@ -1227,16 +1156,17 @@ class ReferenceIndex implements LoggerAwareInterface
* @param string $onlyColumn Name of a specific column to fetch
* @return bool true if it should be excluded
*/
protected function shouldExcludeTableColumnFromReferenceIndex($tableName, $column, $onlyColumn)
{
if (isset(static::$excludedColumns[$column])) {
protected function shouldExcludeTableColumnFromReferenceIndex(
string $tableName,
string $column,
string $onlyColumn
): bool {
if (isset($this->excludedColumns[$column])) {
return true;
}
if (is_array($GLOBALS['TCA'][$tableName]['columns'][$column]) && (!$onlyColumn || $onlyColumn === $column)) {
return false;
}
return true;
}
......@@ -1244,18 +1174,22 @@ class ReferenceIndex implements LoggerAwareInterface
* Enables the runtime-based caches
* Could lead to side effects, depending if the reference index instance is run multiple times
* while records would be changed.
*
* @deprecated since v11, will be removed in v12.
*/
public function enableRuntimeCache()
{
$this->useRuntimeCache = true;
trigger_error('Calling ReferenceIndex->enableRuntimeCache() is obsolete and should be dropped.', E_USER_DEPRECATED);
}
/**
* Disables the runtime-based cache
*
* @deprecated since v11, will be removed in v12.
*/
public function disableRuntimeCache()
{
$this->useRuntimeCache = false;
trigger_error('Calling ReferenceIndex->disableRuntimeCache() is obsolete and should be dropped.', E_USER_DEPRECATED);
}
/**
......
......@@ -1314,7 +1314,6 @@ class RelationHandler
if (BackendUtility::isTableWorkspaceEnabled($table)) {
$referenceIndex->setWorkspaceId($this->getWorkspaceId());
}
$referenceIndex->enableRuntimeCache();
$statisticsArray = $referenceIndex->updateRefIndexTable($table, $uid);
}
return $statisticsArray;
......
......@@ -542,7 +542,6 @@ class FileIndexRepository implements SingletonInterface
{
/** @var ReferenceIndex $refIndexObj */
$refIndexObj = GeneralUtility::makeInstance(ReferenceIndex::class);
$refIndexObj->enableRuntimeCache();
$refIndexObj->updateRefIndexTable($this->table, $id);
}
}
.. include:: ../../Includes.txt
==================================================
Deprecation: #93038 - ReferenceIndex runtime cache
==================================================
See :issue:`93038`
Description
===========
Two methods of class :php:`ReferenceIndex` have been deprecated:
* :php:`ReferenceIndex->enableRuntimeCache()`
* :php:`ReferenceIndex->disableRuntimeCache()`
Impact
======
Calling these methods raises a deprecation level log entry.
Affected Installations
======================
Instances with extensions calling above methods are affected. The extension
scanner locates candidates.
Migration
=========
The method calls can be dropped, cache handling is done internally.
.. index:: PHP-API, FullyScanned, ext:core
......@@ -150,7 +150,6 @@ class Backend implements BackendInterface, SingletonInterface
$this->eventDispatcher = $eventDispatcher;
$this->referenceIndex = GeneralUtility::makeInstance(ReferenceIndex::class);
$this->referenceIndex->enableRuntimeCache();
$this->aggregateRootObjects = new ObjectStorage();
$this->deletedEntities = new ObjectStorage();
$this->changedEntities = new ObjectStorage();
......
......@@ -304,7 +304,6 @@ class Export extends ImportExport
$this->dat['header']['pid_lookup'][$row['pid']][$table][$row['uid']] = 1;
// Initialize reference index object:
$refIndexObj = GeneralUtility::makeInstance(ReferenceIndex::class);
$refIndexObj->enableRuntimeCache();
$relations = $refIndexObj->getRelations($table, $row);
$relations = $this->fixFileIDsInRelations($relations);
$relations = $this->removeSoftrefsHavingTheSameDatabaseRelation($relations);
......
......@@ -68,7 +68,6 @@ class ReferenceIndexUpdatedPrerequisite implements PrerequisiteInterface, Chatty
$this->output->writeln('Reference Index is being updated');
$progressListener = GeneralUtility::makeInstance(ReferenceIndexProgressListener::class);
$progressListener->initialize(new SymfonyStyle(new ArrayInput([]), $this->output));
$this->referenceIndex->enableRuntimeCache();
$result = $this->referenceIndex->updateIndex(false, $progressListener);
return empty($result['errors']);
}
......@@ -80,7 +79,6 @@ class ReferenceIndexUpdatedPrerequisite implements PrerequisiteInterface, Chatty
*/
public function isFulfilled(): bool
{
$this->referenceIndex->enableRuntimeCache();
$result = $this->referenceIndex->updateIndex(true);
return empty($result['errors']);
}
......
......@@ -4585,4 +4585,18 @@ return [
'Breaking-93029-DroppedDeletedFieldFromSys_refindex.rst',
],
],
'TYPO3\CMS\Core\Database\ReferenceIndex->enableRuntimeCache' => [
'numberOfMandatoryArguments' => 0,
'maximumNumberOfArguments' => 0,
'restFiles' => [
'Deprecation-93038-ReferenceIndexRuntimeCache.rst',
],
],
'TYPO3\CMS\Core\Database\ReferenceIndex->disableRuntimeCache' => [
'numberOfMandatoryArguments' => 0,
'maximumNumberOfArguments' => 0,
'restFiles' => [
'Deprecation-93038-ReferenceIndexRuntimeCache.rst',
],
],
];
......@@ -309,7 +309,6 @@ class DatabaseIntegrityController
if (GeneralUtility::_GP('_update') || GeneralUtility::_GP('_check')) {
$testOnly = (bool)GeneralUtility::_GP('_check');
$refIndexObj = GeneralUtility::makeInstance(ReferenceIndex::class);
$refIndexObj->enableRuntimeCache();
$result = $refIndexObj->updateIndex($testOnly);
$recordsCheckedString = $result['resultText'];
$errors = $result['errors'];
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment