DataHandler.php

<?php

/*
 * This file is part of the TYPO3 CMS project.
 *
 * It is free software; you can redistribute it and/or modify it under
 * the terms of the GNU General Public License, either version 2
 * of the License, or any later version.
 *
 * For the full copyright and license information, please read the
 * LICENSE.txt file that was distributed with this source code.
 *
 * The TYPO3 project - inspiring people to share!
 */

namespace TYPO3\CMS\Core\DataHandling;

use Doctrine\DBAL\Exception as DBALException;
use Doctrine\DBAL\Platforms\PostgreSQL94Platform as PostgreSqlPlatform;
use Doctrine\DBAL\Platforms\SQLServer2012Platform as SQLServerPlatform;
use Doctrine\DBAL\Types\IntegerType;
use Psr\Log\LoggerAwareInterface;
use Psr\Log\LoggerAwareTrait;
use TYPO3\CMS\Backend\Utility\BackendUtility;
use TYPO3\CMS\Core\Authentication\BackendUserAuthentication;
use TYPO3\CMS\Core\Cache\CacheManager;
use TYPO3\CMS\Core\Cache\Frontend\FrontendInterface;
use TYPO3\CMS\Core\Configuration\Features;
use TYPO3\CMS\Core\Configuration\FlexForm\Exception\InvalidIdentifierException;
use TYPO3\CMS\Core\Configuration\FlexForm\Exception\InvalidParentRowException;
use TYPO3\CMS\Core\Configuration\FlexForm\Exception\InvalidParentRowLoopException;
use TYPO3\CMS\Core\Configuration\FlexForm\Exception\InvalidParentRowRootException;
use TYPO3\CMS\Core\Configuration\FlexForm\Exception\InvalidPointerFieldValueException;
use TYPO3\CMS\Core\Configuration\FlexForm\FlexFormTools;
use TYPO3\CMS\Core\Configuration\Richtext;
use TYPO3\CMS\Core\Crypto\PasswordHashing\InvalidPasswordHashException;
use TYPO3\CMS\Core\Crypto\PasswordHashing\PasswordHashFactory;
use TYPO3\CMS\Core\Database\Connection;
use TYPO3\CMS\Core\Database\ConnectionPool;
use TYPO3\CMS\Core\Database\Query\QueryBuilder;
use TYPO3\CMS\Core\Database\Query\QueryHelper;
use TYPO3\CMS\Core\Database\Query\Restriction\DeletedRestriction;
use TYPO3\CMS\Core\Database\Query\Restriction\QueryRestrictionContainerInterface;
use TYPO3\CMS\Core\Database\Query\Restriction\WorkspaceRestriction;
use TYPO3\CMS\Core\Database\RelationHandler;
use TYPO3\CMS\Core\DataHandling\History\RecordHistoryStore;
use TYPO3\CMS\Core\DataHandling\Localization\DataMapProcessor;
use TYPO3\CMS\Core\DataHandling\Model\CorrelationId;
use TYPO3\CMS\Core\DataHandling\Model\RecordStateFactory;
use TYPO3\CMS\Core\Exception\SiteNotFoundException;
use TYPO3\CMS\Core\Html\RteHtmlParser;
use TYPO3\CMS\Core\Localization\LanguageService;
use TYPO3\CMS\Core\Messaging\FlashMessage;
use TYPO3\CMS\Core\Messaging\FlashMessageService;
use TYPO3\CMS\Core\Resource\ResourceFactory;
use TYPO3\CMS\Core\Service\OpcodeCacheService;
use TYPO3\CMS\Core\Site\Entity\SiteLanguage;
use TYPO3\CMS\Core\Site\SiteFinder;
use TYPO3\CMS\Core\SysLog\Action\Cache as SystemLogCacheAction;
use TYPO3\CMS\Core\SysLog\Action\Database as SystemLogDatabaseAction;
use TYPO3\CMS\Core\SysLog\Error as SystemLogErrorClassification;
use TYPO3\CMS\Core\SysLog\Type as SystemLogType;
use TYPO3\CMS\Core\Type\Bitmask\Permission;
use TYPO3\CMS\Core\Utility\ArrayUtility;
use TYPO3\CMS\Core\Utility\ExtensionManagementUtility;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Core\Utility\MathUtility;
use TYPO3\CMS\Core\Utility\StringUtility;
use TYPO3\CMS\Core\Versioning\VersionState;

/**
 * The main data handler class which takes care of correctly updating and inserting records.
 * This class was formerly known as TCEmain.
 *
 * This is the TYPO3 Core Engine class for manipulation of the database
 * This class is used by eg. the tce_db BE route (SimpleDataHandlerController) which provides an interface for POST forms to this class.
 *
 * Dependencies:
 * - $GLOBALS['TCA'] must exist
 * - $GLOBALS['LANG'] must exist
 *
 * Also see document 'TYPO3 Core API' for details.
 */
class DataHandler implements LoggerAwareInterface
{
    use LoggerAwareTrait;

    // *********************
    // Public variables you can configure before using the class:
    // *********************
    /**
     * If TRUE, the default log-messages will be stored. This should not be necessary if the locallang-file for the
     * log-display is properly configured. So disabling this will just save some database-space as the default messages are not saved.
     *
     * @var bool
     */
    public $storeLogMessages = true;

    /**
     * If TRUE, actions are logged to sys_log.
     *
     * @var bool
     */
    public $enableLogging = true;

    /**
     * If TRUE, the datamap array is reversed in the order, which is a nice thing if you're creating a whole new
     * bunch of records.
     *
     * @var bool
     */
    public $reverseOrder = false;

    /**
     * If TRUE, only fields which are different from the database values are saved! In fact, if a whole input array
     * is similar, it's not saved then.
     *
     * @var bool
     * @internal should only be used from within TYPO3 Core
     */
    public $checkSimilar = true;

    /**
     * This will read the record after having updated or inserted it. If anything is not properly submitted an error
     * is written to the log. This feature consumes extra time by selecting records
     *
     * @var bool
     */
    public $checkStoredRecords = true;

    /**
     * If set, values '' and 0 will equal each other when the stored records are checked.
     *
     * @var bool
     */
    public $checkStoredRecords_loose = true;

    /**
     * If set, then the 'hideAtCopy' flag for tables will be ignored.
     *
     * @var bool
     */
    public $neverHideAtCopy = false;

    /**
     * If set, then the TCE class has been instantiated during an import action of a T3D
     *
     * @var bool
     */
    public $isImporting = false;

    /**
     * If set, then transformations are NOT performed on the input.
     *
     * @var bool
     */
    public $dontProcessTransformations = false;

    /**
     * Will distinguish between translations (with parent) and localizations (without parent) while still using the same methods to copy the records
     * TRUE: translation of a record connected to the default language
     * FALSE: localization of a record without connection to the default language
     *
     * @var bool
     */
    protected $useTransOrigPointerField = true;

    /**
     * If TRUE, workspace restrictions are bypassed on edit and create actions (process_datamap()).
     * YOU MUST KNOW what you do if you use this feature!
     *
     * @var bool
     * @internal should only be used from within TYPO3 Core
     */
    public $bypassWorkspaceRestrictions = false;

    /**
     * If TRUE, access check, check for deleted etc. for records is bypassed.
     * YOU MUST KNOW what you are doing if you use this feature!
     *
     * @var bool
     */
    public $bypassAccessCheckForRecords = false;

    /**
     * Comma-separated list. This list of tables decides which tables will be copied. If empty then none will.
     * If '*' then all will (that the user has permission to of course)
     *
     * @var string
     * @internal should only be used from within TYPO3 Core
     */
    public $copyWhichTables = '*';

    /**
     * If 0 then branch is NOT copied.
     * If 1 then pages on the 1st level is copied.
     * If 2 then pages on the second level is copied ... and so on
     *
     * @var int
     */
    public $copyTree = 0;

    /**
     * [table][fields]=value: New records are created with default values and you can set this array on the
     * form $defaultValues[$table][$field] = $value to override the default values fetched from TCA.
     * If ->setDefaultsFromUserTS is called UserTSconfig default values will overrule existing values in this array
     * (thus UserTSconfig overrules externally set defaults which overrules TCA defaults)
     *
     * @var array
     * @internal should only be used from within TYPO3 Core
     */
    public $defaultValues = [];

    /**
     * [table][fields]=value: You can set this array on the form $overrideValues[$table][$field] = $value to
     * override the incoming data. You must set this externally. You must make sure the fields in this array are also
     * found in the table, because it's not checked. All columns can be set by this array!
     *
     * @var array
     * @internal should only be used from within TYPO3 Core
     */
    public $overrideValues = [];

    /**
     * If entries are set in this array corresponding to fields for update, they are ignored and thus NOT updated.
     * You could set this array from a series of checkboxes with value=0 and hidden fields before the checkbox with 1.
     * Then an empty checkbox will disable the field.
     *
     * @var array
     * @internal should only be used from within TYPO3 Core
     */
    public $data_disableFields = [];

    /**
     * Use this array to validate suggested uids for tables by setting [table]:[uid]. This is a dangerous option
     * since it will force the inserted record to have a certain UID. The value just have to be TRUE, but if you set
     * it to "DELETE" it will make sure any record with that UID will be deleted first (raw delete).
     * The option is used for import of T3D files when synchronizing between two mirrored servers.
     * As a security measure this feature is available only for Admin Users (for now)
     *
     * @var array
     */
    public $suggestedInsertUids = [];

    /**
     * Object. Call back object for FlexForm traversal. Useful when external classes wants to use the
     * iteration functions inside DataHandler for traversing a FlexForm structure.
     *
     * @var object
     * @internal should only be used from within TYPO3 Core
     */
    public $callBackObj;

    /**
     * A string which can be used as correlationId for RecordHistory entries.
     * The string can later be used to rollback multiple changes at once.
     *
     * @var CorrelationId|null
     */
    protected $correlationId;

    // *********************
    // Internal variables (mapping arrays) which can be used (read-only) from outside
    // *********************
    /**
     * Contains mapping of auto-versionized records.
     *
     * @var array<string, array<int, string>>
     * @internal should only be used from within TYPO3 Core
     */
    public $autoVersionIdMap = [];

    /**
     * When new elements are created, this array contains a map between their "NEW..." string IDs and the eventual UID they got when stored in database
     *
     * @var array
     */
    public $substNEWwithIDs = [];

    /**
     * Like $substNEWwithIDs, but where each old "NEW..." id is mapped to the table it was from.
     *
     * @var array
     * @internal should only be used from within TYPO3 Core
     */
    public $substNEWwithIDs_table = [];

    /**
     * Holds the tables and there the ids of newly created child records from IRRE
     *
     * @var array
     * @internal should only be used from within TYPO3 Core
     */
    public $newRelatedIDs = [];

    /**
     * This array is the sum of all copying operations in this class.
     *
     * @var array
     * @internal should only be used from within TYPO3 Core
     */
    public $copyMappingArray_merged = [];

    /**
     * Per-table array with UIDs that have been deleted.
     *
     * @var array
     */
    protected $deletedRecords = [];

    /**
     * Errors are collected in this variable.
     *
     * @var array
     * @internal should only be used from within TYPO3 Core
     */
    public $errorLog = [];

    /**
     * Fields from the pages-table for which changes will trigger a pagetree refresh
     *
     * @var array
     */
    public $pagetreeRefreshFieldsFromPages = ['pid', 'sorting', 'deleted', 'hidden', 'title', 'doktype', 'is_siteroot', 'fe_group', 'nav_hide', 'nav_title', 'module', 'starttime', 'endtime', 'content_from_pid', 'extendToSubpages'];

    /**
     * Indicates whether the pagetree needs a refresh because of important changes
     *
     * @var bool
     * @internal should only be used from within TYPO3 Core
     */
    public $pagetreeNeedsRefresh = false;

    // *********************
    // Internal Variables, do not touch.
    // *********************

    // Variables set in init() function:

    /**
     * The user-object the script uses. If not set from outside, this is set to the current global $BE_USER.
     *
     * @var BackendUserAuthentication
     */
    public $BE_USER;

    /**
     * Will be set to uid of be_user executing this script
     *
     * @var int
     * @internal should only be used from within TYPO3 Core
     */
    public $userid;

    /**
     * Will be set to username of be_user executing this script
     *
     * @var string
     * @internal should only be used from within TYPO3 Core
     */
    public $username;

    /**
     * Will be set if user is admin
     *
     * @var bool
     * @internal should only be used from within TYPO3 Core
     */
    public $admin;

    /**
     * @var PagePermissionAssembler
     */
    protected $pagePermissionAssembler;

    /**
     * The list of <table>-<fields> that cannot be edited by user. This is compiled from TCA/exclude-flag combined with non_exclude_fields for the user.
     *
     * @var array
     */
    protected $excludedTablesAndFields = [];

    /**
     * Data submitted from the form view, used to control behaviours,
     * e.g. this is used to activate/deactivate fields and thus store NULL values
     *
     * @var array
     */
    protected $control = [];

    /**
     * Set with incoming data array
     *
     * @var array<int|string, array<int|string, array>>
     */
    public $datamap = [];

    /**
     * Set with incoming cmd array
     *
     * @var array<string, array<int|string, array>>
     */
    public $cmdmap = [];

    /**
     * List of changed old record ids to new records ids
     *
     * @var array
     */
    protected $mmHistoryRecords = [];

    /**
     * List of changed old record ids to new records ids
     *
     * @var array
     */
    protected $historyRecords = [];

    // Internal static:

    /**
     * The interval between sorting numbers used with tables with a 'sorting' field defined.
     *
     * Min 1, should be power of 2
     *
     * @var int
     * @internal should only be used from within TYPO3 Core
     */
    public $sortIntervals = 256;

    // Internal caching arrays
    /**
     * User by function checkRecordInsertAccess() to store whether a record can be inserted on a page id
     *
     * @var array
     */
    protected $recInsertAccessCache = [];

    /**
     * Caching array for check of whether records are in a webmount
     *
     * @var array
     */
    protected $isRecordInWebMount_Cache = [];

    /**
     * Caching array for page ids in webmounts
     *
     * @var array
     */
    protected $isInWebMount_Cache = [];

    /**
     * Used for caching page records in pageInfo()
     *
     * @var array<int, array<string, array>>
     */
    protected $pageCache = [];

    // Other arrays:
    /**
     * For accumulation of MM relations that must be written after new records are created.
     *
     * @var array
     * @internal
     */
    public $dbAnalysisStore = [];

    /**
     * Used for tracking references that might need correction after operations
     *
     * @var array<string, array<int, array>>
     * @internal
     */
    public $registerDBList = [];

    /**
     * Used for tracking references that might need correction in pid field after operations (e.g. IRRE)
     *
     * @var array
     * @internal
     */
    public $registerDBPids = [];

    /**
     * Used by the copy action to track the ids of new pages so subpages are correctly inserted!
     * THIS is internally cleared for each executed copy operation! DO NOT USE THIS FROM OUTSIDE!
     * Read from copyMappingArray_merged instead which is accumulating this information.
     *
     * NOTE: This is used by some outside scripts (e.g. hooks), as the results in $copyMappingArray_merged
     * are only available after an action has been completed.
     *
     * @var array<string, array>
     * @internal
     */
    public $copyMappingArray = [];

    /**
     * Array used for remapping uids and values at the end of process_datamap
     *
     * @var array
     * @internal
     */
    public $remapStack = [];

    /**
     * Array used for remapping uids and values at the end of process_datamap
     * (e.g. $remapStackRecords[<table>][<uid>] = <index in $remapStack>)
     *
     * @var array
     * @internal
     */
    public $remapStackRecords = [];

    /**
     * Array used for checking whether new children need to be remapped
     *
     * @var array
     */
    protected $remapStackChildIds = [];

    /**
     * Array used for executing addition actions after remapping happened (set processRemapStack())
     *
     * @var array
     */
    protected $remapStackActions = [];

    /**
     * Registry object to gather reference index update requests and perform updates after
     * main processing has been done. The first call to start() instantiates this object.
     * Recursive sub instances receive this instance via __construct().
     * The final update() call is done at the end of process_cmdmap() or process_datamap()
     * in the outer most instance.
     *
     * @var ReferenceIndexUpdater
     */
    protected $referenceIndexUpdater;

    // Various

    /**
     * Set to "currentRecord" during checking of values.
     *
     * @var array
     * @internal
     */
    public $checkValue_currentRecord = [];

    /**
     * Disable delete clause
     *
     * @var bool
     */
    protected $disableDeleteClause = false;

    /**
     * @var array
     */
    protected $checkModifyAccessListHookObjects;

    /**
     * The outer most instance of \TYPO3\CMS\Core\DataHandling\DataHandler:
     * This object instantiates itself on versioning and localization ...
     *
     * @var \TYPO3\CMS\Core\DataHandling\DataHandler
     */
    protected $outerMostInstance;

    /**
     * Internal cache for collecting records that should trigger cache clearing
     *
     * @var array
     */
    protected static $recordsToClearCacheFor = [];

    /**
     * Internal cache for pids of records which were deleted. It's not possible
     * to retrieve the parent folder/page at a later stage
     *
     * @var array
     */
    protected static $recordPidsForDeletedRecords = [];

    /**
     * Runtime Cache to store and retrieve data computed for a single request
     *
     * @var \TYPO3\CMS\Core\Cache\Frontend\FrontendInterface
     */
    protected $runtimeCache;

    /**
     * Prefix for the cache entries of nested element calls since the runtimeCache has a global scope.
     *
     * @var string
     */
    protected $cachePrefixNestedElementCalls = 'core-datahandler-nestedElementCalls-';

    /**
     * Sets up the data handler cache and some additional options, the main logic is done in the start() method.
     *
     * @param ReferenceIndexUpdater|null $referenceIndexUpdater Hand over from outer most instance to sub instances
     */
    public function __construct(ReferenceIndexUpdater $referenceIndexUpdater = null)
    {
        $this->checkStoredRecords = (bool)$GLOBALS['TYPO3_CONF_VARS']['BE']['checkStoredRecords'];
        $this->checkStoredRecords_loose = (bool)$GLOBALS['TYPO3_CONF_VARS']['BE']['checkStoredRecordsLoose'];
        $this->runtimeCache = $this->getRuntimeCache();
        $this->pagePermissionAssembler = GeneralUtility::makeInstance(PagePermissionAssembler::class, $GLOBALS['TYPO3_CONF_VARS']['BE']['defaultPermissions']);
        if ($referenceIndexUpdater === null) {
            // Create ReferenceIndexUpdater object. This should only happen on outer most instance,
            // sub instances should receive the reference index updater from a parent.
            $referenceIndexUpdater = GeneralUtility::makeInstance(ReferenceIndexUpdater::class);
        }
        $this->referenceIndexUpdater = $referenceIndexUpdater;
    }

    /**
     * @param array $control
     * @internal
     */
    public function setControl(array $control)
    {
        $this->control = $control;
    }

    /**
     * Initializing.
     * For details, see 'TYPO3 Core API' document.
     * This function does not start the processing of data, but merely initializes the object
     *
     * @param array $data Data to be modified or inserted in the database
     * @param array $cmd Commands to copy, move, delete, localize, versionize records.
     * @param BackendUserAuthentication|null $altUserObject An alternative userobject you can set instead of the default, which is $GLOBALS['BE_USER']
     */
    public function start($data, $cmd, $altUserObject = null)
    {
        // Initializing BE_USER
        $this->BE_USER = is_object($altUserObject) ? $altUserObject : $GLOBALS['BE_USER'];
        $this->userid = $this->BE_USER->user['uid'] ?? 0;
        $this->username = $this->BE_USER->user['username'] ?? '';
        $this->admin = $this->BE_USER->user['admin'] ?? false;

        // set correlation id for each new set of data or commands
        $this->correlationId = CorrelationId::forScope(
            md5(StringUtility::getUniqueId(self::class))
        );

        // Get default values from user TSconfig
        $tcaDefaultOverride = $this->BE_USER->getTSConfig()['TCAdefaults.'] ?? null;
        if (is_array($tcaDefaultOverride)) {
            $this->setDefaultsFromUserTS($tcaDefaultOverride);
        }

        // generates the excludelist, based on TCA/exclude-flag and non_exclude_fields for the user:
        if (!$this->admin) {
            $this->excludedTablesAndFields = array_flip($this->getExcludeListArray());
        }
        // Setting the data and cmd arrays
        if (is_array($data)) {
            reset($data);
            $this->datamap = $data;
        }
        if (is_array($cmd)) {
            reset($cmd);
            $this->cmdmap = $cmd;
        }
    }

    /**
     * Function that can mirror input values in datamap-array to other uid numbers.
     * Example: $mirror[table][11] = '22,33' will look for content in $this->datamap[table][11] and copy it to $this->datamap[table][22] and $this->datamap[table][33]
     *
     * @param array $mirror This array has the syntax $mirror[table_name][uid] = [list of uids to copy data-value TO!]
     * @internal
     */
    public function setMirror($mirror)
    {
        if (!is_array($mirror)) {
            return;
        }

        foreach ($mirror as $table => $uid_array) {
            if (!isset($this->datamap[$table])) {
                continue;
            }

            foreach ($uid_array as $id => $uidList) {
                if (!isset($this->datamap[$table][$id])) {
                    continue;
                }

                $theIdsInArray = GeneralUtility::trimExplode(',', $uidList, true);
                foreach ($theIdsInArray as $copyToUid) {
                    $this->datamap[$table][$copyToUid] = $this->datamap[$table][$id];
                }
            }
        }
    }

    /**
     * Initializes default values coming from User TSconfig
     *
     * @param array $userTS User TSconfig array
     * @internal should only be used from within DataHandler
     */
    public function setDefaultsFromUserTS($userTS)
    {
        if (!is_array($userTS)) {
            return;
        }

        foreach ($userTS as $k => $v) {
            $k = mb_substr($k, 0, -1);
            if (!$k || !is_array($v) || !isset($GLOBALS['TCA'][$k])) {
                continue;
            }

            if (is_array($this->defaultValues[$k] ?? false)) {
                $this->defaultValues[$k] = array_merge($this->defaultValues[$k], $v);
            } else {
                $this->defaultValues[$k] = $v;
            }
        }
    }

    /**
     * When a new record is created, all values that haven't been set but are set via PageTSconfig / UserTSconfig
     * get applied here.
     *
     * This is only executed for new records. The most important part is that the pageTS of the actual resolved $pid
     * is taken, and a new field array with empty defaults is set again.
     *
     * @param string $table
     * @param int $pageId
     * @param array $prepopulatedFieldArray
     * @return array
     */
    protected function applyDefaultsForFieldArray(string $table, int $pageId, array $prepopulatedFieldArray): array
    {
        // First set TCAdefaults respecting the given PageID
        $tcaDefaults = BackendUtility::getPagesTSconfig($pageId)['TCAdefaults.'] ?? null;
        // Re-apply $this->defaultValues settings
        $this->setDefaultsFromUserTS($tcaDefaults);
        $cleanFieldArray = $this->newFieldArray($table);
        if (isset($prepopulatedFieldArray['pid'])) {
            $cleanFieldArray['pid'] = $prepopulatedFieldArray['pid'];
        }
        $sortColumn = $GLOBALS['TCA'][$table]['ctrl']['sortby'] ?? null;
        if ($sortColumn !== null && isset($prepopulatedFieldArray[$sortColumn])) {
            $cleanFieldArray[$sortColumn] = $prepopulatedFieldArray[$sortColumn];
        }
        return $cleanFieldArray;
    }

    /*********************************************
     *
     * HOOKS
     *
     *********************************************/
    /**
     * Hook: processDatamap_afterDatabaseOperations
     * (calls $hookObj->processDatamap_afterDatabaseOperations($status, $table, $id, $fieldArray, $this);)
     *
     * Note: When using the hook after INSERT operations, you will only get the temporary NEW... id passed to your hook as $id,
     * but you can easily translate it to the real uid of the inserted record using the $this->substNEWwithIDs array.
     *
     * @param array $hookObjectsArr (reference) Array with hook objects
     * @param string $status (reference) Status of the current operation, 'new' or 'update
     * @param string $table (reference) The table currently processing data for
     * @param string $id (reference) The record uid currently processing data for, [integer] or [string] (like 'NEW...')
     * @param array $fieldArray (reference) The field array of a record
     * @internal should only be used from within DataHandler
     */
    public function hook_processDatamap_afterDatabaseOperations(&$hookObjectsArr, &$status, &$table, &$id, &$fieldArray)
    {
        // Process hook directly:
        if (!isset($this->remapStackRecords[$table][$id])) {
            foreach ($hookObjectsArr as $hookObj) {
                if (method_exists($hookObj, 'processDatamap_afterDatabaseOperations')) {
                    $hookObj->processDatamap_afterDatabaseOperations($status, $table, $id, $fieldArray, $this);
                }
            }
        } else {
            $this->remapStackRecords[$table][$id]['processDatamap_afterDatabaseOperations'] = [
                'status' => $status,
                'fieldArray' => $fieldArray,
                'hookObjectsArr' => $hookObjectsArr,
            ];
        }
    }

    /**
     * Gets the 'checkModifyAccessList' hook objects.
     * The first call initializes the accordant objects.
     *
     * @return array The 'checkModifyAccessList' hook objects (if any)
     * @throws \UnexpectedValueException
     */
    protected function getCheckModifyAccessListHookObjects()
    {
        if (!isset($this->checkModifyAccessListHookObjects)) {
            $this->checkModifyAccessListHookObjects = [];
            foreach ($GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['t3lib/class.t3lib_tcemain.php']['checkModifyAccessList'] ?? [] as $className) {
                $hookObject = GeneralUtility::makeInstance($className);
                if (!$hookObject instanceof DataHandlerCheckModifyAccessListHookInterface) {
                    throw new \UnexpectedValueException($className . ' must implement interface ' . DataHandlerCheckModifyAccessListHookInterface::class, 1251892472);
                }
                $this->checkModifyAccessListHookObjects[] = $hookObject;
            }
        }
        return $this->checkModifyAccessListHookObjects;
    }

    /*********************************************
     *
     * PROCESSING DATA
     *
     *********************************************/
    /**
     * Processing the data-array
     * Call this function to process the data-array set by start()
     *
     * @return bool|void
     */
    public function process_datamap()
    {
        $this->controlActiveElements();

        // Keep versionized(!) relations here locally:
        $registerDBList = [];
        $this->registerElementsToBeDeleted();
        $this->datamap = $this->unsetElementsToBeDeleted($this->datamap);
        // Editing frozen:
        if ($this->BE_USER->workspace !== 0 && ($this->BE_USER->workspaceRec['freeze'] ?? false)) {
            $this->log('sys_workspace', $this->BE_USER->workspace, SystemLogDatabaseAction::VERSIONIZE, 0, SystemLogErrorClassification::USER_ERROR, 'All editing in this workspace has been frozen!');
            return false;
        }
        // First prepare user defined objects (if any) for hooks which extend this function:
        $hookObjectsArr = [];
        foreach ($GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['t3lib/class.t3lib_tcemain.php']['processDatamapClass'] ?? [] as $className) {
            $hookObject = GeneralUtility::makeInstance($className);
            if (method_exists($hookObject, 'processDatamap_beforeStart')) {
                $hookObject->processDatamap_beforeStart($this);
            }
            $hookObjectsArr[] = $hookObject;
        }
        // Pre-process data-map and synchronize localization states
        $this->datamap = GeneralUtility::makeInstance(SlugEnricher::class)->enrichDataMap($this->datamap);
        $this->datamap = DataMapProcessor::instance($this->datamap, $this->BE_USER, $this->referenceIndexUpdater)->process();
        // Organize tables so that the pages-table is always processed first. This is required if you want to make sure that content pointing to a new page will be created.
        $orderOfTables = [];
        // Set pages first.
        if (isset($this->datamap['pages'])) {
            $orderOfTables[] = 'pages';
        }
        $orderOfTables = array_unique(array_merge($orderOfTables, array_keys($this->datamap)));
        // Process the tables...
        foreach ($orderOfTables as $table) {
            // Check if
            //	   - table is set in $GLOBALS['TCA'],
            //	   - table is NOT readOnly
            //	   - the table is set with content in the data-array (if not, there's nothing to process...)
            //	   - permissions for tableaccess OK
            $modifyAccessList = $this->checkModifyAccessList($table);
            if (!$modifyAccessList) {
                $this->log($table, 0, SystemLogDatabaseAction::UPDATE, 0, SystemLogErrorClassification::USER_ERROR, 'Attempt to modify table \'%s\' without permission', 1, [$table]);
            }
            if (!isset($GLOBALS['TCA'][$table]) || $this->tableReadOnly($table) || !is_array($this->datamap[$table]) || !$modifyAccessList) {
                continue;
            }

            if ($this->reverseOrder) {
                $this->datamap[$table] = array_reverse($this->datamap[$table], true);
            }
            // For each record from the table, do:
            // $id is the record uid, may be a string if new records...
            // $incomingFieldArray is the array of fields
            foreach ($this->datamap[$table] as $id => $incomingFieldArray) {
                if (!is_array($incomingFieldArray)) {
                    continue;
                }
                $theRealPid = null;

                // Hook: processDatamap_preProcessFieldArray
                foreach ($hookObjectsArr as $hookObj) {
                    if (method_exists($hookObj, 'processDatamap_preProcessFieldArray')) {
                        $hookObj->processDatamap_preProcessFieldArray($incomingFieldArray, $table, $id, $this);
                    }
                }
                // ******************************
                // Checking access to the record
                // ******************************
                $createNewVersion = false;
                $old_pid_value = '';
                // Is it a new record? (Then Id is a string)
                if (!MathUtility::canBeInterpretedAsInteger($id)) {
                    // Get a fieldArray with tca default values
                    $fieldArray = $this->newFieldArray($table);
                    // A pid must be set for new records.
                    if (isset($incomingFieldArray['pid'])) {
                        $pid_value = $incomingFieldArray['pid'];
                        // Checking and finding numerical pid, it may be a string-reference to another value
                        $canProceed = true;
                        // If a NEW... id
                        if (str_contains($pid_value, 'NEW')) {
                            if ($pid_value[0] === '-') {
                                $negFlag = -1;
                                $pid_value = substr($pid_value, 1);
                            } else {
                                $negFlag = 1;
                            }
                            // Trying to find the correct numerical value as it should be mapped by earlier processing of another new record.
                            if (isset($this->substNEWwithIDs[$pid_value])) {
                                if ($negFlag === 1) {
                                    $old_pid_value = $this->substNEWwithIDs[$pid_value];
                                }
                                $pid_value = (int)($negFlag * $this->substNEWwithIDs[$pid_value]);
                            } else {
                                $canProceed = false;
                            }
                        }
                        $pid_value = (int)$pid_value;
                        if ($canProceed) {
                            $fieldArray = $this->resolveSortingAndPidForNewRecord($table, $pid_value, $fieldArray);
                        }
                    }
                    $theRealPid = $fieldArray['pid'];
                    // Checks if records can be inserted on this $pid.
                    // If this is a page translation, the check needs to be done for the l10n_parent record
                    $languageField = $GLOBALS['TCA'][$table]['ctrl']['languageField'] ?? null;
                    $transOrigPointerField = $GLOBALS['TCA'][$table]['ctrl']['transOrigPointerField'] ?? null;
                    if ($table === 'pages'
                        && $languageField && isset($incomingFieldArray[$languageField]) && $incomingFieldArray[$languageField] > 0
                        && $transOrigPointerField && isset($incomingFieldArray[$transOrigPointerField]) && $incomingFieldArray[$transOrigPointerField] > 0
                    ) {
                        $recordAccess = $this->checkRecordInsertAccess($table, $incomingFieldArray[$transOrigPointerField]);
                    } else {
                        $recordAccess = $this->checkRecordInsertAccess($table, $theRealPid);
                    }
                    if ($recordAccess) {
                        $this->addDefaultPermittedLanguageIfNotSet($table, $incomingFieldArray, $theRealPid);
                        $recordAccess = $this->BE_USER->recordEditAccessInternals($table, $incomingFieldArray, true);
                        if (!$recordAccess) {
                            $this->log($table, 0, SystemLogDatabaseAction::INSERT, 0, SystemLogErrorClassification::USER_ERROR, 'recordEditAccessInternals() check failed. [' . $this->BE_USER->errorMsg . ']');
                        } elseif (!$this->bypassWorkspaceRestrictions && !$this->BE_USER->workspaceAllowsLiveEditingInTable($table)) {
                            // If LIVE records cannot be created due to workspace restrictions, prepare creation of placeholder-record
                            // So, if no live records were allowed in the current workspace, we have to create a new version of this record
                            if (BackendUtility::isTableWorkspaceEnabled($table)) {
                                $createNewVersion = true;
                            } else {
                                $recordAccess = false;
                                $this->log($table, 0, SystemLogDatabaseAction::VERSIONIZE, 0, SystemLogErrorClassification::USER_ERROR, 'Record could not be created in this workspace');
                            }
                        }
                    }
                    // Yes new record, change $record_status to 'insert'
                    $status = 'new';
                } else {
                    // Nope... $id is a number
                    $id = (int)$id;
                    $fieldArray = [];
                    $recordAccess = $this->checkRecordUpdateAccess($table, $id, $incomingFieldArray, $hookObjectsArr);
                    if (!$recordAccess) {
                        if ($this->enableLogging) {
                            $propArr = $this->getRecordProperties($table, $id);
                            $this->log($table, $id, SystemLogDatabaseAction::UPDATE, 0, SystemLogErrorClassification::USER_ERROR, 'Attempt to modify record \'%s\' (%s) without permission. Or non-existing page.', 2, [$propArr['header'], $table . ':' . $id], $propArr['event_pid']);
                        }
                        continue;
                    }
                    // Next check of the record permissions (internals)
                    $recordAccess = $this->BE_USER->recordEditAccessInternals($table, $id);
                    if (!$recordAccess) {
                        $this->log($table, $id, SystemLogDatabaseAction::UPDATE, 0, SystemLogErrorClassification::USER_ERROR, 'recordEditAccessInternals() check failed. [' . $this->BE_USER->errorMsg . ']');
                    } else {
                        // Here we fetch the PID of the record that we point to...
                        $tempdata = $this->recordInfo($table, $id, 'pid' . (BackendUtility::isTableWorkspaceEnabled($table) ? ',t3ver_oid,t3ver_wsid,t3ver_stage' : ''));
                        $theRealPid = $tempdata['pid'] ?? null;
                        // Use the new id of the versionized record we're trying to write to:
                        // (This record is a child record of a parent and has already been versionized.)
                        if (!empty($this->autoVersionIdMap[$table][$id])) {
                            // For the reason that creating a new version of this record, automatically
                            // created related child records (e.g. "IRRE"), update the accordant field:
                            $this->getVersionizedIncomingFieldArray($table, $id, $incomingFieldArray, $registerDBList);
                            // Use the new id of the copied/versionized record:
                            $id = $this->autoVersionIdMap[$table][$id];
                            $recordAccess = true;
                        } elseif (!$this->bypassWorkspaceRestrictions && $tempdata && ($errorCode = $this->workspaceCannotEditRecord($table, $tempdata))) {
                            $recordAccess = false;
                            // Versioning is required and it must be offline version!
                            // Check if there already is a workspace version
                            $workspaceVersion = BackendUtility::getWorkspaceVersionOfRecord($this->BE_USER->workspace, $table, $id, 'uid,t3ver_oid');
                            if ($workspaceVersion) {
                                $id = $workspaceVersion['uid'];
                                $recordAccess = true;
                            } elseif ($this->workspaceAllowAutoCreation($table, $id, $theRealPid)) {
                                // new version of a record created in a workspace - so always refresh pagetree to indicate there is a change in the workspace
                                $this->pagetreeNeedsRefresh = true;

                                /** @var DataHandler $tce */
                                $tce = GeneralUtility::makeInstance(__CLASS__, $this->referenceIndexUpdater);
                                $tce->enableLogging = $this->enableLogging;
                                // Setting up command for creating a new version of the record:
                                $cmd = [];
                                $cmd[$table][$id]['version'] = [
                                    'action' => 'new',
                                    // Default is to create a version of the individual records
                                    'label' => 'Auto-created for WS #' . $this->BE_USER->workspace,
                                ];
                                $tce->start([], $cmd, $this->BE_USER);
                                $tce->process_cmdmap();
                                $this->errorLog = array_merge($this->errorLog, $tce->errorLog);
                                // If copying was successful, share the new uids (also of related children):
                                if (!empty($tce->copyMappingArray[$table][$id])) {
                                    foreach ($tce->copyMappingArray as $origTable => $origIdArray) {
                                        foreach ($origIdArray as $origId => $newId) {
                                            $this->autoVersionIdMap[$origTable][$origId] = $newId;
                                        }
                                    }
                                    // Update registerDBList, that holds the copied relations to child records:
                                    $registerDBList = array_merge($registerDBList, $tce->registerDBList);
                                    // For the reason that creating a new version of this record, automatically
                                    // created related child records (e.g. "IRRE"), update the accordant field:
                                    $this->getVersionizedIncomingFieldArray($table, $id, $incomingFieldArray, $registerDBList);
                                    // Use the new id of the copied/versionized record:
                                    $id = $this->autoVersionIdMap[$table][$id];
                                    $recordAccess = true;
                                } else {
                                    $this->log($table, $id, SystemLogDatabaseAction::VERSIONIZE, 0, SystemLogErrorClassification::USER_ERROR, 'Could not be edited in offline workspace in the branch where found (failure state: \'' . $errorCode . '\'). Auto-creation of version failed!');
                                }
                            } else {
                                $this->log($table, $id, SystemLogDatabaseAction::VERSIONIZE, 0, SystemLogErrorClassification::USER_ERROR, 'Could not be edited in offline workspace in the branch where found (failure state: \'' . $errorCode . '\'). Auto-creation of version not allowed in workspace!');
                            }
                        }
                    }
                    // The default is 'update'
                    $status = 'update';
                }
                // If access was granted above, proceed to create or update record:
                if (!$recordAccess) {
                    continue;
                }

                // Here the "pid" is set IF NOT the old pid was a string pointing to a place in the subst-id array.
                [$tscPID] = BackendUtility::getTSCpid($table, $id, $old_pid_value ?: ($fieldArray['pid'] ?? 0));
                if ($status === 'new') {
                    // Apply TCAdefaults from pageTS
                    $fieldArray = $this->applyDefaultsForFieldArray($table, (int)$tscPID, $fieldArray);
                    // Apply page permissions as well
                    if ($table === 'pages') {
                        $fieldArray = $this->pagePermissionAssembler->applyDefaults(
                            $fieldArray,
                            (int)$tscPID,
                            (int)$this->userid,
                            (int)$this->BE_USER->firstMainGroup
                        );
                    }
                    // Ensure that the default values, that are stored in the $fieldArray (built from internal default values)
                    // Are also placed inside the incomingFieldArray, so this is checked in "fillInFieldArray" and
                    // all default values are also checked for validity
                    // This allows to set TCAdefaults (for example) without having to use FormEngine to have the fields available first.
                    $incomingFieldArray = array_replace_recursive($fieldArray, $incomingFieldArray);
                }
                // Processing of all fields in incomingFieldArray and setting them in $fieldArray
                $fieldArray = $this->fillInFieldArray($table, $id, $fieldArray, $incomingFieldArray, $theRealPid, $status, $tscPID);
                // NOTICE! All manipulation beyond this point bypasses both "excludeFields" AND possible "MM" relations to field!
                // Forcing some values unto field array:
                // NOTICE: This overriding is potentially dangerous; permissions per field is not checked!!!
                $fieldArray = $this->overrideFieldArray($table, $fieldArray);
                // Setting system fields
                if ($status === 'new') {
                    if ($GLOBALS['TCA'][$table]['ctrl']['crdate'] ?? false) {
                        $fieldArray[$GLOBALS['TCA'][$table]['ctrl']['crdate']] = $GLOBALS['EXEC_TIME'];
                    }
                    if ($GLOBALS['TCA'][$table]['ctrl']['cruser_id'] ?? false) {
                        $fieldArray[$GLOBALS['TCA'][$table]['ctrl']['cruser_id']] = $this->userid;
                    }
                } elseif ($this->checkSimilar) {
                    // Removing fields which are equal to the current value:
                    $fieldArray = $this->compareFieldArrayWithCurrentAndUnset($table, $id, $fieldArray);
                }
                if (($GLOBALS['TCA'][$table]['ctrl']['tstamp'] ?? false) && !empty($fieldArray)) {
                    $fieldArray[$GLOBALS['TCA'][$table]['ctrl']['tstamp']] = $GLOBALS['EXEC_TIME'];
                }
                // Set stage to "Editing" to make sure we restart the workflow
                if (BackendUtility::isTableWorkspaceEnabled($table)) {
                    $fieldArray['t3ver_stage'] = 0;
                }
                // Hook: processDatamap_postProcessFieldArray
                foreach ($hookObjectsArr as $hookObj) {
                    if (method_exists($hookObj, 'processDatamap_postProcessFieldArray')) {
                        $hookObj->processDatamap_postProcessFieldArray($status, $table, $id, $fieldArray, $this);
                    }
                }
                // Performing insert/update. If fieldArray has been unset by some userfunction (see hook above), don't do anything
                // Kasper: Unsetting the fieldArray is dangerous; MM relations might be saved already
                if (is_array($fieldArray)) {
                    if ($status === 'new') {
                        if ($table === 'pages') {
                            // for new pages always a refresh is needed
                            $this->pagetreeNeedsRefresh = true;
                        }

                        // This creates a version of the record, instead of adding it to the live workspace
                        if ($createNewVersion) {
                            // new record created in a workspace - so always refresh pagetree to indicate there is a change in the workspace
                            $this->pagetreeNeedsRefresh = true;
                            $fieldArray['pid'] = $theRealPid;
                            $fieldArray['t3ver_oid'] = 0;
                            // Setting state for version (so it can know it is currently a new version...)
                            $fieldArray['t3ver_state'] = (string)new VersionState(VersionState::NEW_PLACEHOLDER);
                            $fieldArray['t3ver_wsid'] = $this->BE_USER->workspace;
                            $this->insertDB($table, $id, $fieldArray, true, (int)($incomingFieldArray['uid'] ?? 0));
                            // Hold auto-versionized ids of placeholders
                            $this->autoVersionIdMap[$table][$this->substNEWwithIDs[$id]] = $this->substNEWwithIDs[$id];
                        } else {
                            $this->insertDB($table, $id, $fieldArray, false, (int)($incomingFieldArray['uid'] ?? 0));
                        }
                    } else {
                        if ($table === 'pages') {
                            // only a certain number of fields needs to be checked for updates
                            // if $this->checkSimilar is TRUE, fields with unchanged values are already removed here
                            $fieldsToCheck = array_intersect($this->pagetreeRefreshFieldsFromPages, array_keys($fieldArray));
                            if (!empty($fieldsToCheck)) {
                                $this->pagetreeNeedsRefresh = true;
                            }
                        }
                        $this->updateDB($table, $id, $fieldArray);
                    }
                }
                // Hook: processDatamap_afterDatabaseOperations
                // Note: When using the hook after INSERT operations, you will only get the temporary NEW... id passed to your hook as $id,
                // but you can easily translate it to the real uid of the inserted record using the $this->substNEWwithIDs array.
                $this->hook_processDatamap_afterDatabaseOperations($hookObjectsArr, $status, $table, $id, $fieldArray);
            }
        }
        // Process the stack of relations to remap/correct
        $this->processRemapStack();
        $this->dbAnalysisStoreExec();
        // Hook: processDatamap_afterAllOperations
        // Note: When this hook gets called, all operations on the submitted data have been finished.
        foreach ($hookObjectsArr as $hookObj) {
            if (method_exists($hookObj, 'processDatamap_afterAllOperations')) {
                $hookObj->processDatamap_afterAllOperations($this);
            }
        }

        if ($this->isOuterMostInstance()) {
            $this->referenceIndexUpdater->update();
            $this->processClearCacheQueue();
            $this->resetElementsToBeDeleted();
        }
    }

    /**
     * @param string $table
     * @param string $value
     * @param string $dbType
     * @return string
     */
    protected function normalizeTimeFormat(string $table, string $value, string $dbType): string
    {
        $connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable($table);
        $platform = $connection->getDatabasePlatform();
        if ($platform instanceof SQLServerPlatform) {
            $defaultLength = QueryHelper::getDateTimeFormats()[$dbType]['empty'];
            $value = substr(
                $value,
                0,
                strlen($defaultLength)
            );
        }
        return $value;
    }

    /**
     * Sets the "sorting" DB field and the "pid" field of an incoming record that should be added (NEW1234)
     * depending on the record that should be added or where it should be added.
     *
     * This method is called from process_datamap()
     *
     * @param string $table the table name of the record to insert
     * @param int $pid the real PID (numeric) where the record should be
     * @param array $fieldArray field+value pairs to add
     * @return array the modified field array
     */
    protected function resolveSortingAndPidForNewRecord(string $table, int $pid, array $fieldArray): array
    {
        $sortColumn = $GLOBALS['TCA'][$table]['ctrl']['sortby'] ?? '';
        // Points to a page on which to insert the element, possibly in the top of the page
        if ($pid >= 0) {
            // Ensure that the "pid" is not a translated page ID, but the default page ID
            $pid = $this->getDefaultLanguagePageId($pid);
            // The numerical pid is inserted in the data array
            $fieldArray['pid'] = $pid;
            // If this table is sorted we better find the top sorting number
            if ($sortColumn) {
                $fieldArray[$sortColumn] = $this->getSortNumber($table, 0, $pid);
            }
        } elseif ($sortColumn) {
            // Points to another record before itself
            // If this table is sorted we better find the top sorting number
            // Because $pid is < 0, getSortNumber() returns an array
            $sortingInfo = $this->getSortNumber($table, 0, $pid);
            $fieldArray['pid'] = $sortingInfo['pid'];
            $fieldArray[$sortColumn] = $sortingInfo['sortNumber'];
        } else {
            // Here we fetch the PID of the record that we point to
            $record = $this->recordInfo($table, abs($pid), 'pid');
            // Ensure that the "pid" is not a translated page ID, but the default page ID