[BUGFIX] Process large list of hashes in chunks updating reference index 46/57846/4
authorGleb Levitin <gleb.levitin@dkd.de>
Thu, 9 Aug 2018 15:14:25 +0000 (17:14 +0200)
committerBenni Mack <benni@typo3.org>
Sun, 28 Oct 2018 12:08:57 +0000 (13:08 +0100)
The list of old hashes by cleaning up the reference index can
be very long. To avoid exceeding query limits like maximum number
of placeholder per query or the max allowed statement length these
large lists are split into chunks of safe length before processing.

Resolves: #85795
Releases: master, 8.7
Change-Id: Iaebc555fe791f9e41f336bcd30d5b6eb149ad4e9
Reviewed-on: https://review.typo3.org/57846
Reviewed-by: Markus Klein <markus.klein@typo3.org>
Tested-by: TYPO3com <no-reply@typo3.com>
Reviewed-by: Guido Schmechel <guido.schmechel@brandung.de>
Reviewed-by: Susanne Moog <susanne.moog@typo3.org>
Tested-by: Susanne Moog <susanne.moog@typo3.org>
Reviewed-by: Benni Mack <benni@typo3.org>
Tested-by: Benni Mack <benni@typo3.org>
typo3/sysext/core/Classes/Database/ReferenceIndex.php

index 07b13a9..fcd9f54 100644 (file)
@@ -21,6 +21,7 @@ use TYPO3\CMS\Backend\Utility\BackendUtility;
 use TYPO3\CMS\Core\Cache\CacheManager;
 use TYPO3\CMS\Core\Configuration\FlexForm\FlexFormTools;
 use TYPO3\CMS\Core\Core\Environment;
+use TYPO3\CMS\Core\Database\Platform\PlatformInformation;
 use TYPO3\CMS\Core\DataHandling\DataHandler;
 use TYPO3\CMS\Core\Messaging\FlashMessage;
 use TYPO3\CMS\Core\Messaging\FlashMessageService;
@@ -291,16 +292,22 @@ class ReferenceIndex implements LoggerAwareInterface
                 $result['deletedNodes'] = count($hashList);
                 $result['deletedNodes_hashList'] = implode(',', $hashList);
                 if (!$testOnly) {
-                    $queryBuilder = $connection->createQueryBuilder();
-                    $queryBuilder
-                        ->delete('sys_refindex')
-                        ->where(
-                            $queryBuilder->expr()->in(
-                                'hash',
-                                $queryBuilder->createNamedParameter($hashList, Connection::PARAM_STR_ARRAY)
+                    $maxBindParameters = PlatformInformation::getMaxBindParameters($connection->getDatabasePlatform());
+                    foreach (array_chunk($hashList, $maxBindParameters - 10, true) as $chunk) {
+                        if (empty($chunk)) {
+                            continue;
+                        }
+                        $queryBuilder = $connection->createQueryBuilder();
+                        $queryBuilder
+                            ->delete('sys_refindex')
+                            ->where(
+                                $queryBuilder->expr()->in(
+                                    'hash',
+                                    $queryBuilder->createNamedParameter($chunk, Connection::PARAM_STR_ARRAY)
+                                )
                             )
-                        )
-                        ->execute();
+                            ->execute();
+                    }
                 }
             }
         }