[BUGFIX] Process large list of hashes in chunks updating reference index 56/58756/2
authorBenni Mack <benni@typo3.org>
Sun, 28 Oct 2018 12:12:52 +0000 (13:12 +0100)
committerBenni Mack <benni@typo3.org>
Sun, 28 Oct 2018 12:37:55 +0000 (13:37 +0100)
The list of old hashes by cleaning up the reference index can
be very long. To avoid exceeding query limits like maximum number
of placeholder per query or the max allowed statement length these
large lists are split into chunks of safe length before processing.

Resolves: #85795
Releases: master, 8.7
Change-Id: Iaebc555fe791f9e41f336bcd30d5b6eb149ad4e9
Reviewed-on: https://review.typo3.org/58756
Tested-by: TYPO3com <no-reply@typo3.com>
Reviewed-by: Susanne Moog <susanne.moog@typo3.org>
Reviewed-by: Benni Mack <benni@typo3.org>
Tested-by: Benni Mack <benni@typo3.org>
typo3/sysext/core/Classes/Database/ReferenceIndex.php

index 9a118d0..08a779f 100644 (file)
@@ -18,6 +18,7 @@ use Doctrine\DBAL\DBALException;
 use TYPO3\CMS\Backend\Utility\BackendUtility;
 use TYPO3\CMS\Core\Cache\CacheManager;
 use TYPO3\CMS\Core\Configuration\FlexForm\FlexFormTools;
+use TYPO3\CMS\Core\Database\Platform\PlatformInformation;
 use TYPO3\CMS\Core\DataHandling\DataHandler;
 use TYPO3\CMS\Core\Messaging\FlashMessage;
 use TYPO3\CMS\Core\Messaging\FlashMessageService;
@@ -286,16 +287,22 @@ class ReferenceIndex
                 $result['deletedNodes'] = count($hashList);
                 $result['deletedNodes_hashList'] = implode(',', $hashList);
                 if (!$testOnly) {
-                    $queryBuilder = $connection->createQueryBuilder();
-                    $queryBuilder
-                        ->delete('sys_refindex')
-                        ->where(
-                            $queryBuilder->expr()->in(
-                                'hash',
-                                $queryBuilder->createNamedParameter($hashList, Connection::PARAM_STR_ARRAY)
+                    $maxBindParameters = PlatformInformation::getMaxBindParameters($connection->getDatabasePlatform());
+                    foreach (array_chunk($hashList, $maxBindParameters - 10, true) as $chunk) {
+                        if (empty($chunk)) {
+                            continue;
+                        }
+                        $queryBuilder = $connection->createQueryBuilder();
+                        $queryBuilder
+                            ->delete('sys_refindex')
+                            ->where(
+                                $queryBuilder->expr()->in(
+                                    'hash',
+                                    $queryBuilder->createNamedParameter($chunk, Connection::PARAM_STR_ARRAY)
+                                )
                             )
-                        )
-                        ->execute();
+                            ->execute();
+                    }
                 }
             }
         }