[BUGFIX] Fix extension data import on sqlite 03/57103/7
authorSusanne Moog <susanne.moog@typo3.org>
Thu, 31 May 2018 11:53:56 +0000 (13:53 +0200)
committerAndreas Fernandez <a.fernandez@scripting-base.de>
Fri, 1 Jun 2018 10:14:41 +0000 (12:14 +0200)
Sqlite limits the maximum number of variables
allowed in a query. When importing the extension
list the chunks are too big (> 999 variables).

This has been chunked to enable import on sqlite.

Resolves: #85129
Releases: master
Change-Id: If65476707946f4078a1b1e55128239aed07f6ed8
Reviewed-on: https://review.typo3.org/57103
Reviewed-by: Oliver Hader <oliver.hader@typo3.org>
Tested-by: Oliver Hader <oliver.hader@typo3.org>
Tested-by: TYPO3com <no-reply@typo3.com>
Reviewed-by: Anja Leichsenring <aleichsenring@ab-softlab.de>
Tested-by: Anja Leichsenring <aleichsenring@ab-softlab.de>
Reviewed-by: Mathias Brodala <mbrodala@pagemachine.de>
Reviewed-by: Andreas Fernandez <a.fernandez@scripting-base.de>
Tested-by: Andreas Fernandez <a.fernandez@scripting-base.de>
typo3/sysext/extensionmanager/Classes/Domain/Repository/ExtensionRepository.php
typo3/sysext/extensionmanager/Classes/Utility/Importer/ExtensionListUtility.php

index 54d0c71..276eadc 100644 (file)
@@ -16,6 +16,7 @@ namespace TYPO3\CMS\Extensionmanager\Domain\Repository;
 
 use TYPO3\CMS\Core\Database\Connection;
 use TYPO3\CMS\Core\Database\ConnectionPool;
+use TYPO3\CMS\Core\Database\Platform\PlatformInformation;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 
 /**
@@ -338,17 +339,24 @@ class ExtensionRepository extends \TYPO3\CMS\Extbase\Persistence\Repository
         $uidsOfCurrentVersion = $this->fetchMaximalVersionsForAllExtensions($repositoryUid);
         $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
             ->getQueryBuilderForTable(self::TABLE_NAME);
+        $connection = GeneralUtility::makeInstance(ConnectionPool::class)
+            ->getConnectionForTable(self::TABLE_NAME);
+        $maxBindParameters = PlatformInformation::getMaxBindParameters(
+            $connection->getDatabasePlatform()
+        );
 
-        $queryBuilder
-            ->update(self::TABLE_NAME)
-            ->where(
-                $queryBuilder->expr()->in(
-                    'uid',
-                    $queryBuilder->createNamedParameter($uidsOfCurrentVersion, Connection::PARAM_INT_ARRAY)
+        foreach (array_chunk($uidsOfCurrentVersion, $maxBindParameters - 10) as $chunk) {
+            $queryBuilder
+                ->update(self::TABLE_NAME)
+                ->where(
+                    $queryBuilder->expr()->in(
+                        'uid',
+                        $queryBuilder->createNamedParameter($chunk, Connection::PARAM_INT_ARRAY)
+                    )
                 )
-            )
-            ->set('current_version', 1)
-            ->execute();
+                ->set('current_version', 1)
+                ->execute();
+        }
     }
 
     /**
index 8242e11..e8b23c1 100644 (file)
@@ -14,6 +14,7 @@ namespace TYPO3\CMS\Extensionmanager\Utility\Importer;
  * The TYPO3 project - inspiring people to share!
  */
 use TYPO3\CMS\Core\Database\ConnectionPool;
+use TYPO3\CMS\Core\Database\Platform\PlatformInformation;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 use TYPO3\CMS\Core\Utility\VersionNumberUtility;
 use TYPO3\CMS\Extensionmanager\Utility\Parser\AbstractExtensionXmlParser;
@@ -73,6 +74,21 @@ class ExtensionListUtility implements \SplObserver
     ];
 
     /**
+     * Table name to be used to store extension models.
+     *
+     * @var string
+     */
+    protected static $tableName = 'tx_extensionmanager_domain_model_extension';
+
+    /**
+     * Maximum of rows that can be used in a bulk insert for the current
+     * database platform.
+     *
+     * @var int
+     */
+    protected $maxRowsPerChunk = 50;
+
+    /**
      * Keeps indexes of fields that should not be quoted.
      *
      * @var array
@@ -132,6 +148,19 @@ class ExtensionListUtility implements \SplObserver
                 1476108717
             );
         }
+
+        $connection = GeneralUtility::makeInstance(ConnectionPool::class)
+            ->getConnectionForTable(self::$tableName);
+        $maxBindParameters = PlatformInformation::getMaxBindParameters(
+            $connection->getDatabasePlatform()
+        );
+        $countOfBindParamsPerRow = count(self::$fieldNames);
+        // flush at least chunks of 50 elements - in case the currently used
+        // database platform does not support that, the threshold is lowered
+        $this->maxRowsPerChunk = min(
+            $this->maxRowsPerChunk,
+            floor($maxBindParameters / $countOfBindParamsPerRow)
+        );
     }
 
     /**
@@ -171,12 +200,11 @@ class ExtensionListUtility implements \SplObserver
      */
     protected function loadIntoDatabase(AbstractExtensionXmlParser &$subject)
     {
-        // flush every 50 rows to database
-        if ($this->sumRecords !== 0 && $this->sumRecords % 50 === 0) {
+        if ($this->sumRecords !== 0 && $this->sumRecords % $this->maxRowsPerChunk === 0) {
             GeneralUtility::makeInstance(ConnectionPool::class)
-                ->getConnectionForTable('tx_extensionmanager_domain_model_extension')
+                ->getConnectionForTable(self::$tableName)
                 ->bulkInsert(
-                    'tx_extensionmanager_domain_model_extension',
+                    self::$tableName,
                     $this->arrRows,
                     self::$fieldNames
                 );