[TASK] Doctrine: Migrate SqlSchemaMigrationService 85/49585/7
authorChristian Kuhn <lolli@schwarzbu.ch>
Wed, 24 Aug 2016 22:47:38 +0000 (00:47 +0200)
committerWouter Wolters <typo3@wouterwolters.nl>
Thu, 1 Sep 2016 16:54:49 +0000 (18:54 +0200)
The schema migration service has some parts that are ext:dbal
specific which can not be mimik'ed easily at the moment.
To still proceed with the doctrine migration, the original
class is copied over to ext:dbal and registiered as xclass,
while a low-level migration is applied to ext:install
SqlSchemaMigrationService.
This way, the schema migration and load order for now works
with both ext:dbal loaded and not loaded within install tool.
The patch further prepares isolation of ext:dbal and dropping
of a bootstrap database method.

Change-Id: I600229c56160a6eb9f1589c070b5e68a2fc95678
Resolves: #77627
Releases: master
Reviewed-on: https://review.typo3.org/49585
Tested-by: Bamboo TYPO3com <info@typo3.com>
Reviewed-by: Morton Jonuschat <m.jonuschat@mojocode.de>
Tested-by: Morton Jonuschat <m.jonuschat@mojocode.de>
Reviewed-by: Wouter Wolters <typo3@wouterwolters.nl>
Tested-by: Wouter Wolters <typo3@wouterwolters.nl>
typo3/sysext/dbal/Classes/Service/SqlSchemaMigrationService.php [new file with mode: 0644]
typo3/sysext/dbal/Tests/Unit/Service/SqlSchemaMigrationServiceTest.php [new file with mode: 0644]
typo3/sysext/dbal/ext_localconf.php
typo3/sysext/install/Classes/Controller/AbstractController.php
typo3/sysext/install/Classes/Service/SqlSchemaMigrationService.php
typo3/sysext/install/Tests/Unit/Service/SqlSchemaMigrationServiceTest.php

diff --git a/typo3/sysext/dbal/Classes/Service/SqlSchemaMigrationService.php b/typo3/sysext/dbal/Classes/Service/SqlSchemaMigrationService.php
new file mode 100644 (file)
index 0000000..e6b5009
--- /dev/null
@@ -0,0 +1,698 @@
+<?php
+namespace TYPO3\CMS\Dbal\Service;
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+/**
+ * XCLASS of ext:install SqlSchemaMigrationService containing
+ * a couple of ext:dbal specifics.
+ */
+class SqlSchemaMigrationService
+{
+    /**
+     * @constant Maximum field width of MySQL
+     */
+    const MYSQL_MAXIMUM_FIELD_WIDTH = 64;
+
+    /**
+     * @var string Prefix of deleted tables
+     */
+    protected $deletedPrefixKey = 'zzz_deleted_';
+
+    /**
+     * @var array Caching output of $GLOBALS['TYPO3_DB']->admin_get_charsets()
+     */
+    protected $character_sets = [];
+
+    /**
+     * Set prefix of deleted tables
+     *
+     * @param string $prefix Prefix string
+     */
+    public function setDeletedPrefixKey($prefix)
+    {
+        $this->deletedPrefixKey = $prefix;
+    }
+
+    /**
+     * Get prefix of deleted tables
+     *
+     * @return string
+     */
+    public function getDeletedPrefixKey()
+    {
+        return $this->deletedPrefixKey;
+    }
+
+    /**
+     * Reads the field definitions for the input SQL-file string
+     *
+     * @param string $fileContent Should be a string read from an SQL-file made with 'mysqldump [database_name] -d'
+     * @return array Array with information about table.
+     */
+    public function getFieldDefinitions_fileContent($fileContent)
+    {
+        $lines = \TYPO3\CMS\Core\Utility\GeneralUtility::trimExplode(LF, $fileContent, true);
+        $table = '';
+        $total = [];
+        foreach ($lines as $value) {
+            if ($value[0] === '#') {
+                // Ignore comments
+                continue;
+            }
+            if ($table === '') {
+                $parts = \TYPO3\CMS\Core\Utility\GeneralUtility::trimExplode(' ', $value, true);
+                if (strtoupper($parts[0]) === 'CREATE' && strtoupper($parts[1]) === 'TABLE') {
+                    $table = str_replace('`', '', $parts[2]);
+                    // tablenames are always lowercase on windows!
+                    if (TYPO3_OS == 'WIN') {
+                        $table = strtolower($table);
+                    }
+                }
+            } else {
+                if ($value[0] === ')' && substr($value, -1) === ';') {
+                    $ttype = [];
+                    if (preg_match('/(ENGINE|TYPE)[ ]*=[ ]*([a-zA-Z]*)/', $value, $ttype)) {
+                        $total[$table]['extra']['ENGINE'] = $ttype[2];
+                    }
+                    // Otherwise, just do nothing: If table engine is not defined, just accept the system default.
+                    // Set the collation, if specified
+                    if (preg_match('/(COLLATE)[ ]*=[ ]*([a-zA-z0-9_-]+)/', $value, $tcollation)) {
+                        $total[$table]['extra']['COLLATE'] = $tcollation[2];
+                    } else {
+                        // Otherwise, get the CHARACTER SET and try to find the default collation for it as returned by "SHOW CHARACTER SET" query (for details, see http://dev.mysql.com/doc/refman/5.1/en/charset-table.html)
+                        if (preg_match('/(CHARSET|CHARACTER SET)[ ]*=[ ]*([a-zA-z0-9_-]+)/', $value, $tcharset)) {
+                            // Note: Keywords "DEFAULT CHARSET" and "CHARSET" are the same, so "DEFAULT" can just be ignored
+                            $charset = $tcharset[2];
+                        } else {
+                            $charset = $this->getDatabaseConnection()->default_charset;
+                        }
+                        $total[$table]['extra']['COLLATE'] = $this->getCollationForCharset($charset);
+                    }
+                    // Remove table marker and start looking for the next "CREATE TABLE" statement
+                    $table = '';
+                } else {
+                    // Strip trailing commas
+                    $lineV = preg_replace('/,$/', '', $value);
+                    $lineV = str_replace('`', '', $lineV);
+                    // Reduce multiple blanks and tabs except newline
+                    $lineV = preg_replace('/\h+/', ' ', $lineV);
+                    $parts = explode(' ', $lineV, 2);
+                    // Field definition
+                    if (!preg_match('/(PRIMARY|UNIQUE|FULLTEXT|SPATIAL|INDEX|KEY)/', $parts[0])) {
+                        // Make sure there is no default value when auto_increment is set
+                        if (stristr($parts[1], 'auto_increment')) {
+                            $parts[1] = preg_replace('/ default \'0\'/i', '', $parts[1]);
+                        }
+                        // "default" is always lower-case
+                        if (stristr($parts[1], ' DEFAULT ')) {
+                            $parts[1] = str_ireplace(' DEFAULT ', ' default ', $parts[1]);
+                        }
+                        // Change order of "default" and "NULL" statements
+                        $parts[1] = preg_replace('/(.*) (default .*) (NOT NULL)/', '$1 $3 $2', $parts[1]);
+                        $parts[1] = preg_replace('/(.*) (default .*) (NULL)/', '$1 $3 $2', $parts[1]);
+                        $key = $parts[0];
+                        $total[$table]['fields'][$key] = $parts[1];
+                    } else {
+                        // Key definition
+                        $search = ['/UNIQUE (INDEX|KEY)/', '/FULLTEXT (INDEX|KEY)/', '/SPATIAL (INDEX|KEY)/', '/INDEX/'];
+                        $replace = ['UNIQUE', 'FULLTEXT', 'SPATIAL', 'KEY'];
+                        $lineV = preg_replace($search, $replace, $lineV);
+                        if (preg_match('/PRIMARY|UNIQUE|FULLTEXT|SPATIAL/', $parts[0])) {
+                            $parts[1] = preg_replace('/^(KEY|INDEX) /', '', $parts[1]);
+                        }
+                        $newParts = explode(' ', $parts[1], 2);
+                        $key = $parts[0] == 'PRIMARY' ? $parts[0] : $newParts[0];
+                        $total[$table]['keys'][$key] = $lineV;
+                        // This is a protection against doing something stupid: Only allow clearing of cache_* and index_* tables.
+                        if (preg_match('/^(cache|index)_/', $table)) {
+                            // Suggest to truncate (clear) this table
+                            $total[$table]['extra']['CLEAR'] = 1;
+                        }
+                    }
+                }
+            }
+        }
+        return $total;
+    }
+
+    /**
+     * Look up the default collation for specified character set based on "SHOW CHARACTER SET" output
+     *
+     * @param string $charset Character set
+     * @return string Corresponding default collation
+     */
+    public function getCollationForCharset($charset)
+    {
+        // Load character sets, if not cached already
+        if (empty($this->character_sets)) {
+            $databaseConnection = $this->getDatabaseConnection();
+            if (method_exists($databaseConnection, 'admin_get_charsets')) {
+                $this->character_sets = $databaseConnection->admin_get_charsets();
+            } else {
+                // Add empty element to avoid that the check will be repeated
+                $this->character_sets[$charset] = [];
+            }
+        }
+        $collation = '';
+        if (isset($this->character_sets[$charset]['Default collation'])) {
+            $collation = $this->character_sets[$charset]['Default collation'];
+        }
+        return $collation;
+    }
+
+    /**
+     * Reads the field definitions for the current database
+     *
+     * @return array Array with information about table.
+     */
+    public function getFieldDefinitions_database()
+    {
+        $total = [];
+        $tempKeys = [];
+        $tempKeysPrefix = [];
+        $databaseConnection = $this->getDatabaseConnection();
+        $databaseConnection->connectDB();
+        echo $databaseConnection->sql_error();
+        $tables = $databaseConnection->admin_get_tables();
+        foreach ($tables as $tableName => $tableStatus) {
+            // Fields
+            $fieldInformation = $databaseConnection->admin_get_fields($tableName);
+            foreach ($fieldInformation as $fN => $fieldRow) {
+                $total[$tableName]['fields'][$fN] = $this->assembleFieldDefinition($fieldRow);
+            }
+            // Keys
+            $keyInformation = $databaseConnection->admin_get_keys($tableName);
+            foreach ($keyInformation as $keyRow) {
+                $keyName = $keyRow['Key_name'];
+                $colName = $keyRow['Column_name'];
+                if ($keyRow['Sub_part'] && $keyRow['Index_type'] !== 'SPATIAL') {
+                    $colName .= '(' . $keyRow['Sub_part'] . ')';
+                }
+                $tempKeys[$tableName][$keyName][$keyRow['Seq_in_index']] = $colName;
+                if ($keyName == 'PRIMARY') {
+                    $prefix = 'PRIMARY KEY';
+                } else {
+                    if ($keyRow['Index_type'] === 'FULLTEXT') {
+                        $prefix = 'FULLTEXT';
+                    } elseif ($keyRow['Index_type'] === 'SPATIAL') {
+                        $prefix = 'SPATIAL';
+                    } elseif ($keyRow['Non_unique']) {
+                        $prefix = 'KEY';
+                    } else {
+                        $prefix = 'UNIQUE';
+                    }
+                    $prefix .= ' ' . $keyName;
+                }
+                $tempKeysPrefix[$tableName][$keyName] = $prefix;
+            }
+            // Table status (storage engine, collaction, etc.)
+            if (is_array($tableStatus)) {
+                $tableExtraFields = [
+                    'Engine' => 'ENGINE',
+                    'Collation' => 'COLLATE'
+                ];
+                foreach ($tableExtraFields as $mysqlKey => $internalKey) {
+                    if (isset($tableStatus[$mysqlKey])) {
+                        $total[$tableName]['extra'][$internalKey] = $tableStatus[$mysqlKey];
+                    }
+                }
+            }
+        }
+        // Compile key information:
+        if (!empty($tempKeys)) {
+            foreach ($tempKeys as $table => $keyInf) {
+                foreach ($keyInf as $kName => $index) {
+                    ksort($index);
+                    $total[$table]['keys'][$kName] = $tempKeysPrefix[$table][$kName] . ' (' . implode(',', $index) . ')';
+                }
+            }
+        }
+        return $total;
+    }
+
+    /**
+     * Compares two arrays with field information and returns information about fields that are MISSING and fields that have CHANGED.
+     * FDsrc and FDcomp can be switched if you want the list of stuff to remove rather than update.
+     *
+     * @param array $FDsrc Field definitions, source (from getFieldDefinitions_fileContent())
+     * @param array $FDcomp Field definitions, comparison. (from getFieldDefinitions_database())
+     * @param string $onlyTableList Table names (in list) which is the ONLY one observed.
+     * @param bool $ignoreNotNullWhenComparing If set, this function ignores NOT NULL statements of the SQL file field definition when comparing current field definition from database with field definition from SQL file. This way, NOT NULL statements will be executed when the field is initially created, but the SQL parser will never complain about missing NOT NULL statements afterwards.
+     * @return array Returns an array with 1) all elements from $FDsrc that is not in $FDcomp (in key 'extra') and 2) all elements from $FDsrc that is different from the ones in $FDcomp
+     */
+    public function getDatabaseExtra($FDsrc, $FDcomp, $onlyTableList = '', $ignoreNotNullWhenComparing = false)
+    {
+        $extraArr = [];
+        $diffArr = [];
+        if (is_array($FDsrc)) {
+            foreach ($FDsrc as $table => $info) {
+                if ($onlyTableList === '' || \TYPO3\CMS\Core\Utility\GeneralUtility::inList($onlyTableList, $table)) {
+                    if (!isset($FDcomp[$table])) {
+                        // If the table was not in the FDcomp-array, the result array is loaded with that table.
+                        $extraArr[$table] = $info;
+                        $extraArr[$table]['whole_table'] = 1;
+                    } else {
+                        $keyTypes = explode(',', 'extra,fields,keys');
+                        foreach ($keyTypes as $theKey) {
+                            if (is_array($info[$theKey])) {
+                                foreach ($info[$theKey] as $fieldN => $fieldC) {
+                                    $fieldN = str_replace('`', '', $fieldN);
+                                    if ($this->isDbalEnabled() && $fieldN === 'ENGINE') {
+                                        continue;
+                                    }
+                                    if ($fieldN == 'COLLATE') {
+                                        // @todo collation support is currently disabled (needs more testing)
+                                        continue;
+                                    }
+                                    if (!isset($FDcomp[$table][$theKey][$fieldN])) {
+                                        $extraArr[$table][$theKey][$fieldN] = $fieldC;
+                                    } else {
+                                        $fieldC = trim($fieldC);
+
+                                        // Lowercase the field type to surround false-positive schema changes to be
+                                        // reported just because of different caseing of characters
+                                        // The regex does just trigger for the first word followed by parentheses
+                                        // that contain a length. It does not trigger for e.g. "PRIMARY KEY" because
+                                        // "PRIMARY KEY" is being returned from the DB in upper case.
+                                        $fieldC = preg_replace_callback(
+                                            '/^([a-zA-Z0-9]+)(\([^)]*\)\s.*)/',
+                                            function ($matches) {
+                                                return strtolower($matches[1]) . $matches[2];
+                                            },
+                                            $fieldC
+                                        );
+
+                                        if ($this->isDbalEnabled()) {
+                                            // Ignore nonstandard MySQL numeric field attributes UNSIGNED and ZEROFILL
+                                            if (preg_match('/^(TINYINT|SMALLINT|MEDIUMINT|INT|INTEGER|BIGINT|REAL|DOUBLE|FLOAT|DECIMAL|NUMERIC)\([^\)]+\)\s+(UNSIGNED|ZEROFILL)/i', $fieldC)) {
+                                                $fieldC = str_ireplace([' UNSIGNED', ' ZEROFILL'], '', $fieldC);
+                                                $FDcomp[$table][$theKey][$fieldN] = str_ireplace([' UNSIGNED', ' ZEROFILL'], '', $FDcomp[$table][$theKey][$fieldN]);
+                                            }
+
+                                            // Replace field and index definitions with functionally equivalent statements
+                                            if ($fieldC !== $FDcomp[$table][$theKey][$fieldN]) {
+                                                switch ($theKey) {
+                                                    case 'fields':
+                                                        $fieldC = $this->getDatabaseConnection()->getEquivalentFieldDefinition($fieldC);
+                                                        break;
+                                                    case 'keys':
+                                                        $fieldC = $this->getDatabaseConnection()->getEquivalentIndexDefinition($fieldC);
+                                                        break;
+                                                }
+                                            }
+                                        }
+                                        if ($ignoreNotNullWhenComparing) {
+                                            $fieldC = str_replace(' NOT NULL', '', $fieldC);
+                                            $FDcomp[$table][$theKey][$fieldN] = str_replace(' NOT NULL', '', $FDcomp[$table][$theKey][$fieldN]);
+                                        }
+                                        if ($fieldC !== $FDcomp[$table][$theKey][$fieldN]) {
+                                            $diffArr[$table][$theKey][$fieldN] = $fieldC;
+                                            $diffArr_cur[$table][$theKey][$fieldN] = $FDcomp[$table][$theKey][$fieldN];
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        $output = [
+            'extra' => $extraArr,
+            'diff' => $diffArr,
+            'diff_currentValues' => $diffArr_cur
+        ];
+        return $output;
+    }
+
+    /**
+     * Returns an array with SQL-statements that is needed to update according to the diff-array
+     *
+     * @param array $diffArr Array with differences of current and needed DB settings. (from getDatabaseExtra())
+     * @param string $keyList List of fields in diff array to take notice of.
+     * @return array Array of SQL statements (organized in keys depending on type)
+     */
+    public function getUpdateSuggestions($diffArr, $keyList = 'extra,diff')
+    {
+        $statements = [];
+        $deletedPrefixKey = $this->deletedPrefixKey;
+        $deletedPrefixLength = strlen($deletedPrefixKey);
+        $remove = 0;
+        if ($keyList == 'remove') {
+            $remove = 1;
+            $keyList = 'extra';
+        }
+        $keyList = explode(',', $keyList);
+        foreach ($keyList as $theKey) {
+            if (is_array($diffArr[$theKey])) {
+                foreach ($diffArr[$theKey] as $table => $info) {
+                    $whole_table = [];
+                    if (isset($info['keys']) && is_array($info['keys'])) {
+                        foreach ($info['keys'] as $fN => $fV) {
+                            if (!$info['whole_table'] && $theKey === 'extra' && $remove) {
+                                $statement = 'ALTER TABLE ' . $table . ($fN === 'PRIMARY' ? ' DROP PRIMARY KEY' : ' DROP KEY ' . $fN) . ';';
+                                $statements['drop'][md5($statement)] = $statement;
+                            }
+                        }
+                    }
+                    if (is_array($info['fields'])) {
+                        foreach ($info['fields'] as $fN => $fV) {
+                            if ($info['whole_table']) {
+                                $whole_table[] = $fN . ' ' . $fV;
+                            } else {
+                                // Special case to work around MySQL problems when adding auto_increment fields:
+                                if (stristr($fV, 'auto_increment')) {
+                                    // The field can only be set "auto_increment" if there exists a PRIMARY key of that field already.
+                                    // The check does not look up which field is primary but just assumes it must be the field with the auto_increment value...
+                                    if (isset($info['keys']['PRIMARY'])) {
+                                        if (!$this->isDbalEnabled()) {
+                                            // Combine adding the field and the primary key into a single statement
+                                            $fV .= ', ADD PRIMARY KEY (' . $fN . ')';
+                                            unset($info['keys']['PRIMARY']);
+                                        }
+                                    } else {
+                                        // In the next step, attempt to clear the table once again (2 = force)
+                                        $info['extra']['CLEAR'] = 2;
+                                    }
+                                }
+                                if ($theKey == 'extra') {
+                                    if ($remove) {
+                                        if (substr($fN, 0, $deletedPrefixLength) !== $deletedPrefixKey) {
+                                            // we've to make sure we don't exceed the maximal length
+                                            $prefixedFieldName = $deletedPrefixKey . substr($fN, ($deletedPrefixLength - self::MYSQL_MAXIMUM_FIELD_WIDTH));
+                                            $statement = 'ALTER TABLE ' . $table . ' CHANGE ' . $fN . ' ' . $prefixedFieldName . ' ' . $fV . ';';
+                                            $statements['change'][md5($statement)] = $statement;
+                                        } else {
+                                            $statement = 'ALTER TABLE ' . $table . ' DROP ' . $fN . ';';
+                                            $statements['drop'][md5($statement)] = $statement;
+                                        }
+                                    } else {
+                                        $statement = 'ALTER TABLE ' . $table . ' ADD ' . $fN . ' ' . $fV . ';';
+                                        $statements['add'][md5($statement)] = $statement;
+                                    }
+                                } elseif ($theKey == 'diff') {
+                                    $statement = 'ALTER TABLE ' . $table . ' CHANGE ' . $fN . ' ' . $fN . ' ' . $fV . ';';
+                                    $statements['change'][md5($statement)] = $statement;
+                                    $statements['change_currentValue'][md5($statement)] = $diffArr['diff_currentValues'][$table]['fields'][$fN];
+                                }
+                            }
+                        }
+                    }
+                    if (is_array($info['keys'])) {
+                        foreach ($info['keys'] as $fN => $fV) {
+                            if ($info['whole_table']) {
+                                $whole_table[] = $fV;
+                            } else {
+                                if ($theKey == 'extra') {
+                                    if (!$remove) {
+                                        $statement = 'ALTER TABLE ' . $table . ' ADD ' . $fV . ';';
+                                        $statements['add'][md5($statement)] = $statement;
+                                    }
+                                } elseif ($theKey == 'diff') {
+                                    $statement = 'ALTER TABLE ' . $table . ($fN == 'PRIMARY' ? ' DROP PRIMARY KEY' : ' DROP KEY ' . $fN) . ';';
+                                    $statements['change'][md5($statement)] = $statement;
+                                    $statement = 'ALTER TABLE ' . $table . ' ADD ' . $fV . ';';
+                                    $statements['change'][md5($statement)] = $statement;
+                                }
+                            }
+                        }
+                    }
+                    if (is_array($info['extra'])) {
+                        $extras = [];
+                        $extras_currentValue = [];
+                        $clear_table = false;
+                        foreach ($info['extra'] as $fN => $fV) {
+                            // Only consider statements which are missing in the database but don't remove existing properties
+                            if (!$remove) {
+                                if (!$info['whole_table']) {
+                                    // If the whole table is created at once, we take care of this later by imploding all elements of $info['extra']
+                                    if ($fN == 'CLEAR') {
+                                        // Truncate table must happen later, not now
+                                        // Valid values for CLEAR: 1=only clear if keys are missing, 2=clear anyway (force)
+                                        if (!empty($info['keys']) || $fV == 2) {
+                                            $clear_table = true;
+                                        }
+                                        continue;
+                                    } else {
+                                        $extras[] = $fN . '=' . $fV;
+                                        $extras_currentValue[] = $fN . '=' . $diffArr['diff_currentValues'][$table]['extra'][$fN];
+                                    }
+                                }
+                            }
+                        }
+                        if ($clear_table) {
+                            $statement = 'TRUNCATE TABLE ' . $table . ';';
+                            $statements['clear_table'][md5($statement)] = $statement;
+                        }
+                        if (!empty($extras)) {
+                            $statement = 'ALTER TABLE ' . $table . ' ' . implode(' ', $extras) . ';';
+                            $statements['change'][md5($statement)] = $statement;
+                            $statements['change_currentValue'][md5($statement)] = implode(' ', $extras_currentValue);
+                        }
+                    }
+                    if ($info['whole_table']) {
+                        if ($remove) {
+                            if (substr($table, 0, $deletedPrefixLength) !== $deletedPrefixKey) {
+                                // we've to make sure we don't exceed the maximal length
+                                $prefixedTableName = $deletedPrefixKey . substr($table, ($deletedPrefixLength - self::MYSQL_MAXIMUM_FIELD_WIDTH));
+                                $statement = 'ALTER TABLE ' . $table . ' RENAME ' . $prefixedTableName . ';';
+                                $statements['change_table'][md5($statement)] = $statement;
+                            } else {
+                                $statement = 'DROP TABLE ' . $table . ';';
+                                $statements['drop_table'][md5($statement)] = $statement;
+                            }
+                            // Count
+                            $count = $this->getDatabaseConnection()->exec_SELECTcountRows('*', $table);
+                            $statements['tables_count'][md5($statement)] = $count ? 'Records in table: ' . $count : '';
+                        } else {
+                            $statement = 'CREATE TABLE ' . $table . ' (
+' . implode(',
+', $whole_table) . '
+)';
+                            if ($info['extra']) {
+                                foreach ($info['extra'] as $k => $v) {
+                                    if ($k == 'COLLATE' || $k == 'CLEAR') {
+                                        // Skip these special statements.
+                                        // @todo collation support is currently disabled (needs more testing)
+                                        continue;
+                                    }
+                                    // Add extra attributes like ENGINE, CHARSET, etc.
+                                    $statement .= ' ' . $k . '=' . $v;
+                                }
+                            }
+                            $statement .= ';';
+                            $statements['create_table'][md5($statement)] = $statement;
+                        }
+                    }
+                }
+            }
+        }
+        return $statements;
+    }
+
+    /**
+     * Converts a result row with field information into the SQL field definition string
+     *
+     * @param array $row MySQL result row
+     * @return string Field definition
+     */
+    public function assembleFieldDefinition($row)
+    {
+        $field = [$row['Type']];
+        if ($row['Null'] == 'NO') {
+            $field[] = 'NOT NULL';
+        }
+        if (!strstr($row['Type'], 'blob') && !strstr($row['Type'], 'text')) {
+            // Add a default value if the field is not auto-incremented (these fields never have a default definition)
+            if (!stristr($row['Extra'], 'auto_increment')) {
+                if ($row['Default'] === null) {
+                    $field[] = 'default NULL';
+                } else {
+                    $field[] = 'default \'' . addslashes($row['Default']) . '\'';
+                }
+            }
+        }
+        if ($row['Extra']) {
+            $field[] = $row['Extra'];
+        }
+        if (trim($row['Comment']) !== '') {
+            $field[] = "COMMENT '" . $row['Comment'] . "'";
+        }
+        return implode(' ', $field);
+    }
+
+    /**
+     * Returns an array where every entry is a single SQL-statement. Input must be formatted like an ordinary MySQL-dump files.
+     *
+     * @param string $sqlcode The SQL-file content. Provided that 1) every query in the input is ended with ';' and that a line in the file contains only one query or a part of a query.
+     * @param bool $removeNonSQL If set, non-SQL content (like comments and blank lines) is not included in the final output
+     * @param string $query_regex Regex to filter SQL lines to include
+     * @return array Array of SQL statements
+     */
+    public function getStatementArray($sqlcode, $removeNonSQL = false, $query_regex = '')
+    {
+        $sqlcodeArr = explode(LF, $sqlcode);
+        // Based on the assumption that the sql-dump has
+        $statementArray = [];
+        $statementArrayPointer = 0;
+        foreach ($sqlcodeArr as $line => $lineContent) {
+            $lineContent = trim($lineContent);
+            $is_set = 0;
+            // Auto_increment fields cannot have a default value!
+            if (stristr($lineContent, 'auto_increment')) {
+                $lineContent = preg_replace('/ default \'0\'/i', '', $lineContent);
+            }
+            if (!$removeNonSQL || $lineContent !== '' && $lineContent[0] !== '#' && substr($lineContent, 0, 2) !== '--') {
+                // '--' is seen as mysqldump comments from server version 3.23.49
+                $statementArray[$statementArrayPointer] .= $lineContent;
+                $is_set = 1;
+            }
+            if (substr($lineContent, -1) === ';') {
+                if (isset($statementArray[$statementArrayPointer])) {
+                    if (!trim($statementArray[$statementArrayPointer]) || $query_regex && !preg_match(('/' . $query_regex . '/i'), trim($statementArray[$statementArrayPointer]))) {
+                        unset($statementArray[$statementArrayPointer]);
+                    }
+                }
+                $statementArrayPointer++;
+            } elseif ($is_set) {
+                $statementArray[$statementArrayPointer] .= LF;
+            }
+        }
+        return $statementArray;
+    }
+
+    /**
+     * Returns tables to create and how many records in each
+     *
+     * @param array $statements Array of SQL statements to analyse.
+     * @param bool $insertCountFlag If set, will count number of INSERT INTO statements following that table definition
+     * @return array Array with table definitions in index 0 and count in index 1
+     */
+    public function getCreateTables($statements, $insertCountFlag = false)
+    {
+        $crTables = [];
+        $insertCount = [];
+        foreach ($statements as $line => $lineContent) {
+            $reg = [];
+            if (preg_match('/^create[[:space:]]*table[[:space:]]*[`]?([[:alnum:]_]*)[`]?/i', substr($lineContent, 0, 100), $reg)) {
+                $table = trim($reg[1]);
+                if ($table) {
+                    // Table names are always lowercase on Windows!
+                    if (TYPO3_OS == 'WIN') {
+                        $table = strtolower($table);
+                    }
+                    $sqlLines = explode(LF, $lineContent);
+                    foreach ($sqlLines as $k => $v) {
+                        if (stristr($v, 'auto_increment')) {
+                            $sqlLines[$k] = preg_replace('/ default \'0\'/i', '', $v);
+                        }
+                    }
+                    $lineContent = implode(LF, $sqlLines);
+                    $crTables[$table] = $lineContent;
+                }
+            } elseif ($insertCountFlag && preg_match('/^insert[[:space:]]*into[[:space:]]*[`]?([[:alnum:]_]*)[`]?/i', substr($lineContent, 0, 100), $reg)) {
+                $nTable = trim($reg[1]);
+                $insertCount[$nTable]++;
+            }
+        }
+        return [$crTables, $insertCount];
+    }
+
+    /**
+     * Extracts all insert statements from $statement array where content is inserted into $table
+     *
+     * @param array $statements Array of SQL statements
+     * @param string $table Table name
+     * @return array Array of INSERT INTO statements where table match $table
+     */
+    public function getTableInsertStatements($statements, $table)
+    {
+        $outStatements = [];
+        foreach ($statements as $line => $lineContent) {
+            $reg = [];
+            if (preg_match('/^insert[[:space:]]*into[[:space:]]*[`]?([[:alnum:]_]*)[`]?/i', substr($lineContent, 0, 100), $reg)) {
+                $nTable = trim($reg[1]);
+                if ($nTable && $table === $nTable) {
+                    $outStatements[] = $lineContent;
+                }
+            }
+        }
+        return $outStatements;
+    }
+
+    /**
+     * Performs the queries passed from the input array.
+     *
+     * @param array $arr Array of SQL queries to execute.
+     * @param array $keyArr Array with keys that must match keys in $arr. Only where a key in this array is set and TRUE will the query be executed (meant to be passed from a form checkbox)
+     * @return mixed Array with error message from database if any occurred. Otherwise TRUE if everything was executed successfully.
+     */
+    public function performUpdateQueries($arr, $keyArr)
+    {
+        $result = [];
+        if (is_array($arr)) {
+            $databaseConnection = $this->getDatabaseConnection();
+            foreach ($arr as $key => $string) {
+                if (isset($keyArr[$key]) && $keyArr[$key]) {
+                    $res = $databaseConnection->admin_query($string);
+                    if ($res === false) {
+                        $result[$key] = $databaseConnection->sql_error();
+                    } elseif (is_resource($res) || is_a($res, '\\mysqli_result')) {
+                        $databaseConnection->sql_free_result($res);
+                    }
+                }
+            }
+        }
+        if (!empty($result)) {
+            return $result;
+        } else {
+            return true;
+        }
+    }
+
+    /**
+     * Returns list of tables in the database
+     *
+     * @return array List of tables.
+     * @see \TYPO3\CMS\Core\Database\DatabaseConnection::admin_get_tables()
+     */
+    public function getListOfTables()
+    {
+        $whichTables = $this->getDatabaseConnection()->admin_get_tables();
+        foreach ($whichTables as $key => &$value) {
+            $value = $key;
+        }
+        unset($value);
+        return $whichTables;
+    }
+
+    /**
+     * Checks if DBAL is enabled for the database connection
+     *
+     * @return bool
+     */
+    protected function isDbalEnabled()
+    {
+        return \TYPO3\CMS\Core\Utility\ExtensionManagementUtility::isLoaded('dbal');
+    }
+
+    /**
+     * @return \TYPO3\CMS\Core\Database\DatabaseConnection|\TYPO3\CMS\Dbal\Database\DatabaseConnection
+     */
+    protected function getDatabaseConnection()
+    {
+        return $GLOBALS['TYPO3_DB'];
+    }
+}
diff --git a/typo3/sysext/dbal/Tests/Unit/Service/SqlSchemaMigrationServiceTest.php b/typo3/sysext/dbal/Tests/Unit/Service/SqlSchemaMigrationServiceTest.php
new file mode 100644 (file)
index 0000000..e155f63
--- /dev/null
@@ -0,0 +1,647 @@
+<?php
+namespace TYPO3\CMS\Dbal\Tests\Unit\Service;
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+use TYPO3\CMS\Core\Tests\UnitTestCase;
+use TYPO3\CMS\Dbal\Service\SqlSchemaMigrationService;
+
+/**
+ * Test case
+ */
+class SqlSchemaMigrationServiceTest extends UnitTestCase
+{
+    /**
+     * Get a SchemaService instance with mocked DBAL enable database connection, DBAL not enabled
+     *
+     * @return \PHPUnit_Framework_MockObject_MockObject|\TYPO3\CMS\Core\Tests\AccessibleObjectInterface
+     */
+    protected function getSqlSchemaMigrationService()
+    {
+        /** @var \TYPO3\CMS\Dbal\Database\DatabaseConnection|\PHPUnit_Framework_MockObject_MockObject|\TYPO3\CMS\Core\Tests\AccessibleObjectInterface $databaseConnection */
+        $subject = $this->getAccessibleMock(SqlSchemaMigrationService::class, ['isDbalEnabled'], [], '', false);
+        $subject->expects($this->any())->method('isDbalEnabled')->will($this->returnValue(false));
+
+        return $subject;
+    }
+
+    /**
+     * Get a SchemaService instance with mocked DBAL enable database connection, DBAL enabled
+     *
+     * @return \PHPUnit_Framework_MockObject_MockObject|\TYPO3\CMS\Core\Tests\AccessibleObjectInterface
+     */
+    protected function getDbalEnabledSqlSchemaMigrationService()
+    {
+        /** @var \TYPO3\CMS\Dbal\Database\DatabaseConnection|\PHPUnit_Framework_MockObject_MockObject|\TYPO3\CMS\Core\Tests\AccessibleObjectInterface $databaseConnection */
+        $databaseConnection = $this->getAccessibleMock(\TYPO3\CMS\Dbal\Database\DatabaseConnection::class, ['dummy'], [], '', false);
+        $databaseConnection->_set(
+            'dbmsSpecifics',
+            new \TYPO3\CMS\Dbal\Database\Specifics\PostgresSpecifics()
+        );
+
+        $subject = $this->getAccessibleMock(SqlSchemaMigrationService::class, ['isDbalEnabled', 'getDatabaseConnection'], [], '', false);
+        $subject->expects($this->any())->method('isDbalEnabled')->will($this->returnValue(true));
+        $subject->expects($this->any())->method('getDatabaseConnection')->will($this->returnValue($databaseConnection));
+
+        return $subject;
+    }
+
+    /**
+     * @test
+     */
+    public function getFieldDefinitionsFileContentHandlesMultipleWhitespacesInFieldDefinitions()
+    {
+        $subject = $this->getSqlSchemaMigrationService();
+        // Multiple whitespaces and tabs in field definition
+        $inputString = 'CREATE table atable (' . LF . 'aFieldName   int(11)' . TAB . TAB . TAB . 'unsigned   DEFAULT \'0\'' . LF . ');';
+        $result = $subject->getFieldDefinitions_fileContent($inputString);
+
+        $this->assertEquals(
+            [
+                'atable' => [
+                    'fields' => [
+                        'aFieldName' => 'int(11) unsigned default \'0\'',
+                    ],
+                    'extra' => [
+                        'COLLATE' => '',
+                    ],
+                ],
+            ],
+            $result
+        );
+    }
+
+    /**
+     * @test
+     */
+    public function getDatabaseExtraFindsChangedFields()
+    {
+        $subject = $this->getSqlSchemaMigrationService();
+        $differenceArray = $subject->getDatabaseExtra(
+            [
+                'tx_foo' => [
+                    'fields' => [
+                        'foo' => 'varchar(999) DEFAULT \'0\' NOT NULL'
+                    ]
+                ]
+            ],
+            [
+                'tx_foo' => [
+                    'fields' => [
+                        'foo' => 'varchar(255) DEFAULT \'0\' NOT NULL'
+                    ]
+                ]
+            ]
+        );
+
+        $this->assertEquals(
+            $differenceArray,
+            [
+                'extra' => [],
+                'diff' => [
+                    'tx_foo' => [
+                        'fields' => [
+                            'foo' => 'varchar(999) DEFAULT \'0\' NOT NULL'
+                        ]
+                    ]
+                ],
+                'diff_currentValues' => [
+                    'tx_foo' => [
+                        'fields' => [
+                            'foo' => 'varchar(255) DEFAULT \'0\' NOT NULL'
+                        ]
+                    ]
+                ]
+            ]
+        );
+    }
+
+    /**
+     * @test
+     */
+    public function getDatabaseExtraFindsChangedFieldsIncludingNull()
+    {
+        $subject = $this->getSqlSchemaMigrationService();
+        $differenceArray = $subject->getDatabaseExtra(
+            [
+                'tx_foo' => [
+                    'fields' => [
+                        'foo' => 'varchar(999) NULL'
+                    ]
+                ]
+            ],
+            [
+                'tx_foo' => [
+                    'fields' => [
+                        'foo' => 'varchar(255) NULL'
+                    ]
+                ]
+            ]
+        );
+
+        $this->assertEquals(
+            $differenceArray,
+            [
+                'extra' => [],
+                'diff' => [
+                    'tx_foo' => [
+                        'fields' => [
+                            'foo' => 'varchar(999) NULL'
+                        ]
+                    ]
+                ],
+                'diff_currentValues' => [
+                    'tx_foo' => [
+                        'fields' => [
+                            'foo' => 'varchar(255) NULL'
+                        ]
+                    ]
+                ]
+            ]
+        );
+    }
+
+    /**
+     * @test
+     */
+    public function getDatabaseExtraFindsChangedFieldsIgnoreNotNull()
+    {
+        $subject = $this->getSqlSchemaMigrationService();
+        $differenceArray = $subject->getDatabaseExtra(
+            [
+                'tx_foo' => [
+                    'fields' => [
+                        'foo' => 'varchar(999) DEFAULT \'0\' NOT NULL'
+                    ]
+                ]
+            ],
+            [
+                'tx_foo' => [
+                    'fields' => [
+                        'foo' => 'varchar(255) DEFAULT \'0\' NOT NULL'
+                    ]
+                ]
+            ],
+            '',
+            true
+        );
+
+        $this->assertEquals(
+            $differenceArray,
+            [
+                'extra' => [],
+                'diff' => [
+                    'tx_foo' => [
+                        'fields' => [
+                            'foo' => 'varchar(999) DEFAULT \'0\''
+                        ]
+                    ]
+                ],
+                'diff_currentValues' => [
+                    'tx_foo' => [
+                        'fields' => [
+                            'foo' => 'varchar(255) DEFAULT \'0\''
+                        ]
+                    ]
+                ]
+            ]
+        );
+    }
+
+    /**
+     * @test
+     */
+    public function getDatabaseExtraIgnoresCaseDifference()
+    {
+        $subject = $this->getSqlSchemaMigrationService();
+        $differenceArray = $subject->getDatabaseExtra(
+            [
+                'tx_foo' => [
+                    'fields' => [
+                        'foo' => 'INT(11) DEFAULT \'0\' NOT NULL',
+                    ]
+                ]
+            ],
+            [
+                'tx_foo' => [
+                    'fields' => [
+                        'foo' => 'int(11) DEFAULT \'0\' NOT NULL',
+                    ]
+                ]
+            ]
+        );
+
+        $this->assertEquals(
+            $differenceArray,
+            [
+                'extra' => [],
+                'diff' => [],
+                'diff_currentValues' => null,
+            ]
+        );
+    }
+
+    /**
+     * @test
+     */
+    public function getDatabaseExtraIgnoresCaseDifferenceButKeepsCaseInSetIntact()
+    {
+        $subject = $this->getSqlSchemaMigrationService();
+        $differenceArray = $subject->getDatabaseExtra(
+            [
+                'tx_foo' => [
+                    'fields' => [
+                        'subtype' => 'SET(\'Tx_MyExt_Domain_Model_Xyz\',\'Tx_MyExt_Domain_Model_Abc\',\'\') NOT NULL DEFAULT \'\',',
+                    ]
+                ]
+            ],
+            [
+                'tx_foo' => [
+                    'fields' => [
+                        'subtype' => 'set(\'Tx_MyExt_Domain_Model_Xyz\',\'Tx_MyExt_Domain_Model_Abc\',\'\') NOT NULL DEFAULT \'\',',
+                    ]
+                ]
+            ]
+        );
+
+        $this->assertEquals(
+            $differenceArray,
+            [
+                'extra' => [],
+                'diff' => [],
+                'diff_currentValues' => null,
+            ]
+        );
+    }
+
+    /**
+     * @test
+     */
+    public function getDatabaseExtraDoesNotLowercaseReservedWordsForTheComparison()
+    {
+        $subject = $this->getSqlSchemaMigrationService();
+        $differenceArray = $subject->getDatabaseExtra(
+            [
+                'tx_foo' => [
+                    'fields' => [
+                        'PRIMARY KEY (md5hash)',
+                    ]
+                ]
+            ],
+            [
+                'tx_foo' => [
+                    'fields' => [
+                        'PRIMARY KEY (md5hash)'],
+                ]
+            ]
+        );
+
+        $this->assertEquals(
+            $differenceArray,
+            [
+                'extra' => [],
+                'diff' => [],
+                'diff_currentValues' => null,
+            ]
+        );
+    }
+
+    /**
+     * @test
+     */
+    public function getDatabaseExtraFindsNewSpatialKeys()
+    {
+        $subject = $this->getSqlSchemaMigrationService();
+        $differenceArray = $subject->getDatabaseExtra(
+            [
+                'tx_foo' => [
+                    'keys' => [
+                        'foo' => 'SPATIAL foo (foo)'
+                    ]
+                ]
+            ],
+            [
+                'tx_foo' => [
+                    'keys' => []
+                ]
+            ]
+        );
+
+        $this->assertEquals(
+            $differenceArray,
+            [
+                'extra' => [
+                    'tx_foo' => [
+                        'keys' => [
+                            'foo' => 'SPATIAL foo (foo)'
+                        ]
+                    ]
+                ],
+                'diff' => [],
+                'diff_currentValues' => null
+            ]
+        );
+    }
+
+    /**
+     * @test
+     */
+    public function checkColumnDefinitionIfCommentIsSupplied()
+    {
+        $subject = $this->getSqlSchemaMigrationService();
+        $fieldDefinition = $subject->assembleFieldDefinition(
+            [
+                'Field' => 'uid',
+                'Type' => 'int(11)',
+                'Null' => 'NO',
+                'Key' => 'PRI',
+                'Default' => null,
+                'Extra' => 'auto_increment',
+                'Comment' => 'I am a comment',
+            ]
+        );
+
+        $this->assertSame(
+            'int(11) NOT NULL auto_increment COMMENT \'I am a comment\'',
+            $fieldDefinition
+        );
+    }
+
+    /**
+     * @test
+     */
+    public function checkColumnDefinitionIfNoCommentIsSupplied()
+    {
+        $subject = $this->getSqlSchemaMigrationService();
+        $fieldDefinition = $subject->assembleFieldDefinition(
+            [
+                'Field' => 'uid',
+                'Type' => 'int(11)',
+                'Null' => 'NO',
+                'Key' => 'PRI',
+                'Default' => null,
+                'Extra' => 'auto_increment',
+            ]
+        );
+
+        $this->assertSame(
+            'int(11) NOT NULL auto_increment',
+            $fieldDefinition
+        );
+    }
+
+    /**
+     * @test
+     */
+    public function getDatabaseExtraIncludesEngineIfMySQLIsUsed()
+    {
+        $subject = $this->getSqlSchemaMigrationService();
+        $differenceArray = $subject->getDatabaseExtra(
+            [
+                'tx_foo' => [
+                    'fields' => [
+                        'foo' => 'INT(11) DEFAULT \'0\' NOT NULL',
+                    ],
+                    'extra' => [
+                        'ENGINE' => 'InnoDB'
+                    ]
+                ]
+            ],
+            [
+                'tx_foo' => [
+                    'fields' => [
+                        'foo' => 'int(11) DEFAULT \'0\' NOT NULL',
+                    ],
+                    'extra' => [
+                        'ENGINE' => 'InnoDB'
+                    ]
+                ]
+            ]
+        );
+
+        $this->assertSame(
+            $differenceArray,
+            [
+                'extra' => [],
+                'diff' => [],
+                'diff_currentValues' => null,
+            ]
+        );
+    }
+
+    /**
+     * @test
+     */
+    public function getDatabaseExtraExcludesEngineIfDbalIsUsed()
+    {
+        $subject = $this->getDbalEnabledSqlSchemaMigrationService();
+        $differenceArray = $subject->getDatabaseExtra(
+            [
+                'tx_foo' => [
+                    'fields' => [
+                        'foo' => 'INT(11) DEFAULT \'0\' NOT NULL',
+                    ],
+                    'extra' => [
+                        'ENGINE' => 'InnoDB'
+                    ]
+                ]
+            ],
+            [
+                'tx_foo' => [
+                    'fields' => [
+                        'foo' => 'int(11) DEFAULT \'0\' NOT NULL',
+                    ],
+                    'extra' => []
+                ]
+            ]
+        );
+
+        $this->assertSame(
+            $differenceArray,
+            [
+                'extra' => [],
+                'diff' => [],
+                'diff_currentValues' => null,
+            ]
+        );
+    }
+
+    /**
+     * @test
+     */
+    public function getDatabaseExtraIncludesUnsignedAttributeIfMySQLIsUsed()
+    {
+        $subject = $this->getSqlSchemaMigrationService();
+        $differenceArray = $subject->getDatabaseExtra(
+            [
+                'tx_foo' => [
+                    'fields' => [
+                        'foo' => 'INT(11) UNSIGNED DEFAULT \'0\' NOT NULL',
+                    ],
+                    'extra' => [
+                        'ENGINE' => 'InnoDB'
+                    ]
+                ]
+            ],
+            [
+                'tx_foo' => [
+                    'fields' => [
+                        'foo' => 'int(11) DEFAULT \'0\' NOT NULL',
+                    ],
+                    'extra' => [
+                        'ENGINE' => 'InnoDB'
+                    ]
+                ]
+            ]
+        );
+
+        $this->assertSame(
+            $differenceArray,
+            [
+                'extra' => [],
+                'diff' => [
+                    'tx_foo' => [
+                        'fields' => [
+                            'foo' => 'int(11) UNSIGNED DEFAULT \'0\' NOT NULL',
+                        ],
+                    ]
+                ],
+                'diff_currentValues' => [
+                    'tx_foo' => [
+                        'fields' => [
+                            'foo' => 'int(11) DEFAULT \'0\' NOT NULL',
+                        ],
+                    ]
+                ]
+            ]
+        );
+    }
+
+    /**
+     * @test
+     */
+    public function getDatabaseExtraExcludesUnsignedAttributeIfDbalIsUsed()
+    {
+        $subject = $this->getDbalEnabledSqlSchemaMigrationService();
+        $differenceArray = $subject->getDatabaseExtra(
+            [
+                'tx_foo' => [
+                    'fields' => [
+                        'foo' => 'INT(11) UNSIGNED DEFAULT \'0\' NOT NULL',
+                    ],
+                    'extra' => [
+                        'ENGINE' => 'InnoDB'
+                    ]
+                ]
+            ],
+            [
+                'tx_foo' => [
+                    'fields' => [
+                        'foo' => 'int(11) DEFAULT \'0\' NOT NULL',
+                    ],
+                    'extra' => [
+                        'ENGINE' => 'InnoDB'
+                    ]
+                ]
+            ]
+        );
+
+        $this->assertSame(
+            $differenceArray,
+            [
+                'extra' => [],
+                'diff' => [],
+                'diff_currentValues' => null
+            ]
+        );
+    }
+
+    /**
+     * @test
+     */
+    public function getDatabaseExtraIgnoresIndexPrefixLengthIfDbalIsUsed()
+    {
+        $subject = $this->getDbalEnabledSqlSchemaMigrationService();
+        $differenceArray = $subject->getDatabaseExtra(
+            [
+                'tx_foo' => [
+                    'keys' => [
+                        'foo' => 'KEY foo (foo(199))'
+                    ]
+                ]
+            ],
+            [
+                'tx_foo' => [
+                    'keys' => [
+                        'foo' => 'KEY foo (foo)'
+                    ]
+                ]
+            ]
+        );
+
+        $this->assertSame(
+            $differenceArray,
+            [
+                'extra' => [],
+                'diff' => [],
+                'diff_currentValues' => null,
+            ]
+        );
+    }
+
+    /**
+     * @test
+     */
+    public function getDatabaseExtraComparesIndexPrefixLengthIfMySQLIsUsed()
+    {
+        $subject = $this->getSqlSchemaMigrationService();
+        $differenceArray = $subject->getDatabaseExtra(
+            [
+                'tx_foo' => [
+                    'keys' => [
+                        'foo' => 'KEY foo (foo(199))'
+                    ]
+                ]
+            ],
+            [
+                'tx_foo' => [
+                    'keys' => [
+                        'foo' => 'KEY foo (foo)'
+                    ]
+                ]
+            ]
+        );
+
+        $this->assertSame(
+            $differenceArray,
+            [
+                'extra' => [],
+                'diff' => [
+                    'tx_foo' => [
+                        'keys' => [
+                            'foo' => 'KEY foo (foo(199))'
+                        ]
+                    ]
+                ],
+                'diff_currentValues' => [
+                    'tx_foo' => [
+                        'keys' => [
+                            'foo' => 'KEY foo (foo)'
+                        ]
+                    ]
+                ]
+            ]
+        );
+    }
+}
index 3181d22..abb47c1 100644 (file)
@@ -5,6 +5,10 @@ defined('TYPO3_MODE') or die();
 $GLOBALS['TYPO3_CONF_VARS']['SYS']['Objects'][\TYPO3\CMS\Core\Database\DatabaseConnection::class] = [
     'className' => \TYPO3\CMS\Dbal\Database\DatabaseConnection::class
 ];
+// xclass of SqlSchemaMigrationService handling dbal specific stuff
+$GLOBALS['TYPO3_CONF_VARS']['SYS']['Objects'][\TYPO3\CMS\Install\Service\SqlSchemaMigrationService::class] = [
+    'className' => \TYPO3\CMS\Dbal\Service\SqlSchemaMigrationService::class
+];
 
 // Register caches if not already done in localconf.php or a previously loaded extension.
 if (!is_array($GLOBALS['TYPO3_CONF_VARS']['SYS']['caching']['cacheConfigurations']['dbal'])) {
index 97a83ef..ac857a1 100644 (file)
@@ -410,10 +410,6 @@ class AbstractController
      */
     protected function loadBaseExtensions()
     {
-        if ($this->isDbalEnabled()) {
-            require(\TYPO3\CMS\Core\Utility\ExtensionManagementUtility::extPath('dbal') . 'ext_localconf.php');
-        }
-
         // @todo: Find out if this could be left out
         require(\TYPO3\CMS\Core\Utility\ExtensionManagementUtility::extPath('extbase') . 'ext_localconf.php');
 
@@ -428,28 +424,12 @@ class AbstractController
             }
             $cacheConfigurationsWithCachesSetToNullBackend[$cacheName] = $cacheConfiguration;
         }
-
         /** @var $cacheManager \TYPO3\CMS\Core\Cache\CacheManager */
         $cacheManager = GeneralUtility::makeInstance(\TYPO3\CMS\Core\Cache\CacheManager::class);
         $cacheManager->setCacheConfigurations($cacheConfigurationsWithCachesSetToNullBackend);
     }
 
     /**
-     * Return TRUE if dbal and adodb extension is loaded.
-     *
-     * @return bool TRUE if dbal and adodb is loaded
-     */
-    protected function isDbalEnabled()
-    {
-        if (\TYPO3\CMS\Core\Utility\ExtensionManagementUtility::isLoaded('adodb')
-            && \TYPO3\CMS\Core\Utility\ExtensionManagementUtility::isLoaded('dbal')
-        ) {
-            return true;
-        }
-        return false;
-    }
-
-    /**
      * Check given action name is one of the allowed actions.
      *
      * @param string $action Given action to validate
index cb7e0fb..5b3ef65 100644 (file)
@@ -14,6 +14,10 @@ namespace TYPO3\CMS\Install\Service;
  * The TYPO3 project - inspiring people to share!
  */
 
+use Doctrine\DBAL\DBALException;
+use TYPO3\CMS\Core\Database\ConnectionPool;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
 /**
  * Verify TYPO3 DB table structure. Mainly used in install tool
  * compare wizard and extension manager.
@@ -31,7 +35,7 @@ class SqlSchemaMigrationService
     protected $deletedPrefixKey = 'zzz_deleted_';
 
     /**
-     * @var array Caching output of $GLOBALS['TYPO3_DB']->admin_get_charsets()
+     * @var array Caching output "SHOW CHARACTER SET"
      */
     protected $character_sets = [];
 
@@ -63,7 +67,7 @@ class SqlSchemaMigrationService
      */
     public function getFieldDefinitions_fileContent($fileContent)
     {
-        $lines = \TYPO3\CMS\Core\Utility\GeneralUtility::trimExplode(LF, $fileContent, true);
+        $lines = GeneralUtility::trimExplode(LF, $fileContent, true);
         $table = '';
         $total = [];
         foreach ($lines as $value) {
@@ -72,7 +76,7 @@ class SqlSchemaMigrationService
                 continue;
             }
             if ($table === '') {
-                $parts = \TYPO3\CMS\Core\Utility\GeneralUtility::trimExplode(' ', $value, true);
+                $parts = GeneralUtility::trimExplode(' ', $value, true);
                 if (strtoupper($parts[0]) === 'CREATE' && strtoupper($parts[1]) === 'TABLE') {
                     $table = str_replace('`', '', $parts[2]);
                     // tablenames are always lowercase on windows!
@@ -96,7 +100,7 @@ class SqlSchemaMigrationService
                             // Note: Keywords "DEFAULT CHARSET" and "CHARSET" are the same, so "DEFAULT" can just be ignored
                             $charset = $tcharset[2];
                         } else {
-                            $charset = $this->getDatabaseConnection()->default_charset;
+                            $charset = 'utf8';
                         }
                         $total[$table]['extra']['COLLATE'] = $this->getCollationForCharset($charset);
                     }
@@ -157,12 +161,11 @@ class SqlSchemaMigrationService
     {
         // Load character sets, if not cached already
         if (empty($this->character_sets)) {
-            $databaseConnection = $this->getDatabaseConnection();
-            if (method_exists($databaseConnection, 'admin_get_charsets')) {
-                $this->character_sets = $databaseConnection->admin_get_charsets();
-            } else {
-                // Add empty element to avoid that the check will be repeated
-                $this->character_sets[$charset] = [];
+            $connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionByName('Default');
+            $statement = $connection->query('SHOW CHARACTER SET');
+            $this->character_sets = [];
+            while ($row = $statement->fetch()) {
+                $this->character_sets[$row['Charset']] = $row;
             }
         }
         $collation = '';
@@ -182,18 +185,28 @@ class SqlSchemaMigrationService
         $total = [];
         $tempKeys = [];
         $tempKeysPrefix = [];
-        $databaseConnection = $this->getDatabaseConnection();
-        $databaseConnection->connectDB();
-        echo $databaseConnection->sql_error();
-        $tables = $databaseConnection->admin_get_tables();
+        $connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionByName('Default');
+        $statement = $connection->query('SHOW TABLE STATUS FROM `' . $connection->getDatabase() . '`');
+        $tables = [];
+        while ($theTable = $statement->fetch()) {
+            $tables[$theTable['Name']] = $theTable;
+        }
         foreach ($tables as $tableName => $tableStatus) {
             // Fields
-            $fieldInformation = $databaseConnection->admin_get_fields($tableName);
+            $statement = $connection->query('SHOW FULL COLUMNS FROM `' . $tableName . '`');
+            $fieldInformation = [];
+            while ($fieldRow = $statement->fetch()) {
+                $fieldInformation[$fieldRow['Field']] = $fieldRow;
+            }
             foreach ($fieldInformation as $fN => $fieldRow) {
                 $total[$tableName]['fields'][$fN] = $this->assembleFieldDefinition($fieldRow);
             }
             // Keys
-            $keyInformation = $databaseConnection->admin_get_keys($tableName);
+            $statement = $connection->query('SHOW KEYS FROM `' . $tableName . '`');
+            $keyInformation = [];
+            while ($keyRow = $statement->fetch()) {
+                $keyInformation[] = $keyRow;
+            }
             foreach ($keyInformation as $keyRow) {
                 $keyName = $keyRow['Key_name'];
                 $colName = $keyRow['Column_name'];
@@ -258,7 +271,7 @@ class SqlSchemaMigrationService
         $diffArr = [];
         if (is_array($FDsrc)) {
             foreach ($FDsrc as $table => $info) {
-                if ($onlyTableList === '' || \TYPO3\CMS\Core\Utility\GeneralUtility::inList($onlyTableList, $table)) {
+                if ($onlyTableList === '' || GeneralUtility::inList($onlyTableList, $table)) {
                     if (!isset($FDcomp[$table])) {
                         // If the table was not in the FDcomp-array, the result array is loaded with that table.
                         $extraArr[$table] = $info;
@@ -269,9 +282,6 @@ class SqlSchemaMigrationService
                             if (is_array($info[$theKey])) {
                                 foreach ($info[$theKey] as $fieldN => $fieldC) {
                                     $fieldN = str_replace('`', '', $fieldN);
-                                    if ($this->isDbalEnabled() && $fieldN === 'ENGINE') {
-                                        continue;
-                                    }
                                     if ($fieldN == 'COLLATE') {
                                         // @todo collation support is currently disabled (needs more testing)
                                         continue;
@@ -294,25 +304,6 @@ class SqlSchemaMigrationService
                                             $fieldC
                                         );
 
-                                        if ($this->isDbalEnabled()) {
-                                            // Ignore nonstandard MySQL numeric field attributes UNSIGNED and ZEROFILL
-                                            if (preg_match('/^(TINYINT|SMALLINT|MEDIUMINT|INT|INTEGER|BIGINT|REAL|DOUBLE|FLOAT|DECIMAL|NUMERIC)\([^\)]+\)\s+(UNSIGNED|ZEROFILL)/i', $fieldC)) {
-                                                $fieldC = str_ireplace([' UNSIGNED', ' ZEROFILL'], '', $fieldC);
-                                                $FDcomp[$table][$theKey][$fieldN] = str_ireplace([' UNSIGNED', ' ZEROFILL'], '', $FDcomp[$table][$theKey][$fieldN]);
-                                            }
-
-                                            // Replace field and index definitions with functionally equivalent statements
-                                            if ($fieldC !== $FDcomp[$table][$theKey][$fieldN]) {
-                                                switch ($theKey) {
-                                                    case 'fields':
-                                                        $fieldC = $this->getDatabaseConnection()->getEquivalentFieldDefinition($fieldC);
-                                                        break;
-                                                    case 'keys':
-                                                        $fieldC = $this->getDatabaseConnection()->getEquivalentIndexDefinition($fieldC);
-                                                        break;
-                                                }
-                                            }
-                                        }
                                         if ($ignoreNotNullWhenComparing) {
                                             $fieldC = str_replace(' NOT NULL', '', $fieldC);
                                             $FDcomp[$table][$theKey][$fieldN] = str_replace(' NOT NULL', '', $FDcomp[$table][$theKey][$fieldN]);
@@ -377,11 +368,9 @@ class SqlSchemaMigrationService
                                     // The field can only be set "auto_increment" if there exists a PRIMARY key of that field already.
                                     // The check does not look up which field is primary but just assumes it must be the field with the auto_increment value...
                                     if (isset($info['keys']['PRIMARY'])) {
-                                        if (!$this->isDbalEnabled()) {
-                                            // Combine adding the field and the primary key into a single statement
-                                            $fV .= ', ADD PRIMARY KEY (' . $fN . ')';
-                                            unset($info['keys']['PRIMARY']);
-                                        }
+                                        // Combine adding the field and the primary key into a single statement
+                                        $fV .= ', ADD PRIMARY KEY (' . $fN . ')';
+                                        unset($info['keys']['PRIMARY']);
                                     } else {
                                         // In the next step, attempt to clear the table once again (2 = force)
                                         $info['extra']['CLEAR'] = 2;
@@ -474,7 +463,9 @@ class SqlSchemaMigrationService
                                 $statements['drop_table'][md5($statement)] = $statement;
                             }
                             // Count
-                            $count = $this->getDatabaseConnection()->exec_SELECTcountRows('*', $table);
+                            $count = GeneralUtility::makeInstance(ConnectionPool::class)
+                                ->getConnectionByName('Default')
+                                ->count('*', $table, []);
                             $statements['tables_count'][md5($statement)] = $count ? 'Records in table: ' . $count : '';
                         } else {
                             $statement = 'CREATE TABLE ' . $table . ' (
@@ -644,14 +635,13 @@ class SqlSchemaMigrationService
     {
         $result = [];
         if (is_array($arr)) {
-            $databaseConnection = $this->getDatabaseConnection();
+            $connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionByName('Default');
             foreach ($arr as $key => $string) {
                 if (isset($keyArr[$key]) && $keyArr[$key]) {
-                    $res = $databaseConnection->admin_query($string);
-                    if ($res === false) {
-                        $result[$key] = $databaseConnection->sql_error();
-                    } elseif (is_resource($res) || is_a($res, '\\mysqli_result')) {
-                        $databaseConnection->sql_free_result($res);
+                    try {
+                        $statement = $connection->query($string);
+                    } catch (DBALException $e) {
+                        $result[$key] = $e->getMessage();
                     }
                 }
             }
@@ -671,29 +661,16 @@ class SqlSchemaMigrationService
      */
     public function getListOfTables()
     {
-        $whichTables = $this->getDatabaseConnection()->admin_get_tables();
-        foreach ($whichTables as $key => &$value) {
+        $connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionByName('Default');
+        $statement = $connection->query('SHOW TABLE STATUS FROM `' . $connection->getDatabase() . '`');
+        $tables = [];
+        while ($theTable = $statement->fetch()) {
+            $tables[$theTable['Name']] = $theTable;
+        }
+        foreach ($tables as $key => &$value) {
             $value = $key;
         }
         unset($value);
-        return $whichTables;
-    }
-
-    /**
-     * Checks if DBAL is enabled for the database connection
-     *
-     * @return bool
-     */
-    protected function isDbalEnabled()
-    {
-        return \TYPO3\CMS\Core\Utility\ExtensionManagementUtility::isLoaded('dbal');
-    }
-
-    /**
-     * @return \TYPO3\CMS\Core\Database\DatabaseConnection|\TYPO3\CMS\Dbal\Database\DatabaseConnection
-     */
-    protected function getDatabaseConnection()
-    {
-        return $GLOBALS['TYPO3_DB'];
+        return $tables;
     }
 }
index be3fa99..930547b 100644 (file)
@@ -36,52 +36,6 @@ class SqlSchemaMigrationServiceTest extends \TYPO3\CMS\Core\Tests\UnitTestCase
     }
 
     /**
-     * Get a SchemaService instance with mocked DBAL enable database connection, DBAL enabled
-     *
-     * @return \PHPUnit_Framework_MockObject_MockObject|\TYPO3\CMS\Core\Tests\AccessibleObjectInterface
-     */
-    protected function getDbalEnabledSqlSchemaMigrationService()
-    {
-        /** @var \TYPO3\CMS\Dbal\Database\DatabaseConnection|\PHPUnit_Framework_MockObject_MockObject|\TYPO3\CMS\Core\Tests\AccessibleObjectInterface $databaseConnection */
-        $databaseConnection = $this->getAccessibleMock(\TYPO3\CMS\Dbal\Database\DatabaseConnection::class, ['dummy'], [], '', false);
-        $databaseConnection->_set(
-            'dbmsSpecifics',
-            new \TYPO3\CMS\Dbal\Database\Specifics\PostgresSpecifics()
-        );
-
-        $subject = $this->getAccessibleMock(SqlSchemaMigrationService::class, ['isDbalEnabled', 'getDatabaseConnection'], [], '', false);
-        $subject->expects($this->any())->method('isDbalEnabled')->will($this->returnValue(true));
-        $subject->expects($this->any())->method('getDatabaseConnection')->will($this->returnValue($databaseConnection));
-
-        return $subject;
-    }
-
-    /**
-     * @test
-     */
-    public function getFieldDefinitionsFileContentHandlesMultipleWhitespacesInFieldDefinitions()
-    {
-        $subject = $this->getSqlSchemaMigrationService();
-        // Multiple whitespaces and tabs in field definition
-        $inputString = 'CREATE table atable (' . LF . 'aFieldName   int(11)' . TAB . TAB . TAB . 'unsigned   DEFAULT \'0\'' . LF . ');';
-        $result = $subject->getFieldDefinitions_fileContent($inputString);
-
-        $this->assertEquals(
-            [
-                'atable' => [
-                    'fields' => [
-                        'aFieldName' => 'int(11) unsigned default \'0\'',
-                    ],
-                    'extra' => [
-                        'COLLATE' => '',
-                    ],
-                ],
-            ],
-            $result
-        );
-    }
-
-    /**
      * @test
      */
     public function getDatabaseExtraFindsChangedFields()
@@ -442,43 +396,6 @@ class SqlSchemaMigrationServiceTest extends \TYPO3\CMS\Core\Tests\UnitTestCase
     /**
      * @test
      */
-    public function getDatabaseExtraExcludesEngineIfDbalIsUsed()
-    {
-        $subject = $this->getDbalEnabledSqlSchemaMigrationService();
-        $differenceArray = $subject->getDatabaseExtra(
-            [
-                'tx_foo' => [
-                    'fields' => [
-                        'foo' => 'INT(11) DEFAULT \'0\' NOT NULL',
-                    ],
-                    'extra' => [
-                        'ENGINE' => 'InnoDB'
-                    ]
-                ]
-            ],
-            [
-                'tx_foo' => [
-                    'fields' => [
-                        'foo' => 'int(11) DEFAULT \'0\' NOT NULL',
-                    ],
-                    'extra' => []
-                ]
-            ]
-        );
-
-        $this->assertSame(
-            $differenceArray,
-            [
-                'extra' => [],
-                'diff' => [],
-                'diff_currentValues' => null,
-            ]
-        );
-    }
-
-    /**
-     * @test
-     */
     public function getDatabaseExtraIncludesUnsignedAttributeIfMySQLIsUsed()
     {
         $subject = $this->getSqlSchemaMigrationService();
@@ -530,78 +447,6 @@ class SqlSchemaMigrationServiceTest extends \TYPO3\CMS\Core\Tests\UnitTestCase
     /**
      * @test
      */
-    public function getDatabaseExtraExcludesUnsignedAttributeIfDbalIsUsed()
-    {
-        $subject = $this->getDbalEnabledSqlSchemaMigrationService();
-        $differenceArray = $subject->getDatabaseExtra(
-            [
-                'tx_foo' => [
-                    'fields' => [
-                        'foo' => 'INT(11) UNSIGNED DEFAULT \'0\' NOT NULL',
-                    ],
-                    'extra' => [
-                        'ENGINE' => 'InnoDB'
-                    ]
-                ]
-            ],
-            [
-                'tx_foo' => [
-                    'fields' => [
-                        'foo' => 'int(11) DEFAULT \'0\' NOT NULL',
-                    ],
-                    'extra' => [
-                        'ENGINE' => 'InnoDB'
-                    ]
-                ]
-            ]
-        );
-
-        $this->assertSame(
-            $differenceArray,
-            [
-                'extra' => [],
-                'diff' => [],
-                'diff_currentValues' => null
-            ]
-        );
-    }
-
-    /**
-     * @test
-     */
-    public function getDatabaseExtraIgnoresIndexPrefixLengthIfDbalIsUsed()
-    {
-        $subject = $this->getDbalEnabledSqlSchemaMigrationService();
-        $differenceArray = $subject->getDatabaseExtra(
-            [
-                'tx_foo' => [
-                    'keys' => [
-                        'foo' => 'KEY foo (foo(199))'
-                    ]
-                ]
-            ],
-            [
-                'tx_foo' => [
-                    'keys' => [
-                        'foo' => 'KEY foo (foo)'
-                    ]
-                ]
-            ]
-        );
-
-        $this->assertSame(
-            $differenceArray,
-            [
-                'extra' => [],
-                'diff' => [],
-                'diff_currentValues' => null,
-            ]
-        );
-    }
-
-    /**
-     * @test
-     */
     public function getDatabaseExtraComparesIndexPrefixLengthIfMySQLIsUsed()
     {
         $subject = $this->getSqlSchemaMigrationService();