[BUGFIX] Doctrine: Fix SchemaMigrator renaming columns instead of adding
[Packages/TYPO3.CMS.git] / typo3 / sysext / core / Classes / Database / Schema / SchemaMigrator.php
1 <?php
2 declare(strict_types=1);
3 namespace TYPO3\CMS\Core\Database\Schema;
4
5 /*
6 * This file is part of the TYPO3 CMS project.
7 *
8 * It is free software; you can redistribute it and/or modify it under
9 * the terms of the GNU General Public License, either version 2
10 * of the License, or any later version.
11 *
12 * For the full copyright and license information, please read the
13 * LICENSE.txt file that was distributed with this source code.
14 *
15 * The TYPO3 project - inspiring people to share!
16 */
17
18 use Doctrine\DBAL\DBALException;
19 use Doctrine\DBAL\Schema\Column;
20 use Doctrine\DBAL\Schema\ColumnDiff;
21 use Doctrine\DBAL\Schema\Comparator;
22 use Doctrine\DBAL\Schema\Schema;
23 use Doctrine\DBAL\Schema\SchemaConfig;
24 use Doctrine\DBAL\Schema\SchemaDiff;
25 use Doctrine\DBAL\Schema\Table;
26 use Doctrine\DBAL\Schema\TableDiff;
27 use TYPO3\CMS\Core\Database\Connection;
28 use TYPO3\CMS\Core\Database\ConnectionPool;
29 use TYPO3\CMS\Core\Database\Schema\Parser\Parser;
30 use TYPO3\CMS\Core\Utility\GeneralUtility;
31 use TYPO3\CMS\Core\Utility\StringUtility;
32
33 /**
34 * Helper methods to handle SQL files and transform them into individual statements
35 * for further processing.
36 *
37 * @internal
38 */
39 class SchemaMigrator
40 {
41 /**
42 * @var string Prefix of deleted tables
43 */
44 protected $deletedPrefix = 'zzz_deleted_';
45
46 /**
47 * @var Table[]
48 */
49 protected $tables = [];
50
51 /**
52 * @var Schema[]
53 */
54 protected $schema = [];
55
56 /**
57 * Compare current and expected schema definitions and provide updates suggestions in the form
58 * of SQL statements.
59 *
60 * @param string[] $statements The CREATE TABLE statements
61 * @param bool $remove TRUE for RENAME/DROP table and column suggestions, FALSE for ADD/CHANGE suggestions
62 * @return array[] SQL statements to migrate the database to the expected schema, indexed by performed operation
63 * @throws \Doctrine\DBAL\DBALException
64 * @throws \Doctrine\DBAL\Schema\SchemaException
65 * @throws \InvalidArgumentException
66 * @throws \RuntimeException
67 * @throws \TYPO3\CMS\Core\Database\Schema\Exception\UnexpectedSignalReturnValueTypeException
68 * @throws \TYPO3\CMS\Extbase\SignalSlot\Exception\InvalidSlotException
69 * @throws \TYPO3\CMS\Extbase\SignalSlot\Exception\InvalidSlotReturnException
70 * @throws \TYPO3\CMS\Core\Database\Schema\Exception\StatementException
71 */
72 public function getUpdateSuggestions(array $statements, bool $remove = false): array
73 {
74 $connectionPool = GeneralUtility::makeInstance(ConnectionPool::class);
75 $this->parseCreateTableStatements($statements);
76
77 $updateSuggestions = [];
78
79 foreach ($connectionPool->getConnectionNames() as $connectionName) {
80 $connection = $connectionPool->getConnectionByName($connectionName);
81
82 $schemaDiff = $this->buildSchemaDiff($connectionName);
83
84 if ($remove === false) {
85 $updateSuggestions[$connectionName] = array_merge_recursive(
86 ['add' => [], 'create_table' => [], 'change' => [], 'change_currentValue' => []],
87 $this->getNewFieldUpdateSuggestions($schemaDiff, $connection),
88 $this->getNewTableUpdateSuggestions($schemaDiff, $connection),
89 $this->getChangedFieldUpdateSuggestions($schemaDiff, $connection)
90 );
91 } else {
92 $updateSuggestions[$connectionName] = array_merge_recursive(
93 ['change' => [], 'change_table' => [], 'drop' => [], 'drop_table' => [], 'tables_count' => []],
94 $this->getUnusedFieldUpdateSuggestions($schemaDiff, $connection),
95 $this->getUnusedTableUpdateSuggestions($schemaDiff, $connection),
96 $this->getDropTableUpdateSuggestions($schemaDiff, $connection),
97 $this->getDropFieldUpdateSuggestions($schemaDiff, $connection)
98 );
99 }
100 }
101
102 return $updateSuggestions;
103 }
104
105 /**
106 * Return the raw Doctrine SchemaDiff objects for each connection. This diff contains
107 * all changes without any pre-processing.
108 *
109 * @param array $statements
110 * @return SchemaDiff[]
111 * @throws \Doctrine\DBAL\DBALException
112 * @throws \Doctrine\DBAL\Schema\SchemaException
113 * @throws \InvalidArgumentException
114 * @throws \RuntimeException
115 * @throws \TYPO3\CMS\Core\Database\Schema\Exception\UnexpectedSignalReturnValueTypeException
116 * @throws \TYPO3\CMS\Extbase\SignalSlot\Exception\InvalidSlotException
117 * @throws \TYPO3\CMS\Extbase\SignalSlot\Exception\InvalidSlotReturnException
118 * @throws \TYPO3\CMS\Core\Database\Schema\Exception\StatementException
119 */
120 public function getSchemaDiffs(array $statements): array
121 {
122 $connectionPool = GeneralUtility::makeInstance(ConnectionPool::class);
123 $this->parseCreateTableStatements($statements);
124
125 $schemaDiffs = [];
126
127 foreach ($connectionPool->getConnectionNames() as $connectionName) {
128 $schemaDiffs[$connectionName] = $this->buildSchemaDiff($connectionName, false);
129 }
130
131 return $schemaDiffs;
132 }
133
134 /**
135 * This method executes statements from the update suggestions, or a subset of them
136 * filtered by the statements hashes, one by one.
137 *
138 * @param string[] $statements The CREATE TABLE statements
139 * @param string[] $selectedStatements The hashes of the update suggestions to execute
140 * @return array
141 * @throws \Doctrine\DBAL\DBALException
142 * @throws \Doctrine\DBAL\Schema\SchemaException
143 * @throws \InvalidArgumentException
144 * @throws \TYPO3\CMS\Extbase\SignalSlot\Exception\InvalidSlotReturnException
145 * @throws \TYPO3\CMS\Extbase\SignalSlot\Exception\InvalidSlotException
146 * @throws \TYPO3\CMS\Core\Database\Schema\Exception\UnexpectedSignalReturnValueTypeException
147 * @throws \TYPO3\CMS\Core\Database\Schema\Exception\StatementException
148 * @throws \RuntimeException
149 */
150 public function migrate(array $statements, array $selectedStatements): array
151 {
152 $result = [];
153 $connectionPool = GeneralUtility::makeInstance(ConnectionPool::class);
154 $updateSuggestionsPerConnection = array_merge_recursive(
155 $this->getUpdateSuggestions($statements),
156 $this->getUpdateSuggestions($statements, true)
157 );
158
159 foreach ($updateSuggestionsPerConnection as $connectionName => $updateSuggestions) {
160 unset($updateSuggestions['tables_count'], $updateSuggestions['change_currentValue']);
161 $updateSuggestions = array_merge(...array_values($updateSuggestions));
162 $statementsToExecute = array_intersect_key($updateSuggestions, $selectedStatements);
163 if (count($statementsToExecute) === 0) {
164 continue;
165 }
166
167 $connection = $connectionPool->getConnectionByName($connectionName);
168 foreach ($statementsToExecute as $hash => $statement) {
169 try {
170 $connection->executeUpdate($statement);
171 } catch (DBALException $e) {
172 $result[$hash] = $e->getPrevious()->getMessage();
173 }
174 }
175 }
176
177 return $result;
178 }
179
180 /**
181 * Perform add/change/create operations on tables and fields in an optimized,
182 * non-interactive, mode using the original doctrine SchemaManager ->toSaveSql()
183 * method.
184 *
185 * @param string[] $statements The CREATE TABLE statements
186 * @param bool $createOnly Only perform changes that add fields or create tables
187 * @return array[] Error messages for statements that occurred during the installation procedure.
188 * @throws \Doctrine\DBAL\DBALException
189 * @throws \Doctrine\DBAL\Schema\SchemaException
190 * @throws \InvalidArgumentException
191 * @throws \RuntimeException
192 * @throws \TYPO3\CMS\Core\Database\Schema\Exception\UnexpectedSignalReturnValueTypeException
193 * @throws \TYPO3\CMS\Extbase\SignalSlot\Exception\InvalidSlotException
194 * @throws \TYPO3\CMS\Extbase\SignalSlot\Exception\InvalidSlotReturnException
195 * @throws \TYPO3\CMS\Core\Database\Schema\Exception\StatementException
196 */
197 public function install(array $statements, bool $createOnly = false): array
198 {
199 $connectionPool = GeneralUtility::makeInstance(ConnectionPool::class);
200 $this->parseCreateTableStatements($statements);
201 $result = [];
202
203 foreach ($connectionPool->getConnectionNames() as $connectionName) {
204 $connection = $connectionPool->getConnectionByName($connectionName);
205
206 $schemaDiff = $this->buildSchemaDiff($connectionName, false);
207
208 $schemaDiff->removedTables = [];
209 foreach ($schemaDiff->changedTables as $key => $changedTable) {
210 $schemaDiff->changedTables[$key]->removedColumns = [];
211 $schemaDiff->changedTables[$key]->removedIndexes = [];
212
213 // With partial ext_tables.sql files the SchemaManager is detecting
214 // existing columns as false positives for a column rename. In this
215 // context every rename is actually a new column.
216 foreach ($changedTable->renamedColumns as $columnName => $renamedColumn) {
217 $changedTable->addedColumns[$renamedColumn->getName()] = GeneralUtility::makeInstance(
218 Column::class,
219 $renamedColumn->getName(),
220 $renamedColumn->getType(),
221 array_diff_key($renamedColumn->toArray(), ['name', 'type'])
222 );
223 unset($changedTable->renamedColumns[$columnName]);
224 }
225
226 if ($createOnly) {
227 $schemaDiff->changedTables[$key]->changedColumns = [];
228 $schemaDiff->changedTables[$key]->renamedIndexes = [];
229 }
230 }
231
232 $statements = $schemaDiff->toSaveSql($connection->getDatabasePlatform());
233
234 foreach ($statements as $hash => $statement) {
235 try {
236 $connection->executeUpdate($statement);
237 $result[$statement] = '';
238 } catch (DBALException $e) {
239 $result[$statement] = $e->getPrevious()->getMessage();
240 }
241 }
242 }
243
244 return $result;
245 }
246
247 /**
248 * Import static data (INSERT statements)
249 *
250 * @param array $statements
251 * @param bool $truncate
252 * @return array
253 */
254 public function importStaticData(array $statements, bool $truncate = false): array
255 {
256 $result = [];
257 $connectionPool = GeneralUtility::makeInstance(ConnectionPool::class);
258 $insertStatements = [];
259
260 foreach ($statements as $statement) {
261 // Only handle insert statements and extract the table at the same time. Extracting
262 // the table name is required to perform the inserts on the right connection.
263 if (preg_match('/^INSERT\s+INTO\s+`?(\w+)`?(.*)/i', $statement, $matches)) {
264 list(, $tableName, $sqlFragment) = $matches;
265 $insertStatements[$tableName][] = sprintf(
266 'INSERT INTO %s %s',
267 $connectionPool->getConnectionForTable($tableName)->quoteIdentifier($tableName),
268 rtrim($sqlFragment, ';')
269 );
270 }
271 }
272
273 foreach ($insertStatements as $tableName => $perTableStatements) {
274 $connection = $connectionPool->getConnectionForTable($tableName);
275
276 if ($truncate) {
277 $connection->truncate($tableName);
278 }
279
280 foreach ((array)$perTableStatements as $statement) {
281 try {
282 $connection->executeUpdate($statement);
283 $result[$statement] = '';
284 } catch (DBALException $e) {
285 $result[$statement] = $e->getPrevious()->getMessage();
286 }
287 }
288 }
289
290 return $result;
291 }
292
293 /**
294 * If the schema is not for the Default connection remove all tables from the schema
295 * that have no mapping in the TYPO3 configuration. This avoids update suggestions
296 * for tables that are in the database but have no direct relation to the TYPO3 instance.
297 *
298 * @param string $connectionName
299 * @param bool $renameUnused
300 * @return \Doctrine\DBAL\Schema\SchemaDiff
301 * @throws \Doctrine\DBAL\DBALException
302 * @throws \InvalidArgumentException
303 */
304 protected function buildSchemaDiff(string $connectionName, bool $renameUnused = true): SchemaDiff
305 {
306 $connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionByName($connectionName);
307
308 // Build the schema definitions
309 $fromSchema = $connection->getSchemaManager()->createSchema();
310 $toSchema = $this->buildExpectedSchemaDefinitions($connectionName);
311
312 // Build SchemaDiff and handle renames of tables and colums
313 $comparator = GeneralUtility::makeInstance(Comparator::class);
314 $schemaDiff = $comparator->compare($fromSchema, $toSchema);
315
316 if ($renameUnused) {
317 $schemaDiff = $this->migrateUnprefixedRemovedTablesToRenames($schemaDiff);
318 $schemaDiff = $this->migrateUnprefixedRemovedFieldsToRenames($schemaDiff);
319 }
320
321 // All tables in the default connection are managed by TYPO3
322 if ($connectionName === ConnectionPool::DEFAULT_CONNECTION_NAME) {
323 return $schemaDiff;
324 }
325
326 // If there are no mapped tables return a SchemaDiff without any changes
327 // to avoid update suggestions for tables not related to TYPO3.
328 if (empty($GLOBALS['TYPO3_CONF_VARS']['DB']['TableMapping'])
329 || !is_array($GLOBALS['TYPO3_CONF_VARS']['DB']['TableMapping'])
330 ) {
331 $schemaDiff = GeneralUtility::makeInstance(SchemaDiff::class, [], [], [], $fromSchema);
332
333 return $schemaDiff;
334 }
335
336 // Collect the table names that have been mapped to this connection.
337 $tablesForConnection = array_keys(
338 array_filter(
339 $GLOBALS['TYPO3_CONF_VARS']['DB']['TableMapping'],
340 function ($tableConnectionName) use ($connectionName) {
341 return $tableConnectionName === $connectionName;
342 }
343 )
344 );
345
346 // Remove all tables that are not assigned to this connection from the diff
347 $schemaDiff->newTables = $this->removeUnrelatedTables($schemaDiff->newTables, $tablesForConnection);
348 $schemaDiff->changedTables = $this->removeUnrelatedTables($schemaDiff->changedTables, $tablesForConnection);
349 $schemaDiff->removedTables = $this->removeUnrelatedTables($schemaDiff->removedTables, $tablesForConnection);
350
351 return $schemaDiff;
352 }
353
354 /**
355 * Build the expected schema definitons from raw SQL statements.
356 *
357 * @param string $connectionName
358 * @return \Doctrine\DBAL\Schema\Schema
359 * @throws \Doctrine\DBAL\DBALException
360 * @throws \InvalidArgumentException
361 */
362 protected function buildExpectedSchemaDefinitions(string $connectionName): Schema
363 {
364 $connectionPool = GeneralUtility::makeInstance(ConnectionPool::class);
365 $connection = $connectionPool->getConnectionByName($connectionName);
366
367 $tablesForConnection = [];
368 foreach ($this->tables as $table) {
369 $tableName = $table->getName();
370
371 // Skip tables for a different connection
372 if ($connectionName !== $this->getConnectionNameForTable($tableName)) {
373 continue;
374 }
375
376 if (!array_key_exists($tableName, $tablesForConnection)) {
377 $tablesForConnection[$tableName] = $table;
378 continue;
379 }
380
381 // Merge multiple table definitions. Later definitions overrule identical
382 // columns, indexes and foreign_keys. Order of definitions is based on
383 // extension load order.
384 $currentTableDefinition = $tablesForConnection[$tableName];
385 $tablesForConnection[$tableName] = GeneralUtility::makeInstance(
386 Table::class,
387 $tableName,
388 array_merge($currentTableDefinition->getColumns(), $table->getColumns()),
389 array_merge($currentTableDefinition->getIndexes(), $table->getIndexes()),
390 array_merge($currentTableDefinition->getForeignKeys(), $table->getForeignKeys()),
391 0,
392 array_merge($currentTableDefinition->getOptions(), $table->getOptions())
393 );
394 }
395
396 $schemaConfig = GeneralUtility::makeInstance(SchemaConfig::class);
397 $schemaConfig->setName($connection->getDatabase());
398
399 return GeneralUtility::makeInstance(Schema::class, $tablesForConnection, [], $schemaConfig);
400 }
401
402 /**
403 * Parse CREATE TABLE statements into Doctrine Table objects.
404 *
405 * @param string[] $statements The SQL CREATE TABLE statements
406 * @throws \Doctrine\DBAL\Schema\SchemaException
407 * @throws \InvalidArgumentException
408 * @throws \RuntimeException
409 * @throws \TYPO3\CMS\Core\Database\Schema\Exception\StatementException
410 * @throws \TYPO3\CMS\Core\Database\Schema\Exception\UnexpectedSignalReturnValueTypeException
411 * @throws \TYPO3\CMS\Extbase\SignalSlot\Exception\InvalidSlotException
412 * @throws \TYPO3\CMS\Extbase\SignalSlot\Exception\InvalidSlotReturnException
413 */
414 protected function parseCreateTableStatements(array $statements)
415 {
416 $tables = [];
417 foreach ($statements as $statement) {
418 $createTableParser = GeneralUtility::makeInstance(Parser::class, $statement);
419
420 // We need to keep multiple table definitions at this point so
421 // that Extensions can modify existing tables.
422 $tables[] = $createTableParser->parse();
423 }
424
425 // Flatten the array of arrays by one level
426 $this->tables = array_merge(...$tables);
427 }
428
429 /**
430 * Extract the update suggestions (SQL statements) for newly added tables
431 * from the complete schema diff.
432 *
433 * @param \Doctrine\DBAL\Schema\SchemaDiff $schemaDiff
434 * @param \TYPO3\CMS\Core\Database\Connection $connection
435 * @return array
436 * @throws \InvalidArgumentException
437 */
438 protected function getNewTableUpdateSuggestions(SchemaDiff $schemaDiff, Connection $connection): array
439 {
440 // Build a new schema diff that only contains added tables
441 $addTableSchemaDiff = GeneralUtility::makeInstance(
442 SchemaDiff::class,
443 $schemaDiff->newTables,
444 [],
445 [],
446 $schemaDiff->fromSchema
447 );
448
449 $statements = $addTableSchemaDiff->toSql($connection->getDatabasePlatform());
450
451 return ['create_table' => $this->calculateUpdateSuggestionsHashes($statements)];
452 }
453
454 /**
455 * Extract the update suggestions (SQL statements) for newly added fields
456 * from the complete schema diff.
457 *
458 * @param \Doctrine\DBAL\Schema\SchemaDiff $schemaDiff
459 * @param \TYPO3\CMS\Core\Database\Connection $connection
460 * @return array
461 * @throws \Doctrine\DBAL\Schema\SchemaException
462 * @throws \InvalidArgumentException
463 */
464 protected function getNewFieldUpdateSuggestions(SchemaDiff $schemaDiff, Connection $connection): array
465 {
466 $changedTables = [];
467
468 foreach ($schemaDiff->changedTables as $index => $changedTable) {
469 if (count($changedTable->addedColumns) !== 0) {
470 // Treat each added column with a new diff to get a dedicated suggestions
471 // just for this single column.
472 foreach ($changedTable->addedColumns as $addedColumn) {
473 $changedTables[$index . ':tbl_' . $addedColumn->getName()] = GeneralUtility::makeInstance(
474 TableDiff::class,
475 $changedTable->name,
476 [$addedColumn],
477 [],
478 [],
479 [],
480 [],
481 [],
482 $schemaDiff->fromSchema->getTable($changedTable->name)
483 );
484 }
485 }
486
487 if (count($changedTable->addedIndexes) !== 0) {
488 // Treat each added index with a new diff to get a dedicated suggestions
489 // just for this index.
490 foreach ($changedTable->addedIndexes as $addedIndex) {
491 $changedTables[$index . ':idx_' . $addedIndex->getName()] = GeneralUtility::makeInstance(
492 TableDiff::class,
493 $changedTable->name,
494 [],
495 [],
496 [],
497 [$addedIndex],
498 [],
499 [],
500 $schemaDiff->fromSchema->getTable($changedTable->name)
501 );
502 }
503 }
504
505 if (count($changedTable->addedForeignKeys) !== 0) {
506 // Treat each added foreign key with a new diff to get a dedicated suggestions
507 // just for this foreign key.
508 foreach ($changedTable->addedForeignKeys as $addedForeignKey) {
509 $fkIndex = $index . ':fk_' . $addedForeignKey->getName();
510 $changedTables[$fkIndex] = GeneralUtility::makeInstance(
511 TableDiff::class,
512 $changedTable->name,
513 [],
514 [],
515 [],
516 [],
517 [],
518 [],
519 $schemaDiff->fromSchema->getTable($changedTable->name)
520 );
521 $changedTables[$fkIndex]->addedForeignKeys = [$addedForeignKey];
522 }
523 }
524 }
525
526 // Build a new schema diff that only contains added fields
527 $addFieldSchemaDiff = GeneralUtility::makeInstance(
528 SchemaDiff::class,
529 [],
530 $changedTables,
531 [],
532 $schemaDiff->fromSchema
533 );
534
535 $statements = $addFieldSchemaDiff->toSql($connection->getDatabasePlatform());
536
537 return ['add' => $this->calculateUpdateSuggestionsHashes($statements)];
538 }
539
540 /**
541 * Extract update suggestions (SQL statements) for changed fields
542 * from the complete schema diff.
543 *
544 * @param \Doctrine\DBAL\Schema\SchemaDiff $schemaDiff
545 * @param \TYPO3\CMS\Core\Database\Connection $connection
546 * @return array
547 * @throws \Doctrine\DBAL\Schema\SchemaException
548 * @throws \InvalidArgumentException
549 */
550 protected function getChangedFieldUpdateSuggestions(SchemaDiff $schemaDiff, Connection $connection): array
551 {
552 $databasePlatform = $connection->getDatabasePlatform();
553 $updateSuggestions = [];
554
555 foreach ($schemaDiff->changedTables as $index => $changedTable) {
556 if (count($changedTable->changedColumns) !== 0) {
557 // Treat each changed column with a new diff to get a dedicated suggestions
558 // just for this single column.
559 $fromTable = $schemaDiff->fromSchema->getTable($changedTable->name);
560 foreach ($changedTable->changedColumns as $changedColumn) {
561 // Field has been renamed and will be handled separately
562 if ($changedColumn->getOldColumnName()->getName() !== $changedColumn->column->getName()) {
563 continue;
564 }
565
566 // Get the current SQL declaration for the column
567 $currentColumn = $fromTable->getColumn($changedColumn->getOldColumnName()->getName());
568 $currentDeclaration = $databasePlatform->getColumnDeclarationSQL(
569 $currentColumn->getName(),
570 $currentColumn->toArray()
571 );
572
573 // Build a dedicated diff just for the current column
574 $tableDiff = GeneralUtility::makeInstance(
575 TableDiff::class,
576 $changedTable->name,
577 [],
578 [$changedColumn],
579 [],
580 [],
581 [],
582 [],
583 $schemaDiff->fromSchema->getTable($changedTable->name)
584 );
585
586 $temporarySchemaDiff = GeneralUtility::makeInstance(
587 SchemaDiff::class,
588 [],
589 [$tableDiff],
590 [],
591 $schemaDiff->fromSchema
592 );
593
594 $statements = $temporarySchemaDiff->toSql($databasePlatform);
595 foreach ($statements as $statement) {
596 $updateSuggestions['change'][md5($statement)] = $statement;
597 $updateSuggestions['change_currentValue'][md5($statement)] = $currentDeclaration;
598 }
599 }
600 }
601
602 // Treat each changed index with a new diff to get a dedicated suggestions
603 // just for this index.
604 if (count($changedTable->changedIndexes) !== 0) {
605 foreach ($changedTable->renamedIndexes as $key => $changedIndex) {
606 $indexDiff = GeneralUtility::makeInstance(
607 TableDiff::class,
608 $changedTable->name,
609 [],
610 [],
611 [],
612 [],
613 [$changedIndex],
614 [],
615 $schemaDiff->fromSchema->getTable($changedTable->name)
616 );
617
618 $temporarySchemaDiff = GeneralUtility::makeInstance(
619 SchemaDiff::class,
620 [],
621 [$indexDiff],
622 [],
623 $schemaDiff->fromSchema
624 );
625
626 $statements = $temporarySchemaDiff->toSql($databasePlatform);
627 foreach ($statements as $statement) {
628 $updateSuggestions['change'][md5($statement)] = $statement;
629 }
630 }
631 }
632
633 // Treat renamed indexes as a field change as it's a simple rename operation
634 if (count($changedTable->renamedIndexes) !== 0) {
635 // Create a base table diff without any changes, there's no constructor
636 // argument to pass in renamed columns.
637 $tableDiff = GeneralUtility::makeInstance(
638 TableDiff::class,
639 $changedTable->name,
640 [],
641 [],
642 [],
643 [],
644 [],
645 [],
646 $schemaDiff->fromSchema->getTable($changedTable->name)
647 );
648
649 // Treat each renamed index with a new diff to get a dedicated suggestions
650 // just for this index.
651 foreach ($changedTable->renamedIndexes as $key => $renamedIndex) {
652 $indexDiff = clone $tableDiff;
653 $indexDiff->renamedIndexes = [$key => $renamedIndex];
654
655 $temporarySchemaDiff = GeneralUtility::makeInstance(
656 SchemaDiff::class,
657 [],
658 [$indexDiff],
659 [],
660 $schemaDiff->fromSchema
661 );
662
663 $statements = $temporarySchemaDiff->toSql($databasePlatform);
664 foreach ($statements as $statement) {
665 $updateSuggestions['change'][md5($statement)] = $statement;
666 }
667 }
668 }
669
670 // Treat each changed foreign key with a new diff to get a dedicated suggestions
671 // just for this foreign key.
672 if (count($changedTable->changedForeignKeys) !== 0) {
673 $tableDiff = GeneralUtility::makeInstance(
674 TableDiff::class,
675 $changedTable->name,
676 [],
677 [],
678 [],
679 [],
680 [],
681 [],
682 $schemaDiff->fromSchema->getTable($changedTable->name)
683 );
684
685 foreach ($changedTable->changedForeignKeys as $changedForeignKey) {
686 $foreignKeyDiff = clone $tableDiff;
687 $foreignKeyDiff->changedForeignKeys = [$changedForeignKey];
688
689 $temporarySchemaDiff = GeneralUtility::makeInstance(
690 SchemaDiff::class,
691 [],
692 [$foreignKeyDiff],
693 [],
694 $schemaDiff->fromSchema
695 );
696
697 $statements = $temporarySchemaDiff->toSql($databasePlatform);
698 foreach ($statements as $statement) {
699 $updateSuggestions['change'][md5($statement)] = $statement;
700 }
701 }
702 }
703 }
704
705 return $updateSuggestions;
706 }
707
708 /**
709 * Extract update suggestions (SQL statements) for tables that are
710 * no longer present in the expected schema from the schema diff.
711 * In this case the update suggestions are renames of the tables
712 * with a prefix to mark them for deletion in a second sweep.
713 *
714 * @param \Doctrine\DBAL\Schema\SchemaDiff $schemaDiff
715 * @param \TYPO3\CMS\Core\Database\Connection $connection
716 * @return array
717 * @throws \Doctrine\DBAL\Schema\SchemaException
718 * @throws \InvalidArgumentException
719 */
720 protected function getUnusedTableUpdateSuggestions(SchemaDiff $schemaDiff, Connection $connection): array
721 {
722 $updateSuggestions = [];
723 foreach ($schemaDiff->changedTables as $tableDiff) {
724 // Skip tables that are not being renamed or where the new name isn't prefixed
725 // with the deletion marker.
726 if ($tableDiff->getNewName() === false
727 || !StringUtility::beginsWith($tableDiff->getNewName()->getName(), $this->deletedPrefix)
728 ) {
729 continue;
730 }
731 // Build a new schema diff that only contains this table
732 $changedFieldDiff = GeneralUtility::makeInstance(
733 SchemaDiff::class,
734 [],
735 [$tableDiff],
736 [],
737 $schemaDiff->fromSchema
738 );
739
740 $statements = $changedFieldDiff->toSql($connection->getDatabasePlatform());
741
742 foreach ($statements as $statement) {
743 $updateSuggestions['change_table'][md5($statement)] = $statement;
744 }
745 $updateSuggestions['tables_count'][md5($statements[0])] = $this->getTableRecordCount($tableDiff->name);
746 }
747
748 return $updateSuggestions;
749 }
750
751 /**
752 * Extract update suggestions (SQL statements) for fields that are
753 * no longer present in the expected schema from the schema diff.
754 * In this case the update suggestions are renames of the fields
755 * with a prefix to mark them for deletion in a second sweep.
756 *
757 * @param \Doctrine\DBAL\Schema\SchemaDiff $schemaDiff
758 * @param \TYPO3\CMS\Core\Database\Connection $connection
759 * @return array
760 * @throws \Doctrine\DBAL\Schema\SchemaException
761 * @throws \InvalidArgumentException
762 */
763 protected function getUnusedFieldUpdateSuggestions(SchemaDiff $schemaDiff, Connection $connection): array
764 {
765 $changedTables = [];
766
767 foreach ($schemaDiff->changedTables as $index => $changedTable) {
768 if (count($changedTable->changedColumns) === 0) {
769 continue;
770 }
771
772 // Treat each changed column with a new diff to get a dedicated suggestions
773 // just for this single column.
774 foreach ($changedTable->changedColumns as $changedColumn) {
775 // Field has not been renamed
776 if ($changedColumn->getOldColumnName()->getName() === $changedColumn->column->getName()) {
777 continue;
778 }
779
780 $changedTables[$index . ':' . $changedColumn->column->getName()] = GeneralUtility::makeInstance(
781 TableDiff::class,
782 $changedTable->name,
783 [],
784 [$changedColumn],
785 [],
786 [],
787 [],
788 [],
789 $schemaDiff->fromSchema->getTable($changedTable->name)
790 );
791 }
792 }
793
794 // Build a new schema diff that only contains unused fields
795 $changedFieldDiff = GeneralUtility::makeInstance(
796 SchemaDiff::class,
797 [],
798 $changedTables,
799 [],
800 $schemaDiff->fromSchema
801 );
802
803 $statements = $changedFieldDiff->toSql($connection->getDatabasePlatform());
804
805 return ['change' => $this->calculateUpdateSuggestionsHashes($statements)];
806 }
807
808 /**
809 * Extract update suggestions (SQL statements) for fields that can
810 * be removed from the complete schema diff.
811 * Fields that can be removed have been prefixed in a previous run
812 * of the schema migration.
813 *
814 * @param \Doctrine\DBAL\Schema\SchemaDiff $schemaDiff
815 * @param \TYPO3\CMS\Core\Database\Connection $connection
816 * @return array
817 * @throws \Doctrine\DBAL\Schema\SchemaException
818 * @throws \InvalidArgumentException
819 */
820 protected function getDropFieldUpdateSuggestions(SchemaDiff $schemaDiff, Connection $connection): array
821 {
822 $changedTables = [];
823
824 foreach ($schemaDiff->changedTables as $index => $changedTable) {
825 if (count($changedTable->removedColumns) !== 0) {
826 // Treat each changed column with a new diff to get a dedicated suggestions
827 // just for this single column.
828 foreach ($changedTable->removedColumns as $removedColumn) {
829 $changedTables[$index . ':tbl_' . $removedColumn->getName()] = GeneralUtility::makeInstance(
830 TableDiff::class,
831 $changedTable->name,
832 [],
833 [],
834 [$removedColumn],
835 [],
836 [],
837 [],
838 $schemaDiff->fromSchema->getTable($changedTable->name)
839 );
840 }
841 }
842
843 if (count($changedTable->removedIndexes) !== 0) {
844 // Treat each removed index with a new diff to get a dedicated suggestions
845 // just for this index.
846 foreach ($changedTable->removedIndexes as $removedIndex) {
847 $changedTables[$index . ':idx_' . $removedIndex->getName()] = GeneralUtility::makeInstance(
848 TableDiff::class,
849 $changedTable->name,
850 [],
851 [],
852 [],
853 [],
854 [],
855 [$removedIndex],
856 $schemaDiff->fromSchema->getTable($changedTable->name)
857 );
858 }
859 }
860
861 if (count($changedTable->removedForeignKeys) !== 0) {
862 // Treat each removed foreign key with a new diff to get a dedicated suggestions
863 // just for this foreign key.
864 foreach ($changedTable->removedForeignKeys as $removedForeignKey) {
865 $fkIndex = $index . ':fk_' . $removedForeignKey->getName();
866 $changedTables[$fkIndex] = GeneralUtility::makeInstance(
867 TableDiff::class,
868 $changedTable->name,
869 [],
870 [],
871 [],
872 [],
873 [],
874 [],
875 $schemaDiff->fromSchema->getTable($changedTable->name)
876 );
877 $changedTables[$fkIndex]->removedForeignKeys = [$removedForeignKey];
878 }
879 }
880 }
881
882 // Build a new schema diff that only contains removable fields
883 $removedFieldDiff = GeneralUtility::makeInstance(
884 SchemaDiff::class,
885 [],
886 $changedTables,
887 [],
888 $schemaDiff->fromSchema
889 );
890
891 $statements = $removedFieldDiff->toSql($connection->getDatabasePlatform());
892
893 return ['drop' => $this->calculateUpdateSuggestionsHashes($statements)];
894 }
895
896 /**
897 * Extract update suggestions (SQL statements) for tables that can
898 * be removed from the complete schema diff.
899 * Tables that can be removed have been prefixed in a previous run
900 * of the schema migration.
901 *
902 * @param \Doctrine\DBAL\Schema\SchemaDiff $schemaDiff
903 * @param \TYPO3\CMS\Core\Database\Connection $connection
904 * @return array
905 * @throws \Doctrine\DBAL\Schema\SchemaException
906 * @throws \InvalidArgumentException
907 */
908 protected function getDropTableUpdateSuggestions(SchemaDiff $schemaDiff, Connection $connection): array
909 {
910 $updateSuggestions = [];
911 foreach ($schemaDiff->removedTables as $removedTable) {
912 // Build a new schema diff that only contains this table
913 $tableDiff = GeneralUtility::makeInstance(
914 SchemaDiff::class,
915 [],
916 [],
917 [$removedTable],
918 $schemaDiff->fromSchema
919 );
920
921 $statements = $tableDiff->toSql($connection->getDatabasePlatform());
922 foreach ($statements as $statement) {
923 $updateSuggestions['drop_table'][md5($statement)] = $statement;
924 }
925
926 // Only store the record count for this table for the first statement,
927 // assuming that this is the actual DROP TABLE statement.
928 $updateSuggestions['tables_count'][md5($statements[0])] = $this->getTableRecordCount(
929 $removedTable->getName()
930 );
931 }
932
933 return $updateSuggestions;
934 }
935
936 /**
937 * Move tables to be removed that are not prefixed with the deleted prefix to the list
938 * of changed tables and set a new prefixed name.
939 * Without this help the Doctrine SchemaDiff has no idea if a table has been renamed and
940 * performs a drop of the old table and creates a new table, which leads to all data in
941 * the old table being lost.
942 *
943 * @param \Doctrine\DBAL\Schema\SchemaDiff $schemaDiff
944 * @return \Doctrine\DBAL\Schema\SchemaDiff
945 * @throws \InvalidArgumentException
946 */
947 protected function migrateUnprefixedRemovedTablesToRenames(SchemaDiff $schemaDiff): SchemaDiff
948 {
949 foreach ($schemaDiff->removedTables as $index => $removedTable) {
950 if (StringUtility::beginsWith($removedTable->getName(), $this->deletedPrefix)) {
951 continue;
952 }
953 $tableDiff = GeneralUtility::makeInstance(
954 TableDiff::class,
955 $removedTable->getName(),
956 $addedColumns = [],
957 $changedColumns = [],
958 $removedColumns = [],
959 $addedIndexes = [],
960 $changedIndexes = [],
961 $removedIndexes = [],
962 $fromTable = $removedTable
963 );
964
965 $tableDiff->newName = $this->deletedPrefix . $removedTable->getName();
966 $schemaDiff->changedTables[$index] = $tableDiff;
967 unset($schemaDiff->removedTables[$index]);
968 }
969
970 return $schemaDiff;
971 }
972
973 /**
974 * Scan the list of changed tables for fields that are going to be dropped. If
975 * the name of the field does not start with the deleted prefix mark the column
976 * for a rename instead of a drop operation.
977 *
978 * @param \Doctrine\DBAL\Schema\SchemaDiff $schemaDiff
979 * @return \Doctrine\DBAL\Schema\SchemaDiff
980 * @throws \InvalidArgumentException
981 */
982 protected function migrateUnprefixedRemovedFieldsToRenames(SchemaDiff $schemaDiff): SchemaDiff
983 {
984 foreach ($schemaDiff->changedTables as $tableIndex => $changedTable) {
985 if (count($changedTable->removedColumns) === 0) {
986 continue;
987 }
988
989 foreach ($changedTable->removedColumns as $columnIndex => $removedColumn) {
990 if (StringUtility::beginsWith($removedColumn->getName(), $this->deletedPrefix)) {
991 continue;
992 }
993
994 // Build a new column object with the same properties as the removed column
995 $renamedColumn = new Column(
996 $this->deletedPrefix . $removedColumn->getName(),
997 $removedColumn->getType(),
998 array_diff_key($removedColumn->toArray(), ['name', 'type'])
999 );
1000
1001 // Build the diff object for the column to rename
1002 $columnDiff = GeneralUtility::makeInstance(
1003 ColumnDiff::class,
1004 $removedColumn->getName(),
1005 $renamedColumn,
1006 $changedProperties = [],
1007 $removedColumn
1008 );
1009
1010 // Add the column with the required rename information to the changed column list
1011 $schemaDiff->changedTables[$tableIndex]->changedColumns[$columnIndex] = $columnDiff;
1012
1013 // Remove the column from the list of columns to be dropped
1014 unset($schemaDiff->changedTables[$tableIndex]->removedColumns[$columnIndex]);
1015 }
1016 }
1017
1018 return $schemaDiff;
1019 }
1020
1021 /**
1022 * Return the amount of records in the given table.
1023 *
1024 * @param string $tableName
1025 * @return int
1026 * @throws \InvalidArgumentException
1027 */
1028 protected function getTableRecordCount(string $tableName): int
1029 {
1030 return GeneralUtility::makeInstance(ConnectionPool::class)
1031 ->getConnectionForTable($tableName)
1032 ->count('*', $tableName, []);
1033 }
1034
1035 /**
1036 * Determine the connection name for a table
1037 *
1038 * @param string $tableName
1039 * @return string
1040 * @throws \InvalidArgumentException
1041 */
1042 protected function getConnectionNameForTable(string $tableName): string
1043 {
1044 $connectionNames = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionNames();
1045
1046 if (array_key_exists($tableName, (array)$GLOBALS['TYPO3_CONF_VARS']['DB']['TableMapping'])) {
1047 return in_array($GLOBALS['TYPO3_CONF_VARS']['DB']['TableMapping'][$tableName], $connectionNames, true)
1048 ? $GLOBALS['TYPO3_CONF_VARS']['DB']['TableMapping'][$tableName]
1049 : ConnectionPool::DEFAULT_CONNECTION_NAME;
1050 }
1051
1052 return ConnectionPool::DEFAULT_CONNECTION_NAME;
1053 }
1054
1055 /**
1056 * Replace the array keys with a md5 sum of the actual SQL statement
1057 *
1058 * @param string[] $statements
1059 * @return string[]
1060 */
1061 protected function calculateUpdateSuggestionsHashes(array $statements): array
1062 {
1063 return array_combine(array_map('md5', $statements), $statements);
1064 }
1065
1066 /**
1067 * Helper for buildSchemaDiff to filter an array of TableDiffs against a list of valid table names.
1068 *
1069 * @param TableDiff[]|Table[] $tableDiffs
1070 * @param string[] $validTableNames
1071 * @return TableDiff[]
1072 * @throws \InvalidArgumentException
1073 */
1074 protected function removeUnrelatedTables(array $tableDiffs, array $validTableNames): array
1075 {
1076 return array_filter(
1077 $tableDiffs,
1078 function ($table) use ($validTableNames) {
1079 if ($table instanceof Table) {
1080 $tableName = $table->getName();
1081 } else {
1082 $tableName = $table->newName ?: $table->name;
1083 }
1084
1085 // If the tablename has a deleted prefix strip it of before comparing
1086 // it against the list of valid table names so that drop operations
1087 // don't get removed.
1088 if (StringUtility::beginsWith($tableName, $this->deletedPrefix)) {
1089 $tableName = substr($tableName, strlen($this->deletedPrefix));
1090 }
1091 return in_array($tableName, $validTableNames, true)
1092 || in_array($this->deletedPrefix . $tableName, $validTableNames, true);
1093 }
1094 );
1095 }
1096 }