[!!!][TASK] Migrate OrphanRecords Command to Symfony Console 38/50438/7
authorBenni Mack <benni@typo3.org>
Sat, 29 Oct 2016 21:37:29 +0000 (23:37 +0200)
committerAnja Leichsenring <aleichsenring@ab-softlab.de>
Sat, 5 Nov 2016 09:25:20 +0000 (10:25 +0100)
The CLI cleaner command "orphan_records" in EXT:lowlevel is
migrated to a command based on Symfony Console.

Previoulsy it was called via
typo3/cli_dispatch.phpsh lowlevel_cleaner orphan_records -s -r

Now you use
typo3/sysext/core/bin/typo3 cleanup:orphanrecords
(optional via --dry-run)

Resolves: #78520
Releases: master
Change-Id: I1e1462ce45e76964d0b67bd782e5f216b3d2cf37
Reviewed-on: https://review.typo3.org/50438
Tested-by: TYPO3com <no-reply@typo3.com>
Reviewed-by: Wouter Wolters <typo3@wouterwolters.nl>
Tested-by: Wouter Wolters <typo3@wouterwolters.nl>
Reviewed-by: Anja Leichsenring <aleichsenring@ab-softlab.de>
Tested-by: Anja Leichsenring <aleichsenring@ab-softlab.de>
typo3/sysext/core/Documentation/Changelog/master/Breaking-78520-LowlevelOrphanRecordsCleaningParametersChanged.rst [new file with mode: 0644]
typo3/sysext/lowlevel/Classes/Command/OrphanRecordsCommand.php [new file with mode: 0644]
typo3/sysext/lowlevel/Classes/OrphanRecordsCommand.php [deleted file]
typo3/sysext/lowlevel/Configuration/Commands.php
typo3/sysext/lowlevel/ext_localconf.php

diff --git a/typo3/sysext/core/Documentation/Changelog/master/Breaking-78520-LowlevelOrphanRecordsCleaningParametersChanged.rst b/typo3/sysext/core/Documentation/Changelog/master/Breaking-78520-LowlevelOrphanRecordsCleaningParametersChanged.rst
new file mode 100644 (file)
index 0000000..c675991
--- /dev/null
@@ -0,0 +1,43 @@
+.. include:: ../../Includes.txt
+
+======================================================================
+Breaking: #78520 - Lowlevel Orphan Records Cleaning parameters changed
+======================================================================
+
+See :issue:`78520`
+
+Description
+===========
+
+The OrphanRecordsCommand is now using Symfony Console. The new command behaves like the old functionality,
+but uses certain different parameters. It can now be called with the following CLI command:
+
+`./typo3/sysext/core/bin/typo3 cleanup:orphanrecords`
+
+The following options can be set
+`--dry-run` to only show the orphaned records
+`-v` and `-vv` to show additional information
+
+The PHP class `TYPO3\CMS\Lowlevel\OrphanRecordsCommand` has been removed.
+
+
+Impact
+======
+
+Calling `typo3/cli_dispatch.phpsh lowlevel cleaner orphan_records` will not work anymore.
+
+Calling the PHP class results in a fatal PHP error.
+
+
+Affected Installations
+======================
+
+Any TYPO3 installation using the previously command callable via `cli_dispatch.phpsh` or the related PHP class.
+
+
+Migration
+=========
+
+Use the new CLI command as shown above.
+
+.. index:: CLI
diff --git a/typo3/sysext/lowlevel/Classes/Command/OrphanRecordsCommand.php b/typo3/sysext/lowlevel/Classes/Command/OrphanRecordsCommand.php
new file mode 100644 (file)
index 0000000..4139cc1
--- /dev/null
@@ -0,0 +1,292 @@
+<?php
+declare(strict_types=1);
+namespace TYPO3\CMS\Lowlevel\Command;
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Style\SymfonyStyle;
+use TYPO3\CMS\Backend\Utility\BackendUtility;
+use TYPO3\CMS\Core\Authentication\BackendUserAuthentication;
+use TYPO3\CMS\Core\Database\Connection;
+use TYPO3\CMS\Core\Database\ConnectionPool;
+use TYPO3\CMS\Core\DataHandling\DataHandler;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
+/**
+ * Finds (and fixes) all records that have an invalid / deleted page ID
+ */
+class OrphanRecordsCommand extends Command
+{
+
+    /**
+     * Configure the command by defining the name, options and arguments
+     */
+    public function configure()
+    {
+        $this
+            ->setDescription('Find and delete records that have lost their connection with the page tree.')
+            ->setHelp('Assumption: All actively used records on the website from TCA configured tables are located in the page tree exclusively.
+
+All records managed by TYPO3 via the TCA array configuration has to belong to a page in the page tree, either directly or indirectly as a version of another record.
+VERY TIME, CPU and MEMORY intensive operation since the full page tree is looked up!
+
+Automatic Repair of Errors:
+- Silently deleting the orphaned records. In theory they should not be used anywhere in the system, but there could be references. See below for more details on this matter.
+
+Manual repair suggestions:
+- Possibly re-connect orphaned records to page tree by setting their "pid" field to a valid page id. A lookup in the sys_refindex table can reveal if there are references to a orphaned record. If there are such references (from records that are not themselves orphans) you might consider to re-connect the record to the page tree, otherwise it should be safe to delete it.
+
+ If you want to get more detailed information, use the --verbose option.')
+            ->addOption(
+                'dry-run',
+                null,
+                InputOption::VALUE_NONE,
+                'If this option is set, the records will not actually be deleted, but just the output which records would be deleted are shown'
+            );
+    }
+
+    /**
+     * Executes the command to find records not attached to the pagetree
+     * and permanently delete these records
+     *
+     * @param InputInterface $input
+     * @param OutputInterface $output
+     *
+     * @return void
+     */
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        // The backend user needs super-powers because datahandler is executed
+        $previouslyAppliedAdminRights = $this->getBackendUser()->user['admin'];
+        $this->getBackendUser()->user['admin'] = 1;
+
+        $io = new SymfonyStyle($input, $output);
+        $io->title($this->getDescription());
+
+        if ($io->isVerbose()) {
+            $io->section('Searching the database now for orphaned records.');
+        }
+
+        // type unsafe comparison and explicit boolean setting on purpose
+        $dryRun = $input->hasOption('dry-run') && $input->getOption('dry-run') != false ? true : false;
+
+        // find all records that should be deleted
+        $allRecords = $this->findAllConnectedRecordsInPage(0, 10000);
+
+        // Find orphans
+        $orphans = [];
+        foreach (array_keys($GLOBALS['TCA']) as $tableName) {
+            $idList = [0];
+            if (is_array($allRecords[$tableName]) && !empty($allRecords[$tableName])) {
+                $idList = $allRecords[$tableName];
+            }
+            // Select all records that are NOT connected
+            $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
+                ->getQueryBuilderForTable($tableName);
+
+            $result = $queryBuilder
+                ->select('uid')
+                ->from($tableName)
+                ->where(
+                    $queryBuilder->expr()->notIn(
+                        'uid',
+                        $queryBuilder->createNamedParameter($idList, Connection::PARAM_INT_ARRAY)
+                    )
+                )
+                ->orderBy('uid')
+                ->execute();
+
+            $totalOrphans = 0;
+            if ($result->rowCount()) {
+                $orphans[$tableName] = [];
+                while ($orphanRecord = $result->fetch()) {
+                    $orphans[$tableName][$orphanRecord['uid']] = $orphanRecord['uid'];
+                }
+                $totalOrphans += count($orphans[$tableName]);
+
+                if ($io->isVeryVerbose() && count($orphans[$tableName])) {
+                    $io->writeln('Found ' . count($orphans[$tableName]) . ' orphan records in table "' . $tableName . '".');
+                }
+            }
+            if (!$io->isQuiet() && $totalOrphans) {
+                $io->note('Found ' . $totalOrphans . ' records in ' . count($orphans) . ' database tables.');
+            }
+        }
+
+        if (count($orphans)) {
+            $io->section('Deletion process starting now.' . ($dryRun ? ' (Not deleting now, just a dry run)' : ''));
+
+            // Actually permanently delete them
+            $this->deleteRecords($orphans, $dryRun, $io);
+
+            $io->success('All done!');
+        } else {
+            $io->success('No orphan records found.');
+        }
+
+        // Restore backend user administration rights
+        $this->getBackendUser()->user['admin'] = $previouslyAppliedAdminRights;
+    }
+
+    /**
+     * Recursive traversal of page tree to fetch all records marekd as "deleted",
+     * via option $GLOBALS[TCA][$tableName][ctrl][delete]
+     * This also takes deleted versioned records into account.
+     *
+     * @param int $pageId the uid of the pages record (can also be 0)
+     * @param int $depth The current depth of levels to go down
+     * @param array $allRecords the records that are already marked as deleted (used when going recursive)
+     *
+     * @return array the modified $deletedRecords array
+     */
+    protected function findAllConnectedRecordsInPage(int $pageId, int $depth, array $allRecords = []): array
+    {
+        // Register page
+        if ($pageId > 0) {
+            $allRecords['pages'][$pageId] = $pageId;
+        }
+        // Traverse tables of records that belongs to page
+        foreach (array_keys($GLOBALS['TCA']) as $tableName) {
+            if ($tableName !== 'pages') {
+                // Select all records belonging to page:
+                $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
+                    ->getQueryBuilderForTable($tableName);
+
+                $queryBuilder->getRestrictions()->removeAll();
+
+                $result = $queryBuilder
+                    ->select('uid')
+                    ->from($tableName)
+                    ->where(
+                        $queryBuilder->expr()->eq(
+                            'pid',
+                            $queryBuilder->createNamedParameter($pageId, \PDO::PARAM_INT)
+                        )
+                    )
+                    ->execute();
+
+                while ($rowSub = $result->fetch()) {
+                    $allRecords[$tableName][$rowSub['uid']] = $rowSub['uid'];
+                    // Add any versions of those records:
+                    $versions = BackendUtility::selectVersionsOfRecord($tableName, $rowSub['uid'], 'uid,t3ver_wsid,t3ver_count', null, true);
+                    if (is_array($versions)) {
+                        foreach ($versions as $verRec) {
+                            if (!$verRec['_CURRENT_VERSION']) {
+                                $allRecords[$tableName][$verRec['uid']] = $verRec['uid'];
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        // Find subpages to root ID and traverse (only when rootID is not a version or is a branch-version):
+        if ($depth > 0) {
+            $depth--;
+            $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
+                ->getQueryBuilderForTable('pages');
+
+            $queryBuilder->getRestrictions()->removeAll();
+
+            $result = $queryBuilder
+                ->select('uid')
+                ->from('pages')
+                ->where(
+                    $queryBuilder->expr()->eq(
+                        'pid',
+                        $queryBuilder->createNamedParameter($pageId, \PDO::PARAM_INT)
+                    )
+                )
+                ->orderBy('sorting')
+                ->execute();
+
+            while ($row = $result->fetch()) {
+                $allRecords = $this->findAllConnectedRecordsInPage($row['uid'], $depth, $allRecords);
+            }
+        }
+
+        // Add any versions of pages
+        if ($pageId > 0) {
+            $versions = BackendUtility::selectVersionsOfRecord('pages', $pageId, 'uid,t3ver_oid,t3ver_wsid,t3ver_count', null, true);
+            if (is_array($versions)) {
+                foreach ($versions as $verRec) {
+                    if (!$verRec['_CURRENT_VERSION']) {
+                        $allRecords = $this->findAllConnectedRecordsInPage($verRec['uid'], $depth, $allRecords);
+                    }
+                }
+            }
+        }
+        return $allRecords;
+    }
+
+    /**
+     * Deletes records via DataHandler
+     *
+     * @param array $orphanedRecords two level array with tables and uids
+     * @param bool $dryRun check if the records should NOT be deleted (use --dry-run to avoid)
+     * @param SymfonyStyle $io
+     * @return void
+     */
+    protected function deleteRecords(array $orphanedRecords, bool $dryRun, SymfonyStyle $io)
+    {
+        // Putting "pages" table in the bottom
+        if (isset($orphanedRecords['pages'])) {
+            $_pages = $orphanedRecords['pages'];
+            unset($orphanedRecords['pages']);
+            // To delete sub pages first assuming they are accumulated from top of page tree.
+            $orphanedRecords['pages'] = array_reverse($_pages);
+        }
+
+        // set up the data handler instance
+        $dataHandler = GeneralUtility::makeInstance(DataHandler::class);
+        $dataHandler->start([], []);
+
+        // Loop through all tables and their records
+        foreach ($orphanedRecords as $table => $list) {
+            if ($io->isVerbose()) {
+                $io->writeln('Flushing ' . count($list) . ' orphaned records from table "' . $table . '"');
+            }
+            foreach ($list as $uid) {
+                if ($io->isVeryVerbose()) {
+                    $io->writeln('Flushing record "' . $table . ':' . $uid . '"');
+                }
+                if (!$dryRun) {
+                    // Notice, we are deleting pages with no regard to subpages/subrecords - we do this since they
+                    // should also be included in the set of deleted pages of course (no un-deleted record can exist
+                    // under a deleted page...)
+                    $dataHandler->deleteRecord($table, $uid, true, true);
+                    // Return errors if any:
+                    if (!empty($dataHandler->errorLog)) {
+                        $errorMessage = array_merge(['DataHandler reported an error'], $dataHandler->errorLog);
+                        $io->error($errorMessage);
+                    } elseif (!$io->isQuiet()) {
+                        $io->writeln('Permanently deleted orphaned record "' . $table . ':' . $uid . '".');
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * Short-hand function for accessing the current backend user
+     * @return BackendUserAuthentication
+     */
+    protected function getBackendUser(): BackendUserAuthentication
+    {
+        return $GLOBALS['BE_USER'];
+    }
+}
diff --git a/typo3/sysext/lowlevel/Classes/OrphanRecordsCommand.php b/typo3/sysext/lowlevel/Classes/OrphanRecordsCommand.php
deleted file mode 100644 (file)
index 0f0fc24..0000000
+++ /dev/null
@@ -1,158 +0,0 @@
-<?php
-namespace TYPO3\CMS\Lowlevel;
-
-/*
- * This file is part of the TYPO3 CMS project.
- *
- * It is free software; you can redistribute it and/or modify it under
- * the terms of the GNU General Public License, either version 2
- * of the License, or any later version.
- *
- * For the full copyright and license information, please read the
- * LICENSE.txt file that was distributed with this source code.
- *
- * The TYPO3 project - inspiring people to share!
- */
-
-use TYPO3\CMS\Core\Database\Connection;
-use TYPO3\CMS\Core\Database\ConnectionPool;
-use TYPO3\CMS\Core\DataHandling\DataHandler;
-use TYPO3\CMS\Core\Utility\GeneralUtility;
-
-/**
- * Looking for Orphan Records
- */
-class OrphanRecordsCommand extends CleanerCommand
-{
-    /**
-     * Constructor
-     */
-    public function __construct()
-    {
-        parent::__construct();
-        // Setting up help:
-        $this->cli_options[] = ['--echotree level', 'When "level" is set to 1 or higher you will see the page of the page tree outputted as it is traversed. A value of 2 for "level" will show even more information.'];
-        $this->cli_help['name'] = 'orphan_records -- To find records that has lost their connection with the page tree';
-        $this->cli_help['description'] = trim('
-Assumptions:
-- That all actively used records on the website from TCA configured tables are located in the page tree exclusively.
-
-All records managed by TYPO3 via the TCA array configuration has to belong to a page in the page tree, either directly or indirectly as a version of another record.
-VERY TIME, CPU and MEMORY intensive operation since the full page tree is looked up!
-
-Automatic Repair of Errors:
-- Silently deleting the orphaned records. In theory they should not be used anywhere in the system, but there could be references. See below for more details on this matter.
-
-Manual repair suggestions:
-- Possibly re-connect orphaned records to page tree by setting their "pid" field to a valid page id. A lookup in the sys_refindex table can reveal if there are references to a orphaned record. If there are such references (from records that are not themselves orphans) you might consider to re-connect the record to the page tree, otherwise it should be safe to delete it.
-');
-        $this->cli_help['todo'] = trim('
-- Implement a check for references to orphaned records and if a reference comes from a record that is not orphaned itself, we might rather like to re-connect the record to the page tree.
-- Implement that orphans can be fixed by setting the PID to a certain page instead of deleting.');
-        $this->cli_help['examples'] = '/.../cli_dispatch.phpsh lowlevel_cleaner orphan_records -s -r
-Will report orphan uids from TCA tables.';
-    }
-
-    /**
-     * Find orphan records
-     * VERY CPU and memory intensive since it will look up the whole page tree!
-     *
-     * @return array
-     */
-    public function main()
-    {
-        // Initialize result array:
-        $resultArray = [
-            'message' => $this->cli_help['name'] . LF . LF . $this->cli_help['description'],
-            'headers' => [
-                'orphans' => ['Index of orphaned records', '', 3],
-                'misplaced_at_rootlevel' => ['Records that should not be at root level but are.', 'Fix manually by moving record into page tree', 2],
-                'misplaced_inside_tree' => ['Records that should be at root level but are not.', 'Fix manually by moving record to tree root', 2],
-                'illegal_record_under_versioned_page' => ['Records that cannot be attached to a versioned page', '(Listed under orphaned records so is fixed along with orphans.)', 2]
-            ],
-            'orphans' => [],
-            'misplaced_at_rootlevel' => [],
-            // Subset of "all": Those that should not be at root level but are. [Warning: Fix by moving record into page tree]
-            'misplaced_inside_tree' => [],
-            // Subset of "all": Those that are inside page tree but should be at root level [Warning: Fix by setting PID to zero]
-            'illegal_record_under_versioned_page' => []
-        ];
-        // zero = tree root, must use tree root if you wish to reverse selection to find orphans!
-        $startingPoint = 0;
-        $this->genTree($startingPoint, 1000, (int)$this->cli_argValue('--echotree'));
-        $resultArray['misplaced_at_rootlevel'] = $this->recStats['misplaced_at_rootlevel'];
-        $resultArray['misplaced_inside_tree'] = $this->recStats['misplaced_inside_tree'];
-        $resultArray['illegal_record_under_versioned_page'] = $this->recStats['illegal_record_under_versioned_page'];
-        // Find orphans:
-        foreach ($GLOBALS['TCA'] as $tableName => $cfg) {
-            $idList = [0];
-            if (is_array($this->recStats['all'][$tableName]) && count($this->recStats['all'][$tableName])) {
-                $idList = $this->recStats['all'][$tableName];
-            }
-            // Select all records belonging to page:
-            $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
-                ->getQueryBuilderForTable($tableName);
-
-            $result = $queryBuilder
-                ->select('uid')
-                ->from($tableName)
-                ->where(
-                    $queryBuilder->expr()->notIn(
-                        'uid',
-                        $queryBuilder->createNamedParameter($idList, Connection::PARAM_INT_ARRAY)
-                    )
-                )
-                ->orderBy('uid')
-                ->execute();
-
-            if ($result->rowCount()) {
-                $resultArray['orphans'][$tableName] = [];
-                while ($orphanRecord = $result->fetch()) {
-                    $resultArray['orphans'][$tableName][$orphanRecord['uid']] = $orphanRecord['uid'];
-                }
-            }
-        }
-        return $resultArray;
-    }
-
-    /**
-     * Mandatory autofix function
-     * Will run auto-fix on the result array. Echos status during processing.
-     *
-     * @param array $resultArray Result array from main() function
-     * @return void
-     */
-    public function main_autoFix($resultArray)
-    {
-        // Putting "pages" table in the bottom:
-        if (isset($resultArray['orphans']['pages'])) {
-            $_pages = $resultArray['orphans']['pages'];
-            unset($resultArray['orphans']['pages']);
-            $resultArray['orphans']['pages'] = $_pages;
-        }
-        // Traversing records:
-        foreach ($resultArray['orphans'] as $table => $list) {
-            echo 'Removing orphans from table "' . $table . '":' . LF;
-            foreach ($list as $uid) {
-                echo ' Flushing orphan record "' . $table . ':' . $uid . '": ';
-                if ($bypass = $this->cli_noExecutionCheck($table . ':' . $uid)) {
-                    echo $bypass;
-                } else {
-                    // Execute CMD array:
-                    $tce = GeneralUtility::makeInstance(DataHandler::class);
-                    $tce->start([], []);
-                    // Notice, we are deleting pages with no regard to subpages/subrecords - we do this
-                    // since they should also be included in the set of orphans of course!
-                    $tce->deleteRecord($table, $uid, true, true);
-                    // Return errors if any:
-                    if (count($tce->errorLog)) {
-                        echo ' ERROR from "TCEmain":' . LF . 'TCEmain:' . implode((LF . 'TCEmain:'), $tce->errorLog);
-                    } else {
-                        echo 'DONE';
-                    }
-                }
-                echo LF;
-            }
-        }
-    }
-}
index 7976e35..81e9b02 100644 (file)
@@ -14,6 +14,10 @@ return [
         'class' => \TYPO3\CMS\Lowlevel\Command\DeletedRecordsCommand::class,
         'user' => '_cli_lowlevel'
     ],
+    'cleanup:orphanrecords' => [
+        'class' => \TYPO3\CMS\Lowlevel\Command\OrphanRecordsCommand::class,
+        'user' => '_cli_lowlevel'
+    ],
     'cleanup:flexforms' => [
         'class' => \TYPO3\CMS\Lowlevel\Command\CleanFlexFormsCommand::class,
         'user' => '_cli_lowlevel'
index 2728298..920a5f8 100644 (file)
@@ -15,6 +15,5 @@ if (TYPO3_MODE === 'BE') {
     $GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['lowlevel']['cleanerModules']['double_files'] = [\TYPO3\CMS\Lowlevel\DoubleFilesCommand::class];
     $GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['lowlevel']['cleanerModules']['rte_images'] = [\TYPO3\CMS\Lowlevel\RteImagesCommand::class];
     $GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['lowlevel']['cleanerModules']['lost_files'] = [\TYPO3\CMS\Lowlevel\LostFilesCommand::class];
-    $GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['lowlevel']['cleanerModules']['orphan_records'] = [\TYPO3\CMS\Lowlevel\OrphanRecordsCommand::class];
     $GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['lowlevel']['cleanerModules']['versions'] = [\TYPO3\CMS\Lowlevel\VersionsCommand::class];
 }