[TASK] Merge indexed_search_mysql 66/49666/8
authorChristian Kuhn <lolli@schwarzbu.ch>
Tue, 30 Aug 2016 20:15:20 +0000 (22:15 +0200)
committerGeorg Ringer <georg.ringer@gmail.com>
Wed, 31 Aug 2016 16:28:00 +0000 (18:28 +0200)
* Merge hook from indexed_search_mysql directly into indexed_search
  and compatibilty7
* Migrate "hook code" merged into indexed_search to doctrine
* Add a feature flag in ext_conf_template "useMysqlFulltext"
* Add a hook to schema migrator if feature flag is enabled to
  register fulltext indexes

Change-Id: I685c16cdcd171257ed1060c4fe0f2b93c4c44ca9
Resolves: #77700
Releases: master
Reviewed-on: https://review.typo3.org/49666
Reviewed-by: Morton Jonuschat <m.jonuschat@mojocode.de>
Tested-by: Morton Jonuschat <m.jonuschat@mojocode.de>
Reviewed-by: Georg Ringer <georg.ringer@gmail.com>
Tested-by: Georg Ringer <georg.ringer@gmail.com>
15 files changed:
composer.json
typo3/sysext/compatibility7/Classes/Controller/SearchFormController.php
typo3/sysext/compatibility7/ext_emconf.php
typo3/sysext/core/Documentation/Changelog/master/Breaking-77700-ExtensionIndexed_search_mysqlMergedIntoIndexed_search.rst [new file with mode: 0644]
typo3/sysext/indexed_search/Classes/Domain/Repository/IndexSearchRepository.php
typo3/sysext/indexed_search/Classes/Service/DatabaseSchemaService.php [new file with mode: 0644]
typo3/sysext/indexed_search/Resources/Private/Language/locallang_em.xlf
typo3/sysext/indexed_search/ext_conf_template.txt
typo3/sysext/indexed_search/ext_localconf.php
typo3/sysext/indexed_search_mysql/Classes/Hook/MysqlFulltextIndexHook.php [deleted file]
typo3/sysext/indexed_search_mysql/composer.json [deleted file]
typo3/sysext/indexed_search_mysql/ext_emconf.php [deleted file]
typo3/sysext/indexed_search_mysql/ext_icon.png [deleted file]
typo3/sysext/indexed_search_mysql/ext_localconf.php [deleted file]
typo3/sysext/indexed_search_mysql/ext_tables.sql [deleted file]

index 01732cd..b79982a 100644 (file)
                "typo3/cms-func-wizards": "self.version",
                "typo3/cms-impexp": "self.version",
                "typo3/cms-indexed-search": "self.version",
-               "typo3/cms-indexed-search-mysql": "self.version",
                "typo3/cms-info": "self.version",
                "typo3/cms-info-pagetsconfig": "self.version",
                "typo3/cms-install": "self.version",
index 083972b..c00c3a1 100644 (file)
@@ -544,10 +544,22 @@ class SearchFormController extends \TYPO3\CMS\Frontend\Plugin\AbstractPlugin
      */
     public function getResultRows($searchWordArray, $freeIndexUid = -1)
     {
+        // unserializing the configuration of indexed_search (!) to see if
+        // the extension is configured to use the mysql fulltext feature
+        $extConf = [];
+        if (isset($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf']['indexed_search'])) {
+            $extConf = unserialize(
+                $GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf']['indexed_search'],
+                ['allowed_classes' => false]
+            );
+        }
+
         // Getting SQL result pointer. This fetches ALL results (1,000,000 if found)
         $this->timeTracker->push('Searching result');
         if ($hookObj = &$this->hookRequest('getResultRows_SQLpointer')) {
             $res = $hookObj->getResultRows_SQLpointer($searchWordArray, $freeIndexUid);
+        } elseif (isset($extConf['useMysqlFulltext']) && $extConf['useMysqlFulltext'] === '1') {
+            $res = $this->getResultRows_SQLpointerMysqlFulltext($searchWordArray, $freeIndexUid);
         } else {
             $res = $this->getResultRows_SQLpointer($searchWordArray, $freeIndexUid);
         }
@@ -659,6 +671,170 @@ class SearchFormController extends \TYPO3\CMS\Frontend\Plugin\AbstractPlugin
     }
 
     /**
+     * Gets a SQL result pointer to traverse for the search records.
+     *
+     * @param array $searchWordsArray Search words
+     * @param int $freeIndexUid Pointer to which indexing configuration you want to search in. -1 means no filtering. 0 means only regular indexed content.
+     * @return bool|\mysqli_result|object MySQLi result object / DBAL object
+     */
+    protected function getResultRows_SQLpointerMysqlFulltext($searchWordsArray, $freeIndexUid = -1)
+    {
+        // Build the search string, detect which fulltext index to use, and decide whether boolean search is needed or not
+        $searchData = $this->getSearchString($searchWordsArray);
+        // Perform SQL Search / collection of result rows array:
+        $resource = false;
+        if ($searchData) {
+            /** @var TimeTracker $timeTracker */
+            $timeTracker = GeneralUtility::makeInstance(TimeTracker::class);
+            // Do the search:
+            $timeTracker->push('execFinalQuery');
+            $resource = $this->execFinalQuery_fulltext($searchData, $freeIndexUid);
+            $timeTracker->pull();
+        }
+        return $resource;
+    }
+
+    /**
+     * Returns a search string for use with MySQL FULLTEXT query
+     *
+     * @param array $searchWordArray Search word array
+     * @return string Search string
+     */
+    protected function getSearchString($searchWordArray)
+    {
+        // Initialize variables:
+        $count = 0;
+        // Change this to TRUE to force BOOLEAN SEARCH MODE (useful if fulltext index is still empty)
+        $searchBoolean = false;
+        $fulltextIndex = 'index_fulltext.fulltextdata';
+        // This holds the result if the search is natural (doesn't contain any boolean operators)
+        $naturalSearchString = '';
+        // This holds the result if the search is boolen (contains +/-/| operators)
+        $booleanSearchString = '';
+
+        $searchType = (string)$this->getSearchType();
+
+        // Traverse searchwords and prefix them with corresponding operator
+        foreach ($searchWordArray as $searchWordData) {
+            // Making the query for a single search word based on the search-type
+            $searchWord = $searchWordData['sword'];
+            $wildcard = '';
+            if (strstr($searchWord, ' ')) {
+                $searchType = '20';
+            }
+            switch ($searchType) {
+                case '1':
+                case '2':
+                case '3':
+                    // First part of word
+                    $wildcard = '*';
+                    // Part-of-word search requires boolean mode!
+                    $searchBoolean = true;
+                    break;
+                case '10':
+                    $indexerObj = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\IndexedSearch\Indexer::class);
+                    // Initialize the indexer-class
+                    /** @var \TYPO3\CMS\IndexedSearch\Indexer $indexerObj */
+                    $searchWord = $indexerObj->metaphone($searchWord, $indexerObj->storeMetaphoneInfoAsWords);
+                    unset($indexerObj);
+                    $fulltextIndex = 'index_fulltext.metaphonedata';
+                    break;
+                case '20':
+                    $searchBoolean = true;
+                    // Remove existing quotes and fix misplaced quotes.
+                    $searchWord = trim(str_replace('"', ' ', $searchWord));
+                    break;
+            }
+            // Perform search for word:
+            switch ($searchWordData['oper']) {
+                case 'AND NOT':
+                    $booleanSearchString .= ' -' . $searchWord . $wildcard;
+                    $searchBoolean = true;
+                    break;
+                case 'OR':
+                    $booleanSearchString .= ' ' . $searchWord . $wildcard;
+                    $searchBoolean = true;
+                    break;
+                default:
+                    $booleanSearchString .= ' +' . $searchWord . $wildcard;
+                    $naturalSearchString .= ' ' . $searchWord;
+            }
+            $count++;
+        }
+        if ($searchType == '20') {
+            $searchString = '"' . trim($naturalSearchString) . '"';
+        } elseif ($searchBoolean) {
+            $searchString = trim($booleanSearchString);
+        } else {
+            $searchString = trim($naturalSearchString);
+        }
+        return [
+            'searchBoolean' => $searchBoolean,
+            'searchString' => $searchString,
+            'fulltextIndex' => $fulltextIndex
+        ];
+    }
+
+    /**
+     * Execute final query, based on phash integer list. The main point is sorting the result in the right order.
+     *
+     * @param array $searchData Array with search string, boolean indicator, and fulltext index reference
+     * @param int $freeIndexUid Pointer to which indexing configuration you want to search in. -1 means no filtering. 0 means only regular indexed content.
+     * @return bool|\mysqli_result|object MySQLi result object / DBAL object
+     */
+    protected function execFinalQuery_fulltext($searchData, $freeIndexUid = -1)
+    {
+        // Setting up methods of filtering results based on page types, access, etc.
+        $pageJoin = '';
+        // Indexing configuration clause:
+        $freeIndexUidClause = $this->freeIndexUidWhere($freeIndexUid);
+        // Calling hook for alternative creation of page ID list
+        $searchRootPageIdList = $this->getSearchRootPageIdList();
+        if ($hookObj = &$this->hookRequest('execFinalQuery_idList')) {
+            $pageWhere = $hookObj->execFinalQuery_idList('');
+        } elseif ($this->getJoinPagesForQuery()) {
+            // Alternative to getting all page ids by ->getTreeList() where "excludeSubpages" is NOT respected.
+            $pageJoin = ',
+                               pages';
+            $pageWhere = 'pages.uid = ISEC.page_id
+                               ' . $GLOBALS['TSFE']->cObj->enableFields('pages') . '
+                               AND pages.no_search=0
+                               AND pages.doktype<200
+                       ';
+        } elseif ($searchRootPageIdList[0] >= 0) {
+
+            // Collecting all pages IDs in which to search;
+            // filtering out ALL pages that are not accessible due to enableFields. Does NOT look for "no_search" field!
+            $idList = [];
+            foreach ($searchRootPageIdList as $rootId) {
+                /** @var \TYPO3\CMS\Frontend\ContentObject\ContentObjectRenderer $cObj */
+                $cObj = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Frontend\ContentObject\ContentObjectRenderer::class);
+                $idList[] = $cObj->getTreeList(-1 * $rootId, 9999);
+            }
+            $pageWhere = ' ISEC.page_id IN (' . implode(',', $idList) . ')';
+        } else {
+            // Disable everything... (select all)
+            $pageWhere = ' 1=1';
+        }
+        $searchBoolean = '';
+        if ($searchData['searchBoolean']) {
+            $searchBoolean = ' IN BOOLEAN MODE';
+        }
+        $resource = $GLOBALS['TYPO3_DB']->exec_SELECTquery(
+            'index_fulltext.*, ISEC.*, IP.*',
+            'index_fulltext, index_section ISEC, index_phash IP' . $pageJoin,
+            'MATCH (' . $searchData['fulltextIndex'] . ')
+                AGAINST (' . $GLOBALS['TYPO3_DB']->fullQuoteStr($searchData['searchString'], 'index_fulltext') . $searchBoolean . ') ' .
+            $this->mediaTypeWhere() . ' ' . $this->languageWhere() . $freeIndexUidClause . '
+                AND index_fulltext.phash = IP.phash
+                AND ISEC.phash = IP.phash
+                AND ' . $pageWhere . $this->sectionTableWhere(),
+            'IP.phash,ISEC.phash,ISEC.phash_t3,ISEC.rl0,ISEC.rl1,ISEC.rl2,ISEC.page_id,ISEC.uniqid,IP.phash_grouping,IP.data_filename ,IP.data_page_id ,IP.data_page_reg1,IP.data_page_type,IP.data_page_mp,IP.gr_list,IP.item_type,IP.item_title,IP.item_description,IP.item_mtime,IP.tstamp,IP.item_size,IP.contentHash,IP.crdate,IP.parsetime,IP.sys_language_uid,IP.item_crdate,IP.cHashParams,IP.externalUrl,IP.recordUid,IP.freeIndexUid,IP.freeIndexSetId'
+        );
+        return $resource;
+    }
+
+    /**
      * Compiles the HTML display of the incoming array of result rows.
      *
      * @param array $sWArr Search words array (for display of text describing what was searched for)
index ccc976d..9339bbe 100644 (file)
@@ -14,12 +14,11 @@ $EM_CONF[$_EXTKEY] = [
     'constraints' => [
         'depends' => [
             'typo3' => '8.4.0-8.4.99',
+            'indexed_search'
         ],
         'conflicts' => [
             'compatibility6' => '0.0.0',
         ],
-        'suggests' => [
-            'indexed_search'
-        ],
+        'suggests' => [],
     ],
 ];
diff --git a/typo3/sysext/core/Documentation/Changelog/master/Breaking-77700-ExtensionIndexed_search_mysqlMergedIntoIndexed_search.rst b/typo3/sysext/core/Documentation/Changelog/master/Breaking-77700-ExtensionIndexed_search_mysqlMergedIntoIndexed_search.rst
new file mode 100644 (file)
index 0000000..ac65d30
--- /dev/null
@@ -0,0 +1,31 @@
+============================================================================
+Breaking: #77700 - Extension indexed_search_mysql merged into indexed_search
+============================================================================
+
+Description
+===========
+
+The extension ``indexed_search_mysql`` has been removed and its functionality merged into
+extension ``indexed_search``. The ``MySQL`` specific search based on fulltext indexes can
+be enabled with a feature flag within the extension configuration of the extension manager.
+
+
+Impact
+======
+
+If extension ``indexed_search_mysql`` has been loaded, the feature flag ``useMysqlFulltext``
+within ``indexed_search`` has to be set, otherwise ``indexed_search`` falls back to the
+potentially slower non-fulltext based default search algorithm.
+
+
+Affected Installations
+======================
+
+Instances with extension ``indexed_search_mysql`` loaded.
+
+
+Migration
+=========
+
+Full functionality can be kept by enabling the feature toggle ``useMysqlFulltext`` within
+the extension configuration of ``indexed_search``.
\ No newline at end of file
index 5086c00..d2d75eb 100644 (file)
@@ -17,9 +17,11 @@ namespace TYPO3\CMS\IndexedSearch\Domain\Repository;
 use Doctrine\DBAL\Driver\Statement;
 use TYPO3\CMS\Core\Database\ConnectionPool;
 use TYPO3\CMS\Core\Database\Query\QueryHelper;
+use TYPO3\CMS\Core\Database\Query\Restriction\FrontendRestrictionContainer;
 use TYPO3\CMS\Core\TimeTracker\TimeTracker;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 use TYPO3\CMS\Core\Utility\MathUtility;
+use TYPO3\CMS\Core\Utility\StringUtility;
 use TYPO3\CMS\IndexedSearch\Indexer;
 use TYPO3\CMS\IndexedSearch\Utility;
 
@@ -207,10 +209,21 @@ class IndexSearchRepository
      */
     public function doSearch($searchWords, $freeIndexUid = -1)
     {
-        // Getting SQL result pointer:
+        // unserializing the configuration so we can use it here:
+        $extConf = [];
+        if (isset($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf']['indexed_search'])) {
+            $extConf = unserialize(
+                $GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf']['indexed_search'],
+                ['allowed_classes' => false]
+            );
+        }
+
+            // Getting SQL result pointer:
         $this->getTimeTracker()->push('Searching result');
         if ($hookObj = &$this->hookRequest('getResultRows_SQLpointer')) {
             $result = $hookObj->getResultRows_SQLpointer($searchWords, $freeIndexUid);
+        } elseif (isset($extConf['useMysqlFulltext']) && $extConf['useMysqlFulltext'] === '1') {
+            $result = $this->getResultRows_SQLpointerMysqlFulltext($searchWords, $freeIndexUid);
         } else {
             $result = $this->getResultRows_SQLpointer($searchWords, $freeIndexUid);
         }
@@ -316,6 +329,236 @@ class IndexSearchRepository
         }
     }
 
+    /**
+     * Gets a SQL result pointer to traverse for the search records.
+     *
+     * mysql fulltext specific version triggered by ext_conf_template setting 'useMysqlFulltext'
+     *
+     * @param array $searchWordsArray Search words
+     * @param int $freeIndexUid Pointer to which indexing configuration you want to search in. -1 means no filtering. 0 means only regular indexed content.
+     * @return bool|\mysqli_result|object MySQLi result object / DBAL object
+     */
+    protected function getResultRows_SQLpointerMysqlFulltext($searchWordsArray, $freeIndexUid = -1)
+    {
+        $connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable('index_fulltext');
+        if (!StringUtility::beginsWith($connection->getServerVersion(), 'MySQL')) {
+            throw new \RuntimeException(
+                'Extension indexed_search is configured to use mysql fulltext, but table \'index_fulltext\''
+                . ' is running on a different DBMS.',
+                1472585525
+            );
+        }
+        // Build the search string, detect which fulltext index to use, and decide whether boolean search is needed or not
+        $searchData = $this->getSearchString($searchWordsArray);
+        // Perform SQL Search / collection of result rows array:
+        $resource = false;
+        if ($searchData) {
+            /** @var TimeTracker $timeTracker */
+            $timeTracker = GeneralUtility::makeInstance(TimeTracker::class);
+            // Do the search:
+            $timeTracker->push('execFinalQuery');
+            $resource = $this->execFinalQuery_fulltext($searchData, $freeIndexUid);
+            $timeTracker->pull();
+        }
+        return $resource;
+    }
+
+    /**
+     * Returns a search string for use with MySQL FULLTEXT query
+     *
+     * mysql fulltext specific helper method
+     *
+     * @param array $searchWordArray Search word array
+     * @return string Search string
+     */
+    protected function getSearchString($searchWordArray)
+    {
+        // Initialize variables:
+        $count = 0;
+        // Change this to TRUE to force BOOLEAN SEARCH MODE (useful if fulltext index is still empty)
+        $searchBoolean = false;
+        $fulltextIndex = 'index_fulltext.fulltextdata';
+        // This holds the result if the search is natural (doesn't contain any boolean operators)
+        $naturalSearchString = '';
+        // This holds the result if the search is boolen (contains +/-/| operators)
+        $booleanSearchString = '';
+
+        $searchType = (string)$this->getSearchType();
+
+        // Traverse searchwords and prefix them with corresponding operator
+        foreach ($searchWordArray as $searchWordData) {
+            // Making the query for a single search word based on the search-type
+            $searchWord = $searchWordData['sword'];
+            $wildcard = '';
+            if (strstr($searchWord, ' ')) {
+                $searchType = '20';
+            }
+            switch ($searchType) {
+                case '1':
+                case '2':
+                case '3':
+                    // First part of word
+                    $wildcard = '*';
+                    // Part-of-word search requires boolean mode!
+                    $searchBoolean = true;
+                    break;
+                case '10':
+                    $indexerObj = GeneralUtility::makeInstance(Indexer::class);
+                    // Initialize the indexer-class
+                    /** @var Indexer $indexerObj */
+                    $searchWord = $indexerObj->metaphone($searchWord, $indexerObj->storeMetaphoneInfoAsWords);
+                    unset($indexerObj);
+                    $fulltextIndex = 'index_fulltext.metaphonedata';
+                    break;
+                case '20':
+                    $searchBoolean = true;
+                    // Remove existing quotes and fix misplaced quotes.
+                    $searchWord = trim(str_replace('"', ' ', $searchWord));
+                    break;
+            }
+            // Perform search for word:
+            switch ($searchWordData['oper']) {
+                case 'AND NOT':
+                    $booleanSearchString .= ' -' . $searchWord . $wildcard;
+                    $searchBoolean = true;
+                    break;
+                case 'OR':
+                    $booleanSearchString .= ' ' . $searchWord . $wildcard;
+                    $searchBoolean = true;
+                    break;
+                default:
+                    $booleanSearchString .= ' +' . $searchWord . $wildcard;
+                    $naturalSearchString .= ' ' . $searchWord;
+            }
+            $count++;
+        }
+        if ($searchType == '20') {
+            $searchString = '"' . trim($naturalSearchString) . '"';
+        } elseif ($searchBoolean) {
+            $searchString = trim($booleanSearchString);
+        } else {
+            $searchString = trim($naturalSearchString);
+        }
+        return [
+            'searchBoolean' => $searchBoolean,
+            'searchString' => $searchString,
+            'fulltextIndex' => $fulltextIndex
+        ];
+    }
+
+    /**
+     * Execute final query, based on phash integer list. The main point is sorting the result in the right order.
+     *
+     * mysql fulltext specific helper method
+     *
+     * @param array $searchData Array with search string, boolean indicator, and fulltext index reference
+     * @param int $freeIndexUid Pointer to which indexing configuration you want to search in. -1 means no filtering. 0 means only regular indexed content.
+     * @return Statement
+     */
+    protected function execFinalQuery_fulltext($searchData, $freeIndexUid = -1)
+    {
+        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('index_fulltext');
+        $queryBuilder->getRestrictions()->removeAll();
+        $queryBuilder->select('index_fulltext.*', 'ISEC.*', 'IP.*')
+            ->from('index_fulltext')
+            ->join(
+                'index_fulltext',
+                'index_phash',
+                'IP',
+                $queryBuilder->expr()->eq('index_fulltext.phash', $queryBuilder->quoteIdentifier('IP.phash'))
+            )
+            ->join(
+                'IP',
+                'index_section',
+                'ISEC',
+                $queryBuilder->expr()->eq('IP.phash', $queryBuilder->quoteIdentifier('ISEC.phash'))
+            );
+
+        // Calling hook for alternative creation of page ID list
+        $searchRootPageIdList = $this->getSearchRootPageIdList();
+        if ($hookObj = &$this->hookRequest('execFinalQuery_idList')) {
+            $pageWhere = $hookObj->execFinalQuery_idList('');
+            $queryBuilder->andWhere(QueryHelper::stripLogicalOperatorPrefix($pageWhere));
+        } elseif ($this->getJoinPagesForQuery()) {
+            // Alternative to getting all page ids by ->getTreeList() where "excludeSubpages" is NOT respected.
+            $queryBuilder
+                ->join(
+                    'ISEC',
+                    'pages',
+                    'pages',
+                    $queryBuilder->expr()->eq('ISEC.page_id', $queryBuilder->quoteIdentifier('pages.uid'))
+                )
+                ->andWhere($queryBuilder->expr()->eq('pages.no_search', 0))
+                ->andWhere($queryBuilder->expr()->lt('pages.doktype', 200));
+            $queryBuilder->setRestrictions(GeneralUtility::makeInstance(FrontendRestrictionContainer::class));
+        } elseif ($searchRootPageIdList[0] >= 0) {
+            // Collecting all pages IDs in which to search;
+            // filtering out ALL pages that are not accessible due to enableFields. Does NOT look for "no_search" field!
+            $idList = [];
+            foreach ($searchRootPageIdList as $rootId) {
+                /** @var \TYPO3\CMS\Frontend\ContentObject\ContentObjectRenderer $cObj */
+                $cObj = GeneralUtility::makeInstance(\TYPO3\CMS\Frontend\ContentObject\ContentObjectRenderer::class);
+                $idList[] = $cObj->getTreeList(-1 * $rootId, 9999);
+            }
+            $idList = GeneralUtility::intExplode(',', implode(',', $idList));
+            $queryBuilder->andWhere($queryBuilder->expr()->in('ISEC.page_id', $idList));
+        }
+
+        $searchBoolean = '';
+        if ($searchData['searchBoolean']) {
+            $searchBoolean = ' IN BOOLEAN MODE';
+        }
+        $queryBuilder->andWhere(
+            'MATCH (' . $queryBuilder->quoteIdentifier($searchData['fulltextIndex']) . ')'
+            . ' AGAINST (' . $queryBuilder->createNamedParameter($searchData['searchString'])
+            . $searchBoolean
+            . ')'
+        );
+
+        $queryBuilder->andWhere(
+            QueryHelper::stripLogicalOperatorPrefix($this->mediaTypeWhere()),
+            QueryHelper::stripLogicalOperatorPrefix($this->languageWhere()),
+            QueryHelper::stripLogicalOperatorPrefix($this->freeIndexUidWhere($freeIndexUid)),
+            QueryHelper::stripLogicalOperatorPrefix($this->sectionTableWhere())
+        );
+
+        $queryBuilder->groupBy(
+            'IP.phash',
+            'ISEC.phash',
+            'ISEC.phash_t3',
+            'ISEC.rl0',
+            'ISEC.rl1',
+            'ISEC.rl2',
+            'ISEC.page_id',
+            'ISEC.uniqid',
+            'IP.phash_grouping',
+            'IP.data_filename',
+            'IP.data_page_id',
+            'IP.data_page_reg1',
+            'IP.data_page_type',
+            'IP.data_page_mp',
+            'IP.gr_list',
+            'IP.item_type',
+            'IP.item_title',
+            'IP.item_description',
+            'IP.item_mtime',
+            'IP.tstamp',
+            'IP.item_size',
+            'IP.contentHash',
+            'IP.crdate',
+            'IP.parsetime',
+            'IP.sys_language_uid',
+            'IP.item_crdate',
+            'IP.cHashParams',
+            'IP.externalUrl',
+            'IP.recordUid',
+            'IP.freeIndexUid',
+            'IP.freeIndexSetId'
+        );
+
+        return $queryBuilder->execute();
+    }
+
     /***********************************
      *
      * Helper functions on searching (SQL)
diff --git a/typo3/sysext/indexed_search/Classes/Service/DatabaseSchemaService.php b/typo3/sysext/indexed_search/Classes/Service/DatabaseSchemaService.php
new file mode 100644 (file)
index 0000000..21ce31e
--- /dev/null
@@ -0,0 +1,51 @@
+<?php
+namespace TYPO3\CMS\IndexedSearch\Service;
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+/**
+ * This service provides the mysql specific changes of the schema definition
+ */
+class DatabaseSchemaService
+{
+    /**
+     * A slot method to inject the required mysql fulltext definition
+     * to schema migration
+     *
+     * @param array $sqlString
+     * @return array
+     */
+    public function addMysqlFulltextIndex(array $sqlString)
+    {
+        // Check again if the extension flag is enabled to be on the safe side
+        // even if the slot registration is moved around in ext_localconf
+        $extConf = [];
+        if (isset($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf']['indexed_search'])) {
+            $extConf = unserialize(
+                $GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf']['indexed_search'],
+                ['allowed_classes' => false]
+            );
+        }
+        if (isset($extConf['useMysqlFulltext']) && $extConf['useMysqlFulltext'] === '1') {
+            // @todo: With MySQL 5.7 fulltext index on InnoDB is possible, check for that and keep inno if so.
+            $sqlString[] = 'CREATE TABLE index_fulltext ('
+                . LF . 'fulltextdata mediumtext,'
+                . LF . 'metaphonedata mediumtext,'
+                . LF . 'FULLTEXT fulltextdata (fulltextdata),'
+                . LF . 'FULLTEXT metaphonedata (metaphonedata)'
+                . LF . ') ENGINE=MyISAM;';
+        }
+        return [$sqlString];
+    }
+}
index d2f1734..a6393f6 100644 (file)
@@ -57,6 +57,9 @@
                        <trans-unit id="indexedsearch.config.indexExternalURLs">
                                <source>Index External HTML URLs: If set, links to external URLs will be indexed if they are of type "text/html".</source>
                        </trans-unit>
+                       <trans-unit id="indexedsearch.config.useMysqlFulltext">
+                               <source>Use MySQL specific fulltext search - Update database schema in install tool after toggling this flag</source>
+                       </trans-unit>
                </body>
        </file>
 </xliff>
index b93e160..c6faa9a 100644 (file)
@@ -51,3 +51,6 @@ ignoreExtensions =
 
   # cat=basic; type=boolean; label=LLL:EXT:indexed_search/Resources/Private/Language/locallang_em.xlf:indexedsearch.config.indexExternalURLs
 indexExternalURLs = 0
+
+  # cat=basic; type=boolean; label=LLL:EXT:indexed_search/Resources/Private/Language/locallang_em.xlf:indexedsearch.config.useMysqlFulltext
+useMysqlFulltext = 0
index cb13a83..1182220 100644 (file)
@@ -43,10 +43,34 @@ $GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['indexed_search']['external_parsers'] = [
     'jpeg' => \TYPO3\CMS\IndexedSearch\FileContentParser::class,
     'tif'  => \TYPO3\CMS\IndexedSearch\FileContentParser::class
 ];
-$GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['indexed_search']['use_tables'] = 'index_phash,index_fulltext,index_rel,index_words,index_section,index_grlist,index_stat_search,index_stat_word,index_debug,index_config';
 
 // unserializing the configuration so we can use it here:
-$extConf = unserialize($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf']['indexed_search'], ['allowed_classes' => false]);
+$extConf = [];
+if (isset($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf']['indexed_search'])) {
+    $extConf = unserialize(
+        $GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf']['indexed_search'],
+        ['allowed_classes' => false]
+    );
+}
+
+if (isset($extConf['useMysqlFulltext']) && $extConf['useMysqlFulltext'] === '1') {
+    // Use all index_* tables except "index_rel" and "index_words"
+    $GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['indexed_search']['use_tables'] =
+        'index_phash,index_fulltext,index_section,index_grlist,index_stat_search,index_stat_word,index_debug,index_config';
+    // Register schema analyzer slot to hook in required fulltext index definition
+    $signalSlotDispatcher = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Extbase\SignalSlot\Dispatcher::class);
+    $signalSlotDispatcher->connect(
+        \TYPO3\CMS\Install\Service\SqlExpectedSchemaService::class,
+        'tablesDefinitionIsBeingBuilt',
+        \TYPO3\CMS\IndexedSearch\Service\DatabaseSchemaService::class,
+        'addMysqlFulltextIndex'
+    );
+    unset($signalSlotDispatcher);
+} else {
+    $GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['indexed_search']['use_tables'] =
+        'index_phash,index_fulltext,index_rel,index_words,index_section,index_grlist,index_stat_search,index_stat_word,index_debug,index_config';
+}
+
 // Use the advanced doubleMetaphone parser instead of the internal one (usage of metaphone parsers is generally disabled by default)
 if (isset($extConf['enableMetaphoneSearch']) && (int)$extConf['enableMetaphoneSearch'] == 2) {
     $GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['indexed_search']['metaphone'] = \TYPO3\CMS\IndexedSearch\Utility\DoubleMetaPhoneUtility::class;
diff --git a/typo3/sysext/indexed_search_mysql/Classes/Hook/MysqlFulltextIndexHook.php b/typo3/sysext/indexed_search_mysql/Classes/Hook/MysqlFulltextIndexHook.php
deleted file mode 100644 (file)
index 61e98c3..0000000
+++ /dev/null
@@ -1,199 +0,0 @@
-<?php
-namespace TYPO3\CMS\IndexedSearchMysql\Hook;
-
-/*
- * This file is part of the TYPO3 CMS project.
- *
- * It is free software; you can redistribute it and/or modify it under
- * the terms of the GNU General Public License, either version 2
- * of the License, or any later version.
- *
- * For the full copyright and license information, please read the
- * LICENSE.txt file that was distributed with this source code.
- *
- * The TYPO3 project - inspiring people to share!
- */
-use TYPO3\CMS\Core\TimeTracker\TimeTracker;
-use TYPO3\CMS\Core\Utility\GeneralUtility;
-
-/**
- * Class that hooks into Indexed Search and replaces standard SQL queries with MySQL fulltext index queries.
- */
-class MysqlFulltextIndexHook
-{
-    /**
-     * @var \TYPO3\CMS\IndexedSearch\Controller\SearchFormController|\TYPO3\CMS\IndexedSearch\Domain\Repository\IndexSearchRepository
-     */
-    public $pObj;
-
-    const ANY_PART_OF_THE_WORD = '1';
-    const LAST_PART_OF_THE_WORD = '2';
-    const FIRST_PART_OF_THE_WORD = '3';
-    const SOUNDS_LIKE = '10';
-    const SENTENCE = '20';
-    /**
-     * Gets a SQL result pointer to traverse for the search records.
-     *
-     * @param array $searchWordsArray Search words
-     * @param int $freeIndexUid Pointer to which indexing configuration you want to search in. -1 means no filtering. 0 means only regular indexed content.
-     * @return bool|\mysqli_result|object MySQLi result object / DBAL object
-     */
-    public function getResultRows_SQLpointer($searchWordsArray, $freeIndexUid = -1)
-    {
-        // Build the search string, detect which fulltext index to use, and decide whether boolean search is needed or not
-        $searchData = $this->getSearchString($searchWordsArray);
-        // Perform SQL Search / collection of result rows array:
-        $resource = false;
-        if ($searchData) {
-            /** @var TimeTracker $timeTracker */
-            $timeTracker = GeneralUtility::makeInstance(TimeTracker::class);
-            // Do the search:
-            $timeTracker->push('execFinalQuery');
-            $resource = $this->execFinalQuery_fulltext($searchData, $freeIndexUid);
-            $timeTracker->pull();
-        }
-        return $resource;
-    }
-
-    /**
-     * Returns a search string for use with MySQL FULLTEXT query
-     *
-     * @param array $searchWordArray Search word array
-     * @return string Search string
-     */
-    public function getSearchString($searchWordArray)
-    {
-        // Initialize variables:
-        $count = 0;
-        // Change this to TRUE to force BOOLEAN SEARCH MODE (useful if fulltext index is still empty)
-        $searchBoolean = false;
-        $fulltextIndex = 'index_fulltext.fulltextdata';
-        // This holds the result if the search is natural (doesn't contain any boolean operators)
-        $naturalSearchString = '';
-        // This holds the result if the search is boolen (contains +/-/| operators)
-        $booleanSearchString = '';
-
-        $searchType = (string)$this->pObj->getSearchType();
-
-        // Traverse searchwords and prefix them with corresponding operator
-        foreach ($searchWordArray as $searchWordData) {
-            // Making the query for a single search word based on the search-type
-            $searchWord = $searchWordData['sword'];
-            $wildcard = '';
-            if (strstr($searchWord, ' ')) {
-                $searchType = self::SENTENCE;
-            }
-            switch ($searchType) {
-                case self::ANY_PART_OF_THE_WORD:
-
-                case self::LAST_PART_OF_THE_WORD:
-
-                case self::FIRST_PART_OF_THE_WORD:
-                    // First part of word
-                    $wildcard = '*';
-                    // Part-of-word search requires boolean mode!
-                    $searchBoolean = true;
-                    break;
-                case self::SOUNDS_LIKE:
-                    $indexerObj = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\IndexedSearch\Indexer::class);
-                    // Initialize the indexer-class
-                    /** @var \TYPO3\CMS\IndexedSearch\Indexer $indexerObj */
-                    $searchWord = $indexerObj->metaphone($searchWord, $indexerObj->storeMetaphoneInfoAsWords);
-                    unset($indexerObj);
-                    $fulltextIndex = 'index_fulltext.metaphonedata';
-                    break;
-                case self::SENTENCE:
-                    $searchBoolean = true;
-                    // Remove existing quotes and fix misplaced quotes.
-                    $searchWord = trim(str_replace('"', ' ', $searchWord));
-                    break;
-                }
-                // Perform search for word:
-                switch ($searchWordData['oper']) {
-                    case 'AND NOT':
-                        $booleanSearchString .= ' -' . $searchWord . $wildcard;
-                        $searchBoolean = true;
-                        break;
-                    case 'OR':
-                        $booleanSearchString .= ' ' . $searchWord . $wildcard;
-                        $searchBoolean = true;
-                        break;
-                    default:
-                        $booleanSearchString .= ' +' . $searchWord . $wildcard;
-                        $naturalSearchString .= ' ' . $searchWord;
-            }
-            $count++;
-        }
-        if ($searchType == self::SENTENCE) {
-            $searchString = '"' . trim($naturalSearchString) . '"';
-        } elseif ($searchBoolean) {
-            $searchString = trim($booleanSearchString);
-        } else {
-            $searchString = trim($naturalSearchString);
-        }
-        return [
-            'searchBoolean' => $searchBoolean,
-            'searchString' => $searchString,
-            'fulltextIndex' => $fulltextIndex
-        ];
-    }
-
-    /**
-     * Execute final query, based on phash integer list. The main point is sorting the result in the right order.
-     *
-     * @param array $searchData Array with search string, boolean indicator, and fulltext index reference
-     * @param int $freeIndexUid Pointer to which indexing configuration you want to search in. -1 means no filtering. 0 means only regular indexed content.
-     * @return bool|\mysqli_result|object MySQLi result object / DBAL object
-     */
-    protected function execFinalQuery_fulltext($searchData, $freeIndexUid = -1)
-    {
-        // Setting up methods of filtering results based on page types, access, etc.
-        $pageJoin = '';
-        // Indexing configuration clause:
-        $freeIndexUidClause = $this->pObj->freeIndexUidWhere($freeIndexUid);
-        // Calling hook for alternative creation of page ID list
-        $searchRootPageIdList = $this->pObj->getSearchRootPageIdList();
-        if ($hookObj = &$this->pObj->hookRequest('execFinalQuery_idList')) {
-            $pageWhere = $hookObj->execFinalQuery_idList('');
-        } elseif ($this->pObj->getJoinPagesForQuery()) {
-            // Alternative to getting all page ids by ->getTreeList() where "excludeSubpages" is NOT respected.
-            $pageJoin = ',
-                               pages';
-            $pageWhere = 'pages.uid = ISEC.page_id
-                               ' . $GLOBALS['TSFE']->cObj->enableFields('pages') . '
-                               AND pages.no_search=0
-                               AND pages.doktype<200
-                       ';
-        } elseif ($searchRootPageIdList[0] >= 0) {
-
-            // Collecting all pages IDs in which to search;
-            // filtering out ALL pages that are not accessible due to enableFields. Does NOT look for "no_search" field!
-            $idList = [];
-            foreach ($searchRootPageIdList as $rootId) {
-                /** @var \TYPO3\CMS\Frontend\ContentObject\ContentObjectRenderer $cObj */
-                $cObj = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Frontend\ContentObject\ContentObjectRenderer::class);
-                $idList[] = $cObj->getTreeList(-1 * $rootId, 9999);
-            }
-            $pageWhere = ' ISEC.page_id IN (' . implode(',', $idList) . ')';
-        } else {
-            // Disable everything... (select all)
-            $pageWhere = ' 1=1';
-        }
-        $searchBoolean = '';
-        if ($searchData['searchBoolean']) {
-            $searchBoolean = ' IN BOOLEAN MODE';
-        }
-        $resource = $GLOBALS['TYPO3_DB']->exec_SELECTquery(
-            'index_fulltext.*, ISEC.*, IP.*',
-            'index_fulltext, index_section ISEC, index_phash IP' . $pageJoin,
-            'MATCH (' . $searchData['fulltextIndex'] . ')
-                AGAINST (' . $GLOBALS['TYPO3_DB']->fullQuoteStr($searchData['searchString'], 'index_fulltext') . $searchBoolean . ') ' .
-                $this->pObj->mediaTypeWhere() . ' ' . $this->pObj->languageWhere() . $freeIndexUidClause . '
-                AND index_fulltext.phash = IP.phash
-                AND ISEC.phash = IP.phash
-                AND ' . $pageWhere . $this->pObj->sectionTableWhere(),
-            'IP.phash,ISEC.phash,ISEC.phash_t3,ISEC.rl0,ISEC.rl1,ISEC.rl2,ISEC.page_id,ISEC.uniqid,IP.phash_grouping,IP.data_filename ,IP.data_page_id ,IP.data_page_reg1,IP.data_page_type,IP.data_page_mp,IP.gr_list,IP.item_type,IP.item_title,IP.item_description,IP.item_mtime,IP.tstamp,IP.item_size,IP.contentHash,IP.crdate,IP.parsetime,IP.sys_language_uid,IP.item_crdate,IP.cHashParams,IP.externalUrl,IP.recordUid,IP.freeIndexUid,IP.freeIndexSetId'
-        );
-        return $resource;
-    }
-}
diff --git a/typo3/sysext/indexed_search_mysql/composer.json b/typo3/sysext/indexed_search_mysql/composer.json
deleted file mode 100644 (file)
index af7c959..0000000
+++ /dev/null
@@ -1,19 +0,0 @@
-{
-       "name": "typo3/cms-indexed-search-mysql",
-       "type": "typo3-cms-framework",
-       "description": "TYPO3 Core",
-       "homepage": "https://typo3.org",
-       "license": ["GPL-2.0+"],
-
-       "require": {
-               "typo3/cms-core": "*"
-       },
-       "replace": {
-               "indexed_search_mysql": "*"
-       },
-       "autoload": {
-               "psr-4": {
-                       "TYPO3\\CMS\\IndexedSearchMysql\\": "Classes/"
-               }
-       }
-}
diff --git a/typo3/sysext/indexed_search_mysql/ext_emconf.php b/typo3/sysext/indexed_search_mysql/ext_emconf.php
deleted file mode 100644 (file)
index 497b0b7..0000000
+++ /dev/null
@@ -1,22 +0,0 @@
-<?php
-$EM_CONF[$_EXTKEY] = [
-    'title' => 'MySQL driver for Indexed Search Engine',
-    'description' => 'MySQL specific driver for Indexed Search Engine. Allows usage of MySQL-only features like FULLTEXT indexes.',
-    'category' => 'misc',
-    'state' => 'beta',
-    'uploadfolder' => 0,
-    'createDirs' => '',
-    'clearCacheOnLoad' => 0,
-    'author' => 'Michael Stucki',
-    'author_email' => 'michael@typo3.org',
-    'author_company' => '',
-    'version' => '8.4.0',
-    'constraints' => [
-        'depends' => [
-            'typo3' => '8.4.0-8.4.99',
-            'indexed_search' => '8.4.0-8.4.99',
-        ],
-        'conflicts' => [],
-        'suggests' => [],
-    ],
-];
diff --git a/typo3/sysext/indexed_search_mysql/ext_icon.png b/typo3/sysext/indexed_search_mysql/ext_icon.png
deleted file mode 100644 (file)
index 51c2367..0000000
Binary files a/typo3/sysext/indexed_search_mysql/ext_icon.png and /dev/null differ
diff --git a/typo3/sysext/indexed_search_mysql/ext_localconf.php b/typo3/sysext/indexed_search_mysql/ext_localconf.php
deleted file mode 100644 (file)
index c43f49f..0000000
+++ /dev/null
@@ -1,7 +0,0 @@
-<?php
-defined('TYPO3_MODE') or die();
-
-// Configure hook to query the fulltext index
-$GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['indexed_search']['pi1_hooks']['getResultRows_SQLpointer'] = \TYPO3\CMS\IndexedSearchMysql\Hook\MysqlFulltextIndexHook::class;
-// Use all index_* tables except "index_rel" and "index_words"
-$GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['indexed_search']['use_tables'] = 'index_phash,index_fulltext,index_section,index_grlist,index_stat_search,index_stat_word,index_debug,index_config';
diff --git a/typo3/sysext/indexed_search_mysql/ext_tables.sql b/typo3/sysext/indexed_search_mysql/ext_tables.sql
deleted file mode 100644 (file)
index 0a17b87..0000000
+++ /dev/null
@@ -1,16 +0,0 @@
-#
-# Table structure for table 'index_fulltext'
-#
-# Differences compared to original definition in EXT:indexed_search are as follows:
-# - Add new mediumtext field "metaphonedata"
-# - Add new FULLTEXT index "fulltextdata"
-# - Add new FULLTEXT index "metaphonedata"
-# - Change table engine from InnoDB to MyISAM (required for FULLTEXT indexing)
-CREATE TABLE index_fulltext (
-  phash int(11) DEFAULT '0' NOT NULL,
-  fulltextdata mediumtext,
-  metaphonedata mediumtext,
-  PRIMARY KEY (phash),
-  FULLTEXT fulltextdata (fulltextdata),
-  FULLTEXT metaphonedata (metaphonedata)
-) ENGINE=MyISAM;