Commit bf63a1c2 authored by Georg Ringer's avatar Georg Ringer Committed by Tymoteusz Motylewski
Browse files

[!!!][TASK] Rewrite backend modules of indexed_search

 * Move info modules and admin module to a central place
 * Use extbase & fluid
 * Add translations

The old modules are removed with a different commit to make it possible
to compare the output of old and new!

Releases: master
Resolves: #61511
Change-Id: Id750b0ad78e8ab115b917d4772281d3aceca6eee
Reviewed-on: http://review.typo3.org/32708


Reviewed-by: default avatarMarkus Klein <klein.t3@reelworx.at>
Tested-by: default avatarSusanne Moog <typo3@susannemoog.de>
Reviewed-by: Tymoteusz Motylewski's avatarTymoteusz Motylewski <t.motylewski@gmail.com>
Tested-by: Tymoteusz Motylewski's avatarTymoteusz Motylewski <t.motylewski@gmail.com>
parent ef072552
================================================
Breaking: #61510 - Improvement of indexed_search
================================================
Description
===========
The extension indexed_search is improved in the backend and frontend.
Backend
-------
Previously the functionality of indexed_search has been scattered to multiple modules.
Information about indexed_search was available in a custom module in "Admin tools" and 2 sections in the "Info" module.
The complete code has been moved to a central place, which is now a custom module in the area "Web" and has been rewritten
by using Extbase & Fluid. Translations and a modern UI have been added as well.
Impact
======
Changes in the Backend
----------------------
The previous user configuration for indexed_search modules is not working anymore.
Therefore editors won't see the module anymore after login.
Changes in the Frontend
-----------------------
The TypoScript configuration changed. If indexed_search is installed, it is automatically activated: ::
config.index_enable = 1
config.index_externals = 1
Affected installations
======================
All installations using indexed_search
Migration
=========
Backend
-------
Reconfigure the backend users and groups if users need to see the module of indexed_search.
\ No newline at end of file
<?php
namespace TYPO3\CMS\IndexedSearch\Controller;
/**
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
use TYPO3\CMS\Core\Authentication\BackendUserAuthentication;
use TYPO3\CMS\Core\Database\DatabaseConnection;
use TYPO3\CMS\Extbase\Mvc\Controller\ActionController;
use TYPO3\CMS\Extbase\Mvc\Web\Request as WebRequest;
use TYPO3\CMS\IndexedSearch\Domain\Repository\AdministrationRepository;
use TYPO3\CMS\Backend\Utility\BackendUtility;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\IndexedSearch\Indexer;
use TYPO3\CMS\Lang\LanguageService;
/**
* Administration controller
*/
class AdministrationController extends ActionController {
/**
* @var AdministrationRepository
*/
protected $administrationRepository;
/**
* @var int Current page id
*/
protected $pageUid = 0;
/**
* @var int Max lists per page
*/
protected $maxListPerPage = 50;
/**
* @var array External parsers
*/
protected $external_parsers = array();
/**
* @var array Configuration defined in the Extension Manager
*/
protected $indexerConfig = array();
/**
* @var bool is metaphone enabled
*/
protected $enableMetaphoneSearch = FALSE;
/**
* Indexer object
*
* @var \TYPO3\CMS\IndexedSearch\Indexer
*/
protected $indexer;
/**
* Function will be called before every other action
*
* @return void
*/
public function initializeAction() {
$this->pageUid = (int)GeneralUtility::_GET('id');
$this->maxListPerPage = GeneralUtility::_GP('listALL') ? 100000 : 100;
$this->indexerConfig = unserialize($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf']['indexed_search']);
$this->enableMetaphoneSearch = (bool)$this->indexerConfig['enableMetaphoneSearch'];
$this->indexer = GeneralUtility::makeInstance(Indexer::class);
parent::initializeAction();
}
/**
* Override the action name if found in the uc of the user
*
* @param \TYPO3\CMS\Extbase\Mvc\RequestInterface $request
* @param \TYPO3\CMS\Extbase\Mvc\ResponseInterface $response
* @throws \TYPO3\CMS\Extbase\Mvc\Exception\UnsupportedRequestTypeException
*/
public function processRequest(\TYPO3\CMS\Extbase\Mvc\RequestInterface $request, \TYPO3\CMS\Extbase\Mvc\ResponseInterface $response) {
$vars = GeneralUtility::_GET('tx_indexedsearch_web_indexedsearchisearch');
$beUser = $this->getBackendUserAuthentication();
if (is_array($vars) && isset($vars['action']) && method_exists($this, $vars['action'] . 'Action')) {
$action = $vars['action'];
switch($action) {
case 'saveStopwordsKeywords':
$action = 'statisticDetails';
break;
case 'deleteIndexedItem':
$action = 'statistic';
break;
}
$beUser->uc['indexed_search']['action'] = $action;
$beUser->uc['indexed_search']['arguments'] = $request->getArguments();
$beUser->writeUC();
} elseif (isset($beUser->uc['indexed_search']['action'])) {
if ($request instanceof WebRequest) {
$request->setControllerActionName($beUser->uc['indexed_search']['action']);
}
if (isset($beUser->uc['indexed_search']['arguments'])) {
$request->setArguments($beUser->uc['indexed_search']['arguments']);
}
}
parent::processRequest($request, $response);
}
/**
* @param \TYPO3\CMS\IndexedSearch\Domain\Repository\AdministrationRepository $administrationRepository
* @return void
*/
public function injectAdministrationRepository(AdministrationRepository $administrationRepository) {
$this->administrationRepository = $administrationRepository;
}
/**
* Index action contains the most important statistics
*
* @return void
*/
public function indexAction() {
$this->view->assignMultiple(array(
'records' => $this->administrationRepository->getRecordsNumbers(),
'phash' => $this->administrationRepository->getPageHashTypes()
));
if ($this->pageUid) {
$last24hours = ' AND tstamp > ' . ($GLOBALS['EXEC_TIME'] - 24 * 60 * 60);
$last30days = ' AND tstamp > ' . ($GLOBALS['EXEC_TIME'] - 30 * 24 * 60 * 60);
$this->view->assignMultiple(array(
'pageUid' => $this->pageUid,
'all' => $this->administrationRepository->getGeneralSearchStatistic('', $this->pageUid),
'last24hours' => $this->administrationRepository->getGeneralSearchStatistic($last24hours, $this->pageUid),
'last30days' => $this->administrationRepository->getGeneralSearchStatistic($last30days, $this->pageUid),
));
}
}
/**
* Statistics for pages
*
* @return void
*/
public function pagesAction() {
$this->view->assign('records', $this->administrationRepository->getPageStatistic());
}
/**
* Statistics for external documents
*
* @return void
*/
public function externalDocumentsAction() {
$this->view->assign('records', $this->administrationRepository->getExternalDocumentsStatistic());
}
/**
* Statistics for a given page hash
*
* @param int $pageHash
* @return void
*/
public function statisticDetailsAction($pageHash = 0) {
$pageHash = (int)$pageHash;
$db = $this->getDatabaseConnection();
$pageHashRow = $db->exec_SELECTgetSingleRow('*', 'index_phash', 'phash = ' . (int)$pageHash);
if (!is_array($pageHashRow)) {
$this->redirect('statistic');
}
$debugRow = $db->exec_SELECTgetRows('*', 'index_debug', 'phash = ' . (int)$pageHash);
$debugInfo = array();
$lexer = '';
if (is_array($debugRow)) {
$debugInfo = unserialize($debugRow[0]['debuginfo']);
$lexer = $debugInfo['lexer'];
unset($debugInfo['lexer']);
}
$pageRecord = BackendUtility::getRecord('pages', $pageHashRow['data_page_id']);
$keywords = is_array($pageRecord) ? array_flip(GeneralUtility::trimExplode(',', $pageRecord['keywords'], TRUE)) : array();
$wordRecords = $db->exec_SELECTgetRows(
'index_words.*, index_rel.*',
'index_rel, index_words',
'index_rel.phash = ' . (int)$pageHash . ' AND index_words.wid = index_rel.wid',
'',
'index_words.baseword'
);
foreach($wordRecords as $id => $row) {
if (isset($keywords[$row['baseword']])) {
$wordRecords[$id]['is_keyword'] = TRUE;
}
}
$metaphoneRows = $metaphone = array();
if ($this->enableMetaphoneSearch && is_array($wordRecords)) {
// Group metaphone hash
foreach ($wordRecords as $row) {
$metaphoneRows[$row['metaphone']][] = $row['baseword'];
}
foreach ($metaphoneRows as $hash => $words) {
if (count($words) > 1) {
$metaphone[] = array(
'metaphone' => $this->indexer->metaphone($words[0], 1), $hash,
'words' => $words,
'hash' => $hash
);
}
}
}
$this->view->assignMultiple(array(
'phash' => $pageHash,
'phashRow' => $pageHashRow,
'words' => $wordRecords,
'sections' => $db->exec_SELECTgetRows(
'*',
'index_section',
'index_section.phash = ' . (int)$pageHash
),
'topCount' => $db->exec_SELECTgetRows(
'index_words.baseword, index_words.metaphone, index_rel.*',
'index_rel, index_words',
'index_rel.phash = ' . (int)$pageHash . ' AND index_words.wid = index_rel.wid
AND index_words.is_stopword=0',
'',
'index_rel.count DESC',
'20'
),
'topFrequency' => $db->exec_SELECTgetRows(
'index_words.baseword, index_words.metaphone, index_rel.*',
'index_rel, index_words',
'index_rel.phash = ' . (int)$pageHash . ' AND index_words.wid = index_rel.wid
AND index_words.is_stopword=0',
'',
'index_rel.freq DESC',
'20'
),
'debug' => $debugInfo,
'lexer' => $lexer,
'metaphone' => $metaphone,
'page' => $pageRecord,
'keywords' => $keywords
));
}
/**
* Save stop words and keywords
*
* @param string $pageHash
* @param int $pageId
* @param array $stopwords
* @param array $keywords
* @return void
*/
public function saveStopwordsKeywordsAction($pageHash, $pageId, $stopwords = array(), $keywords = array()) {
if ($this->getBackendUserAuthentication()->isAdmin()) {
if (is_array($stopwords) && !empty($stopwords)) {
$this->administrationRepository->saveStopWords($stopwords);
}
if (is_array($keywords) && !empty($keywords)) {
$this->administrationRepository->saveKeywords($keywords, $pageId);
}
}
$this->redirect('statisticDetails', NULL, NULL, array('pageHash' => $pageHash));
}
/**
* Statistics for a given word id
*
* @param int $id
* @param int $pageHash
* @return void
*/
public function wordDetailAction($id = 0, $pageHash = 0) {
$rows = $this->getDatabaseConnection()->exec_SELECTgetRows(
'index_phash.*, index_section.*, index_rel.*',
'index_rel, index_section, index_phash',
'index_rel.wid = ' . (int)$id . ' AND index_rel.phash = index_section.phash' . ' AND index_section.phash = index_phash.phash',
'',
'index_rel.freq DESC'
);
$this->view->assignMultiple(array(
'rows' => $rows,
'phash' => $pageHash
));
}
/**
* General statistics
*
* @param int $depth
* @param string $mode
* @return void
*/
public function statisticAction($depth = 1, $mode = 'overview') {
if (is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['indexed_search']['external_parsers'])) {
foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['indexed_search']['external_parsers'] as $extension => $_objRef) {
/** @var \TYPO3\CMS\IndexedSearch\FileContentParser $fileContentParser */
$fileContentParser = GeneralUtility::getUserObj($_objRef);
if ($fileContentParser->softInit($extension)) {
$this->external_parsers[$extension] = $fileContentParser;
}
}
}
$this->administrationRepository->external_parsers = $this->external_parsers;
$allLines = $this->administrationRepository->getTree($this->pageUid, $depth, $mode);
$this->view->assignMultiple(array(
'levelTranslations' => explode('|', $this->getLanguageService()->sL('LLL:EXT:lang/locallang_core.xlf:labels.enterSearchLevels')),
'tree' => $allLines,
'pageUid' => $this->pageUid,
'mode' => $mode,
'depth' => $depth
));
}
/**
* Remove item from index
*
* @param string $id
* @param int $depth
* @param string $mode
* @return void
*/
public function deleteIndexedItemAction($id, $depth = 1, $mode = 'overview') {
$this->administrationRepository->removeIndexedPhashRow($id, $this->pageUid, $depth);
$this->redirect('statistic', NULL, NULL, array('depth' => $depth, 'mode' => $mode));
}
/**
* @return DatabaseConnection
*/
protected function getDatabaseConnection() {
return $GLOBALS['TYPO3_DB'];
}
/**
* @return BackendUserAuthentication
*/
protected function getBackendUserAuthentication() {
return $GLOBALS['BE_USER'];
}
/**
* @return LanguageService
*/
protected function getLanguageService() {
return $GLOBALS['LANG'];
}
}
......@@ -29,7 +29,7 @@ class SearchFormController extends \TYPO3\CMS\Frontend\Plugin\AbstractPlugin {
public $prefixId = 'tx_indexedsearch';
// Same as class name
public $scriptRelPath = 'pi/class.tx_indexedsearch.php';
public $scriptRelPath = 'Classes/Controller/SearchFormController.php';
// Path to this script relative to the extension dir.
public $extKey = 'indexed_search';
......
<?php
namespace TYPO3\CMS\IndexedSearch\Domain\Repository;
/**
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
use TYPO3\CMS\Backend\FrontendBackendUserAuthentication;
use TYPO3\CMS\Backend\Tree\View\PageTreeView;
use TYPO3\CMS\Backend\Utility\IconUtility;
use TYPO3\CMS\Core\Authentication\BackendUserAuthentication;
use TYPO3\CMS\Core\Cache\CacheManager;
use TYPO3\CMS\Core\Cache\Frontend\FrontendInterface;
use TYPO3\CMS\Core\DataHandling\DataHandler;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Backend\Utility\BackendUtility;
use TYPO3\CMS\Dbal\Database\DatabaseConnection;
use TYPO3\CMS\IndexedSearch\FileContentParser;
/**
* Administration repository
*/
class AdministrationRepository {
/**
* List of fileContentParsers
*
* @var FileContentParser[]
*/
public $external_parsers = array();
/**
* @var array
*/
protected $allPhashListed = array();
/**
* @var array
*/
protected $iconFileNameCache = array();
/**
* Get group list information
*
* @param int $phash
* @return array
*/
public function getGrlistRecord($phash) {
$db = $this->getDatabaseConnection();
$res = $db->exec_SELECTquery('index_grlist.*', 'index_grlist', 'phash=' . (int)$phash);
$allRows = array();
$numberOfRows = $db->sql_num_rows($res);
while ($row = $db->sql_fetch_assoc($res)) {
$row['pcount'] = $numberOfRows;
$allRows[] = $row;
}
$db->sql_free_result($res);
return $allRows;
}
/**
* Get number of fulltext records
*
* @param int $phash
* @return int|bool
*/
public function getNumberOfFulltextRecords($phash) {
return $this->getDatabaseConnection()->exec_SELECTcountRows('phash', 'index_fulltext', 'phash=' . (int)$phash);
}
/**
* Get number of words
*
* @param int $phash
* @return int|bool
*/
public function getNumberOfWords($phash) {
return $this->getDatabaseConnection()->exec_SELECTcountRows('*', 'index_rel', 'phash=' . (int)$phash);
}
/**
* Get statistic of external documents
*
* @return array
*/
public function getExternalDocumentsStatistic() {
$result = array();
$db = $this->getDatabaseConnection();
$res = $db->exec_SELECTquery(
'count(*) AS pcount,index_phash.*',
'index_phash',
'item_type<>\'0\'',
'phash_grouping,phash,cHashParams,data_filename,data_page_id,data_page_reg1,data_page_type,data_page_mp,gr_list,item_type,item_title,item_description,item_mtime,tstamp,item_size,contentHash,crdate,parsetime,sys_language_uid,item_crdate,externalUrl,recordUid,freeIndexUid,freeIndexSetId',
'item_type'
);
while ($row = $db->sql_fetch_assoc($res)) {
$this->addAdditionalInformation($row);
$result[] = $row;
if ($row['pcount'] > 1) {
$res2 = $db->exec_SELECTquery(
'index_phash.*',
'index_phash',
'phash_grouping=' . (int)$row['phash_grouping'] . ' AND phash<>' . (int)$row['phash']
);
while ($row2 = $db->sql_fetch_assoc($res2)) {
$this->addAdditionalInformation($row2);
$result[] = $row2;
}
$db->sql_free_result($res2);
}
}
$db->sql_free_result($res);
return $result;
}
/**
* Get count of the tables used for indexed_search
*
* @return array
*/
public function getRecordsNumbers() {
$tables = array(
'index_phash',
'index_words',
'index_rel',
'index_grlist',
'index_section',
'index_fulltext',
);
$recordList = array();
foreach ($tables as $tableName) {
$recordList[$tableName] = $this->getDatabaseConnection()->exec_SELECTcountRows('*', $tableName);