Indexer.php

<?php

/*
 * This file is part of the TYPO3 CMS project.
 *
 * It is free software; you can redistribute it and/or modify it under
 * the terms of the GNU General Public License, either version 2
 * of the License, or any later version.
 *
 * For the full copyright and license information, please read the
 * LICENSE.txt file that was distributed with this source code.
 *
 * The TYPO3 project - inspiring people to share!
 */

namespace TYPO3\CMS\IndexedSearch;

use TYPO3\CMS\Core\Configuration\ExtensionConfiguration;
use TYPO3\CMS\Core\Core\Environment;
use TYPO3\CMS\Core\Database\Connection;
use TYPO3\CMS\Core\Database\ConnectionPool;
use TYPO3\CMS\Core\Html\HtmlParser;
use TYPO3\CMS\Core\Http\RequestFactory;
use TYPO3\CMS\Core\TimeTracker\TimeTracker;
use TYPO3\CMS\Core\Utility\ExtensionManagementUtility;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Core\Utility\MathUtility;
use TYPO3\CMS\Core\Utility\PathUtility;
use TYPO3\CMS\Frontend\Controller\TypoScriptFrontendController;
use TYPO3\CMS\IndexedSearch\Hook\CrawlerFilesHook;
use TYPO3\CMS\IndexedSearch\Utility\IndexedSearchUtility;

/**
 * Indexing class for TYPO3 frontend
 */
class Indexer
{

    /**
     * @var array
     */
    public $reasons = [
        -1 => 'mtime matched the document, so no changes detected and no content updated',
        -2 => 'The minimum age was not exceeded',
        1 => 'The configured max-age was exceeded for the document and thus it\'s indexed.',
        2 => 'The minimum age was exceed and mtime was set and the mtime was different, so the page was indexed.',
        3 => 'The minimum age was exceed, but mtime was not set, so the page was indexed.',
        4 => 'Page has never been indexed (is not represented in the index_phash table).'
    ];

    /**
     * HTML code blocks to exclude from indexing
     *
     * @var string
     */
    public $excludeSections = 'script,style';

    /**
     * Supported Extensions for external files
     *
     * @var array
     */
    public $external_parsers = [];

    /**
     * External parser objects, keys are file extension names. Values are objects with certain methods.
     * Fe-group list (pages might be indexed separately for each usergroup combination to support search
     * in access limited pages!)
     *
     * @var string
     */
    public $defaultGrList = '0,-1';

    /**
     * Min/Max times
     *
     * @var int
     */
    public $tstamp_maxAge = 0;

    /**
     * If set, this tells a number of seconds that is the maximum age of an indexed document.
     * Regardless of mtime the document will be re-indexed if this limit is exceeded.
     *
     * @var int
     */
    public $tstamp_minAge = 0;

    /**
     * If set, this tells a minimum limit before a document can be indexed again. This is regardless of mtime.
     *
     * @var int
     */
    public $maxExternalFiles = 0;

    /**
     * Max number of external files to index.
     *
     * @var bool
     */
    public $forceIndexing = false;

    /**
     * Set when crawler is detected (internal)
     *
     * @var array
     */
    public $defaultContentArray = [
        'title' => '',
        'description' => '',
        'keywords' => '',
        'body' => ''
    ];

    /**
     * @var int
     */
    public $wordcount = 0;

    /**
     * @var int
     */
    public $externalFileCounter = 0;

    /**
     * @var array
     */
    public $conf = [];

    /**
     * Configuration set internally (see init functions for required keys and their meaning)
     *
     * @var array
     */
    public $indexerConfig = [];

    /**
     * Indexer configuration, coming from TYPO3's system configuration for EXT:indexed_search
     *
     * @var array
     */
    public $hash = [];

    /**
     * Hash array, contains phash and phash_grouping
     *
     * @var array
     */
    public $file_phash_arr = [];

    /**
     * Hash array for files
     *
     * @var array
     */
    public $contentParts = [];

    /**
     * Content of TYPO3 page
     *
     * @var string
     */
    public $content_md5h = '';

    /**
     * @var array
     */
    public $internal_log = [];

    /**
     * Internal log
     *
     * @var string
     */
    public $indexExternalUrl_content = '';

    /**
     * @var int
     */
    public $freqRange = 32000;

    /**
     * @var float
     */
    public $freqMax = 0.1;

    /**
     * @var bool
     */
    public $enableMetaphoneSearch = false;

    /**
     * @var bool
     */
    public $storeMetaphoneInfoAsWords;

    /**
     * @var string
     */
    public $metaphoneContent = '';

    /**
     * Metaphone object, if any
     *
     * @var \TYPO3\CMS\IndexedSearch\Utility\DoubleMetaPhoneUtility
     */
    public $metaphoneObj;

    /**
     * Lexer object for word splitting
     *
     * @var \TYPO3\CMS\IndexedSearch\Lexer
     */
    public $lexerObj;

    /**
     * @var bool
     */
    public $flagBitMask;

    /**
     * @var TimeTracker
     */
    protected $timeTracker;

    /**
     * Indexer constructor.
     */
    public function __construct()
    {
        $this->timeTracker = GeneralUtility::makeInstance(TimeTracker::class);
        // Indexer configuration from Extension Manager interface
        $this->indexerConfig = GeneralUtility::makeInstance(ExtensionConfiguration::class)->get('indexed_search');
        $this->tstamp_minAge = MathUtility::forceIntegerInRange((int)($this->indexerConfig['minAge'] ?? 0) * 3600, 0);
        $this->tstamp_maxAge = MathUtility::forceIntegerInRange((int)($this->indexerConfig['maxAge'] ?? 0) * 3600, 0);
        $this->maxExternalFiles = MathUtility::forceIntegerInRange($this->indexerConfig['maxExternalFiles'], 0, 1000, 5);
        $this->flagBitMask = MathUtility::forceIntegerInRange($this->indexerConfig['flagBitMask'], 0, 255);
        // Workaround: If the extension configuration was not updated yet, the value is not existing
        $this->enableMetaphoneSearch = !isset($this->indexerConfig['enableMetaphoneSearch']) || $this->indexerConfig['enableMetaphoneSearch'];
        $this->storeMetaphoneInfoAsWords = !IndexedSearchUtility::isTableUsed('index_words') && $this->enableMetaphoneSearch;
    }

    /********************************
     *
     * Initialization
     *
     *******************************/

    /**
     * Initializes the object.
     * @param array|null $configuration will be used to set $this->conf, otherwise $this->conf MUST be set with proper values prior to this call
     */
    public function init(array $configuration = null)
    {
        if (is_array($configuration)) {
            $this->conf = $configuration;
        }
        // Setting phash / phash_grouping which identifies the indexed page based on some of these variables:
        $this->setT3Hashes();
        // Initialize external document parsers:
        // Example configuration, see ext_localconf.php of this file!
        if ($this->conf['index_externals']) {
            $this->initializeExternalParsers();
        }
        // Initialize lexer (class that deconstructs the text into words):
        $lexerObjectClassName = $GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['indexed_search']['lexer'] ?: Lexer::class;
        $this->lexerObj = GeneralUtility::makeInstance($lexerObjectClassName);
        $this->lexerObj->debug = $this->indexerConfig['debugMode'];
        // Initialize metaphone hook:
        // Make sure that the hook is loaded _after_ indexed_search as this may overwrite the hook depending on the configuration.
        if ($this->enableMetaphoneSearch && $GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['indexed_search']['metaphone']) {
            $this->metaphoneObj = GeneralUtility::makeInstance($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['indexed_search']['metaphone']);
            $this->metaphoneObj->pObj = $this;
        }
    }

    /**
     * Initialize external parsers
     *
     * @internal
     * @see init()
     */
    public function initializeExternalParsers()
    {
        foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['indexed_search']['external_parsers'] ?? [] as $extension => $className) {
            $this->external_parsers[$extension] = GeneralUtility::makeInstance($className);
            $this->external_parsers[$extension]->pObj = $this;
            // Init parser and if it returns FALSE, unset its entry again:
            if (!$this->external_parsers[$extension]->initParser($extension)) {
                unset($this->external_parsers[$extension]);
            }
        }
    }

    /********************************
     *
     * Indexing; TYPO3 pages (HTML content)
     *
     *******************************/
    /**
     * Start indexing of the TYPO3 page
     */
    public function indexTypo3PageContent()
    {
        $check = $this->checkMtimeTstamp($this->conf['mtime'], $this->hash['phash']);
        $is_grlist = $this->is_grlist_set($this->hash['phash']);
        if ($check > 0 || !$is_grlist || $this->forceIndexing) {
            // Setting message:
            if ($this->forceIndexing) {
                $this->log_setTSlogMessage('Indexing needed, reason: Forced', 1);
            } elseif ($check > 0) {
                $this->log_setTSlogMessage('Indexing needed, reason: ' . $this->reasons[$check], 1);
            } else {
                $this->log_setTSlogMessage('Indexing needed, reason: Updates gr_list!', 1);
            }
            // Divide into title,keywords,description and body:
            $this->log_push('Split content', '');
            $this->contentParts = $this->splitHTMLContent($this->conf['content']);
            if ($this->conf['indexedDocTitle']) {
                $this->contentParts['title'] = $this->conf['indexedDocTitle'];
            }
            $this->log_pull();
            // Calculating a hash over what is to be the actual page content. Maybe this hash should not include title,description and keywords? The bodytext is the primary concern. (on the other hand a changed page-title would make no difference then, so don't!)
            $this->content_md5h = IndexedSearchUtility::md5inthash(implode('', $this->contentParts));
            // This function checks if there is already a page (with gr_list = 0,-1) indexed and if that page has the very same contentHash.
            // If the contentHash is the same, then we can rest assured that this page is already indexed and regardless of mtime and origContent we don't need to do anything more.
            // This will also prevent pages from being indexed if a fe_users has logged in and it turns out that the page content is not changed anyway. fe_users logged in should always search with hash_gr_list = "0,-1" OR "[their_group_list]". This situation will be prevented only if the page has been indexed with no user login on before hand. Else the page will be indexed by users until that event. However that does not present a serious problem.
            $checkCHash = $this->checkContentHash();
            if (!is_array($checkCHash) || $check === 1) {
                $Pstart = IndexedSearchUtility::milliseconds();
                $this->log_push('Converting charset of content (' . $this->conf['metaCharset'] . ') to utf-8', '');
                $this->charsetEntity2utf8($this->contentParts, $this->conf['metaCharset']);
                $this->log_pull();
                // Splitting words
                $this->log_push('Extract words from content', '');
                $splitInWords = $this->processWordsInArrays($this->contentParts);
                $this->log_pull();
                // Analyze the indexed words.
                $this->log_push('Analyze the extracted words', '');
                $indexArr = $this->indexAnalyze($splitInWords);
                $this->log_pull();
                // Submitting page (phash) record
                $this->log_push('Submitting page', '');
                $this->submitPage();
                $this->log_pull();
                // Check words and submit to word list if not there
                $this->log_push('Check word list and submit words', '');
                if (IndexedSearchUtility::isTableUsed('index_words')) {
                    $this->checkWordList($indexArr);
                    $this->submitWords($indexArr, $this->hash['phash']);
                }
                $this->log_pull();
                // Set parsetime
                $this->updateParsetime($this->hash['phash'], IndexedSearchUtility::milliseconds() - $Pstart);
                // Checking external files if configured for.
                $this->log_push('Checking external files', '');
                if ($this->conf['index_externals']) {
                    $this->extractLinks($this->conf['content']);
                }
                $this->log_pull();
            } else {
                // Update the timestamp
                $this->updateTstamp($this->hash['phash'], $this->conf['mtime']);
                $this->updateSetId($this->hash['phash']);
                // $checkCHash['phash'] is the phash of the result row that is similar to the current phash regarding the content hash.
                $this->update_grlist($checkCHash['phash'], $this->hash['phash']);
                $this->updateRootline();
                $this->log_setTSlogMessage('Indexing not needed, the contentHash, ' . $this->content_md5h . ', has not changed. Timestamp, grlist and rootline updated if necessary.');
            }
        } else {
            $this->log_setTSlogMessage('Indexing not needed, reason: ' . $this->reasons[$check]);
        }
    }

    /**
     * Splits HTML content and returns an associative array, with title, a list of metatags, and a list of words in the body.
     *
     * @param string $content HTML content to index. To some degree expected to be made by TYPO3 (ei. splitting the header by ":")
     * @return array Array of content, having keys "title", "body", "keywords" and "description" set.
     * @see splitRegularContent()
     */
    public function splitHTMLContent($content)
    {
        // divide head from body ( u-ouh :) )
        $contentArr = $this->defaultContentArray;
        $contentArr['body'] = stristr($content, '<body');
        $headPart = substr($content, 0, -strlen($contentArr['body']));
        // get title
        $this->embracingTags($headPart, 'TITLE', $contentArr['title'], $dummy2, $dummy);
        $titleParts = explode(':', $contentArr['title'], 2);
        $contentArr['title'] = trim($titleParts[1] ?? $titleParts[0]);
        // get keywords and description metatags
        if ($this->conf['index_metatags']) {
            $meta = [];
            $i = 0;
            while ($this->embracingTags($headPart, 'meta', $dummy, $headPart, $meta[$i])) {
                $i++;
            }
            // @todo The code below stops at first unset tag. Is that correct?
            for ($i = 0; isset($meta[$i]); $i++) {
                // decode HTML entities, meta tag content needs to be encoded later
                $meta[$i] = GeneralUtility::get_tag_attributes($meta[$i], true);
                if (stripos($meta[$i]['name'], 'keywords') !== false) {
                    $contentArr['keywords'] .= ',' . $this->addSpacesToKeywordList($meta[$i]['content']);
                }
                if (stripos($meta[$i]['name'], 'description') !== false) {
                    $contentArr['description'] .= ',' . $meta[$i]['content'];
                }
            }
        }
        // Process <!--TYPO3SEARCH_begin--> or <!--TYPO3SEARCH_end--> tags:
        $this->typoSearchTags($contentArr['body']);
        // Get rid of unwanted sections (ie. scripting and style stuff) in body
        $tagList = explode(',', $this->excludeSections);
        foreach ($tagList as $tag) {
            while ($this->embracingTags($contentArr['body'], $tag, $dummy, $contentArr['body'], $dummy2)) {
            }
        }
        // remove tags, but first make sure we don't concatenate words by doing it
        $contentArr['body'] = str_replace('<', ' <', $contentArr['body']);
        $contentArr['body'] = trim(strip_tags($contentArr['body']));
        $contentArr['keywords'] = trim($contentArr['keywords']);
        $contentArr['description'] = trim($contentArr['description']);
        // Return array
        return $contentArr;
    }

    /**
     * Extract the charset value from HTML meta tag.
     *
     * @param string $content HTML content
     * @return string The charset value if found.
     */
    public function getHTMLcharset($content)
    {
        if (preg_match('/<meta[[:space:]]+[^>]*http-equiv[[:space:]]*=[[:space:]]*["\']CONTENT-TYPE["\'][^>]*>/i', $content, $reg)) {
            if (preg_match('/charset[[:space:]]*=[[:space:]]*([[:alnum:]-]+)/i', $reg[0], $reg2)) {
                return $reg2[1];
            }
        }

        return '';
    }

    /**
     * Converts a HTML document to utf-8
     *
     * @param string $content HTML content, any charset
     * @param string $charset Optional charset (otherwise extracted from HTML)
     * @return string Converted HTML
     */
    public function convertHTMLToUtf8($content, $charset = '')
    {
        // Find charset:
        $charset = $charset ?: $this->getHTMLcharset($content);
        $charset = trim(strtolower($charset));
        // Convert charset:
        if ($charset && $charset !== 'utf-8') {
            $content = mb_convert_encoding($content, 'utf-8', $charset);
        }
        // Convert entities, assuming document is now UTF-8:
        return html_entity_decode($content);
    }

    /**
     * Finds first occurrence of embracing tags and returns the embraced content and the original string with
     * the tag removed in the two passed variables. Returns FALSE if no match found. ie. useful for finding
     * <title> of document or removing <script>-sections
     *
     * @param string $string String to search in
     * @param string $tagName Tag name, eg. "script
     * @param string $tagContent Passed by reference: Content inside found tag
     * @param string $stringAfter Passed by reference: Content after found tag
     * @param string $paramList Passed by reference: Attributes of the found tag.
     * @return bool Returns FALSE if tag was not found, otherwise TRUE.
     */
    public function embracingTags($string, $tagName, &$tagContent, &$stringAfter, &$paramList)
    {
        $endTag = '</' . $tagName . '>';
        $startTag = '<' . $tagName;
        // stristr used because we want a case-insensitive search for the tag.
        $isTagInText = stristr($string, $startTag);
        // if the tag was not found, return FALSE
        if (!$isTagInText) {
            return false;
        }
        [$paramList, $isTagInText] = explode('>', substr($isTagInText, strlen($startTag)), 2);
        $afterTagInText = stristr($isTagInText, $endTag);
        if ($afterTagInText) {
            $stringBefore = substr($string, 0, strpos(strtolower($string), strtolower($startTag)));
            $tagContent = substr($isTagInText, 0, strlen($isTagInText) - strlen($afterTagInText));
            $stringAfter = $stringBefore . substr($afterTagInText, strlen($endTag));
        } else {
            $tagContent = '';
            $stringAfter = $isTagInText;
        }
        return true;
    }

    /**
     * Removes content that shouldn't be indexed according to TYPO3SEARCH-tags.
     *
     * @param string $body HTML Content, passed by reference
     * @return bool Returns TRUE if a TYPOSEARCH_ tag was found, otherwise FALSE.
     */
    public function typoSearchTags(&$body)
    {
        $expBody = preg_split('/\\<\\!\\-\\-[\\s]?TYPO3SEARCH_/', $body);
        if (count($expBody) > 1) {
            $body = '';
            $prev = '';
            foreach ($expBody as $val) {
                $part = explode('-->', $val, 2);
                if (trim($part[0]) === 'begin') {
                    $body .= $part[1];
                    $prev = '';
                } elseif (trim($part[0]) === 'end') {
                    $body .= $prev;
                } else {
                    $prev = $val;
                }
            }
            return true;
        }
        return false;
    }

    /**
     * Extract links (hrefs) from HTML content and if indexable media is found, it is indexed.
     *
     * @param string $content HTML content
     */
    public function extractLinks($content)
    {
        $crawler = null;
        // Get links:
        $list = $this->extractHyperLinks($content);
        if ($this->indexerConfig['useCrawlerForExternalFiles'] && ExtensionManagementUtility::isLoaded('crawler')) {
            /**
             * todo: remove dependency to class tx_crawler_lib
             * @link https://forge.typo3.org/issues/83603
             */
            $crawler = GeneralUtility::makeInstance('tx_crawler_lib');
        }
        // Traverse links:
        foreach ($list as $linkInfo) {
            // Decode entities:
            if ($linkInfo['localPath']) {
                // localPath means: This file is sent by a download script. While the indexed URL has to point to $linkInfo['href'], the absolute path to the file is specified here!
                $linkSource = htmlspecialchars_decode($linkInfo['localPath']);
            } else {
                $linkSource = htmlspecialchars_decode($linkInfo['href']);
            }
            // Parse URL:
            $qParts = parse_url($linkSource);
            // Check for jumpurl (TYPO3 specific thing...)
            if ($qParts['query'] && strpos($qParts['query'], 'jumpurl=') !== false) {
                parse_str($qParts['query'], $getP);
                $linkSource = $getP['jumpurl'];
                $qParts = parse_url($linkSource);
            }
            if (!$linkInfo['localPath'] && $qParts['scheme']) {
                if ($this->indexerConfig['indexExternalURLs']) {
                    // Index external URL (http or otherwise)
                    $this->indexExternalUrl($linkSource);
                }
            } elseif (!$qParts['query']) {
                $linkSource = urldecode($linkSource);
                if (GeneralUtility::isAllowedAbsPath($linkSource)) {
                    $localFile = $linkSource;
                } else {
                    $localFile = GeneralUtility::getFileAbsFileName(Environment::getPublicPath() . '/' . $linkSource);
                }
                if ($localFile && @is_file($localFile)) {
                    // Index local file:
                    if ($linkInfo['localPath']) {
                        $fI = pathinfo($linkSource);
                        $ext = strtolower($fI['extension']);
                        if (is_object($crawler)) {
                            $params = [
                                'document' => $linkSource,
                                'alturl' => $linkInfo['href'],
                                'conf' => $this->conf
                            ];
                            unset($params['conf']['content']);
                            $crawler->addQueueEntry_callBack(0, $params, CrawlerFilesHook::class, $this->conf['id']);
                            $this->log_setTSlogMessage('media "' . $params['document'] . '" added to "crawler" queue.', 1);
                        } else {
                            $this->indexRegularDocument($linkInfo['href'], false, $linkSource, $ext);
                        }
                    } else {
                        if (is_object($crawler)) {
                            $params = [
                                'document' => $linkSource,
                                'conf' => $this->conf
                            ];
                            unset($params['conf']['content']);
                            $crawler->addQueueEntry_callBack(0, $params, CrawlerFilesHook::class, $this->conf['id']);
                            $this->log_setTSlogMessage('media "' . $params['document'] . '" added to "crawler" queue.', 1);
                        } else {
                            $this->indexRegularDocument($linkSource);
                        }
                    }
                }
            }
        }
    }

    /**
     * Extracts all links to external documents from the HTML content string
     *
     * @param string $html
     * @return array Array of hyperlinks (keys: tag, href, localPath (empty if not local))
     * @see extractLinks()
     */
    public function extractHyperLinks($html)
    {
        $htmlParser = GeneralUtility::makeInstance(HtmlParser::class);
        $htmlParts = $htmlParser->splitTags('a', $html);
        $hyperLinksData = [];
        foreach ($htmlParts as $index => $tagData) {
            if ($index % 2 !== 0) {
                $tagAttributes = $htmlParser->get_tag_attributes($tagData, true);
                $firstTagName = $htmlParser->getFirstTagName($tagData);
                if (strtolower($firstTagName) === 'a') {
                    if ($tagAttributes[0]['href'] && $tagAttributes[0]['href'][0] !== '#') {
                        $hyperLinksData[] = [
                            'tag' => $tagData,
                            'href' => $tagAttributes[0]['href'],
                            'localPath' => $this->createLocalPath(urldecode($tagAttributes[0]['href']))
                        ];
                    }
                }
            }
        }
        return $hyperLinksData;
    }

    /**
     * Extracts the "base href" from content string.
     *
     * @param string $html Content to analyze
     * @return string The base href or an empty string if not found
     */
    public function extractBaseHref($html)
    {
        $href = '';
        $htmlParser = GeneralUtility::makeInstance(HtmlParser::class);
        $htmlParts = $htmlParser->splitTags('base', $html);
        foreach ($htmlParts as $index => $tagData) {
            if ($index % 2 !== 0) {
                $tagAttributes = $htmlParser->get_tag_attributes($tagData, true);
                $firstTagName = $htmlParser->getFirstTagName($tagData);
                if (strtolower($firstTagName) === 'base') {
                    $href = $tagAttributes[0]['href'];
                    if ($href) {
                        break;
                    }
                }
            }
        }
        return $href;
    }

    /******************************************
     *
     * Indexing; external URL
     *
     ******************************************/
    /**
     * Index External URLs HTML content
     *
     * @param string $externalUrl URL, eg. "https://typo3.org/
     * @see indexRegularDocument()
     */
    public function indexExternalUrl($externalUrl)
    {
        // Get headers:
        $urlHeaders = $this->getUrlHeaders($externalUrl);
        if (is_array($urlHeaders) && stripos($urlHeaders['Content-Type'], 'text/html') !== false) {
            $content = ($this->indexExternalUrl_content = GeneralUtility::getUrl($externalUrl));
            if ((string)$content !== '') {
                // Create temporary file:
                $tmpFile = GeneralUtility::tempnam('EXTERNAL_URL');
                if ($tmpFile) {
                    GeneralUtility::writeFile($tmpFile, $content);
                    // Index that file:
                    $this->indexRegularDocument($externalUrl, true, $tmpFile, 'html');
                    // Using "TRUE" for second parameter to force indexing of external URLs (mtime doesn't make sense, does it?)
                    unlink($tmpFile);
                }
            }
        }
    }

    /**
     * Getting HTTP request headers of URL
     *
     * @param string $url The URL
     * @return mixed If no answer, returns FALSE. Otherwise an array where HTTP headers are keys
     */
    public function getUrlHeaders($url)
    {
        try {
            $response = GeneralUtility::makeInstance(RequestFactory::class)->request($url, 'HEAD');
            $headers = $response->getHeaders();
            $retVal = [];
            foreach ($headers as $key => $value) {
                $retVal[$key] = implode('', $value);
            }
            return $retVal;
        } catch (\Exception $e) {
            // fail silently if the HTTP request failed
            return false;
        }
    }

    /**
     * Checks if the file is local
     *
     * @param string $sourcePath
     * @return string Absolute path to file if file is local, else empty string
     */
    protected function createLocalPath($sourcePath)
    {
        $localPath = '';
        $pathFunctions = [
            'createLocalPathUsingAbsRefPrefix',
            'createLocalPathUsingDomainURL',
            'createLocalPathFromAbsoluteURL',
            'createLocalPathFromRelativeURL'
        ];
        foreach ($pathFunctions as $functionName) {
            $localPath = $this->{$functionName}($sourcePath);
            if ($localPath != '') {
                break;
            }
        }
        return $localPath;
    }

    /**
     * Attempts to create a local file path by matching a current request URL.
     *
     * @param string $sourcePath
     * @return string
     */
    protected function createLocalPathUsingDomainURL($sourcePath)
    {
        $localPath = '';
        $baseURL = GeneralUtility::getIndpEnv('TYPO3_SITE_URL');
        $baseURLLength = strlen($baseURL);
        if (strpos($sourcePath, $baseURL) === 0) {
            $sourcePath = substr($sourcePath, $baseURLLength);
            $localPath = Environment::getPublicPath() . '/' . $sourcePath;
            if (!self::isAllowedLocalFile($localPath)) {
                $localPath = '';
            }
        }
        return $localPath;
    }

    /**
     * Attempts to create a local file path by matching absRefPrefix. This
     * requires TSFE. If TSFE is missing, this function does nothing.
     *
     * @param string $sourcePath
     * @return string
     */
    protected function createLocalPathUsingAbsRefPrefix($sourcePath)
    {
        $localPath = '';
        if (isset($GLOBALS['TSFE']) && $GLOBALS['TSFE'] instanceof TypoScriptFrontendController) {
            $absRefPrefix = $GLOBALS['TSFE']->config['config']['absRefPrefix'];
            $absRefPrefixLength = strlen($absRefPrefix);
            if ($absRefPrefixLength > 0 && strpos($sourcePath, $absRefPrefix) === 0) {
                $sourcePath = substr($sourcePath, $absRefPrefixLength);
                $localPath = Environment::getPublicPath() . '/' . $sourcePath;
                if (!self::isAllowedLocalFile($localPath)) {
                    $localPath = '';
                }
            }
        }
        return $localPath;
    }

    /**
     * Attempts to create a local file path from the absolute URL without
     * schema.
     *
     * @param string $sourcePath
     * @return string
     */
    protected function createLocalPathFromAbsoluteURL($sourcePath)
    {
        $localPath = '';
        if ($sourcePath[0] === '/') {
            $sourcePath = substr($sourcePath, 1);
            $localPath = Environment::getPublicPath() . '/' . $sourcePath;
            if (!self::isAllowedLocalFile($localPath)) {
                $localPath = '';
            }
        }
        return $localPath;
    }

    /**
     * Attempts to create a local file path from the relative URL.
     *
     * @param string $sourcePath
     * @return string
     */
    protected function createLocalPathFromRelativeURL($sourcePath)
    {
        $localPath = '';
        if (self::isRelativeURL($sourcePath)) {
            $localPath = Environment::getPublicPath() . '/' . $sourcePath;
            if (!self::isAllowedLocalFile($localPath)) {
                $localPath = '';
            }
        }
        return $localPath;
    }

    /**
     * Checks if URL is relative.
     *
     * @param string $url
     * @return bool
     */
    protected static function isRelativeURL($url)
    {
        $urlParts = @parse_url($url);
        return (!isset($urlParts['scheme']) || $urlParts['scheme'] === '') && $urlParts['path'][0] !== '/';
    }

    /**
     * Checks if the path points to the file inside the web site
     *
     * @param string $filePath
     * @return bool
     */
    protected static function isAllowedLocalFile($filePath)
    {
        $filePath = GeneralUtility::resolveBackPath($filePath);
        $insideWebPath = strpos($filePath, Environment::getPublicPath()) === 0;
        $isFile = is_file($filePath);
        return $insideWebPath && $isFile;
    }

    /******************************************
     *
     * Indexing; external files (PDF, DOC, etc)
     *
     ******************************************/
    /**
     * Indexing a regular document given as $file (relative to public web path, local file)
     *
     * @param string $file Relative Filename, relative to public web path. It can also be an absolute path as long as it is inside the lockRootPath (validated with \TYPO3\CMS\Core\Utility\GeneralUtility::isAbsPath()). Finally, if $contentTmpFile is set, this value can be anything, most likely a URL
     * @param bool $force If set, indexing is forced (despite content hashes, mtime etc).
     * @param string $contentTmpFile Temporary file with the content to read it from (instead of $file). Used when the $file is a URL.
     * @param string $altExtension File extension for temporary file.
     */
    public function indexRegularDocument($file, $force = false, $contentTmpFile = '', $altExtension = '')
    {
        // Init
        $fI = pathinfo($file);
        $ext = $altExtension ?: strtolower($fI['extension']);
        // Create abs-path:
        if (!$contentTmpFile) {
            if (!GeneralUtility::isAbsPath($file)) {
                // Relative, prepend public web path:
                $absFile = GeneralUtility::getFileAbsFileName(Environment::getPublicPath() . '/' . $file);
            } else {
                // Absolute, pass-through:
                $absFile = $file;
            }
            $absFile = GeneralUtility::isAllowedAbsPath($absFile) ? $absFile : '';
        } else {
            $absFile = $contentTmpFile;
        }
        // Indexing the document:
        if ($absFile && @is_file($absFile)) {
            if ($this->external_parsers[$ext]) {
                $fileInfo = stat($absFile);
                $cParts = $this->fileContentParts($ext, $absFile);
                foreach ($cParts as $cPKey) {
                    $this->internal_log = [];
                    $this->log_push('Index: ' . str_replace('.', '_', PathUtility::basename($file)) . ($cPKey ? '#' . $cPKey : ''), '');
                    $Pstart = IndexedSearchUtility::milliseconds();
                    $subinfo = ['key' => $cPKey];
                    // Setting page range. This is "0" (zero) when no division is made, otherwise a range like "1-3"
                    $phash_arr = ($this->file_phash_arr = $this->setExtHashes($file, $subinfo));
                    $check = $this->checkMtimeTstamp($fileInfo['mtime'], $phash_arr['phash']);
                    if ($check > 0 || $force) {
                        if ($check > 0) {
                            $this->log_setTSlogMessage('Indexing needed, reason: ' . $this->reasons[$check], 1);
                        } else {
                            $this->log_setTSlogMessage('Indexing forced by flag', 1);
                        }
                        // Check external file counter:
                        if ($this->externalFileCounter < $this->maxExternalFiles || $force) {
                            // Divide into title,keywords,description and body:
                            $this->log_push('Split content', '');
                            $contentParts = $this->readFileContent($ext, $absFile, $cPKey);
                            $this->log_pull();
                            if (is_array($contentParts)) {
                                // Calculating a hash over what is to be the actual content. (see indexTypo3PageContent())
                                $content_md5h = IndexedSearchUtility::md5inthash(implode('', $contentParts));
                                if ($this->checkExternalDocContentHash($phash_arr['phash_grouping'], $content_md5h) || $force) {
                                    // Increment counter:
                                    $this->externalFileCounter++;
                                    // Splitting words
                                    $this->log_push('Extract words from content', '');
                                    $splitInWords = $this->processWordsInArrays($contentParts);
                                    $this->log_pull();
                                    // Analyze the indexed words.
                                    $this->log_push('Analyze the extracted words', '');
                                    $indexArr = $this->indexAnalyze($splitInWords);
                                    $this->log_pull();
                                    // Submitting page (phash) record
                                    $this->log_push('Submitting page', '');
                                    // Unfortunately I cannot determine WHEN a file is originally made - so I must return the modification time...
                                    $this->submitFilePage($phash_arr, $file, $subinfo, $ext, $fileInfo['mtime'], $fileInfo['ctime'], $fileInfo['size'], $content_md5h, $contentParts);
                                    $this->log_pull();
                                    // Check words and submit to word list if not there
                                    $this->log_push('Check word list and submit words', '');
                                    if (IndexedSearchUtility::isTableUsed('index_words')) {
                                        $this->checkWordList($indexArr);
                                        $this->submitWords($indexArr, $phash_arr['phash']);
                                    }
                                    $this->log_pull();
                                    // Set parsetime
                                    $this->updateParsetime($phash_arr['phash'], IndexedSearchUtility::milliseconds() - $Pstart);
                                } else {
                                    // Update the timestamp
                                    $this->updateTstamp($phash_arr['phash'], $fileInfo['mtime']);
                                    $this->log_setTSlogMessage('Indexing not needed, the contentHash, ' . $content_md5h . ', has not changed. Timestamp updated.');
                                }
                            } else {
                                $this->log_setTSlogMessage('Could not index file! Unsupported extension.');
                            }
                        } else {
                            $this->log_setTSlogMessage('The limit of ' . $this->maxExternalFiles . ' has already been exceeded, so no indexing will take place this time.');
                        }
                    } else {
                        $this->log_setTSlogMessage('Indexing not needed, reason: ' . $this->reasons[$check]);
                    }
                    // Checking and setting sections:
                    $this->submitFile_section($phash_arr['phash']);
                    // Setting a section-record for the file. This is done also if the file is not indexed. Notice that section records are deleted when the page is indexed.
                    $this->log_pull();
                }
            } else {
                $this->log_setTSlogMessage('Indexing not possible; The extension "' . $ext . '" was not supported.');
            }
        } else {
            $this->log_setTSlogMessage('Indexing not possible; File "' . $absFile . '" not found or valid.');
        }
    }

    /**
     * Reads the content of an external file being indexed.
     * The content from the external parser MUST be returned in utf-8!
     *
     * @param string $fileExtension File extension, eg. "pdf", "doc" etc.
     * @param string $absoluteFileName Absolute filename of file (must exist and be validated OK before calling function)
     * @param string $sectionPointer Pointer to section (zero for all other than PDF which will have an indication of pages into which the document should be splitted.)
     * @return array Standard content array (title, description, keywords, body keys)
     */
    public function readFileContent($fileExtension, $absoluteFileName, $sectionPointer)
    {
        $contentArray = null;
        // Consult relevant external document parser:
        if (is_object($this->external_parsers[$fileExtension])) {
            $contentArray = $this->external_parsers[$fileExtension]->readFileContent($fileExtension, $absoluteFileName, $sectionPointer);
        }
        return $contentArray;
    }

    /**
     * Creates an array with pointers to divisions of document.
     *
     * @param string $ext File extension
     * @param string $absFile Absolute filename (must exist and be validated OK before calling function)
     * @return array Array of pointers to sections that the document should be divided into
     */
    public function fileContentParts($ext, $absFile)
    {
        $cParts = [0];
        // Consult relevant external document parser:
        if (is_object($this->external_parsers[$ext])) {
            $cParts = $this->external_parsers[$ext]->fileContentParts($ext, $absFile);
        }
        return $cParts;
    }

    /**
     * Splits non-HTML content (from external files for instance)
     *