[TASK] Apply PSR-2 11/47711/2
authorFrancois Suter <francois@typo3.org>
Fri, 15 Apr 2016 15:07:55 +0000 (17:07 +0200)
committerFrancois Suter <francois@typo3.org>
Fri, 15 Apr 2016 15:17:46 +0000 (17:17 +0200)
Code cleanup, use ::class syntax, use IconManager, enforce
strict comparisons wherever possible, change usage of
FlashMessage.

Resolves: #75678
Releases: 2.0
Change-Id: I8e0483b97fe0c7e5f4a7f99c7522c56e63d1b885
Reviewed-on: https://review.typo3.org/47711
Reviewed-by: Francois Suter <francois@typo3.org>
Tested-by: Francois Suter <francois@typo3.org>
28 files changed:
ChangeLog
Classes/Ajax/AjaxHandler.php
Classes/Cache/CacheHandler.php
Classes/Cache/CacheParametersProcessorInterface.php
Classes/Component/DataProvider.php
Classes/Exception/InvalidQueryException.php
Classes/Hook/DataFilterHook.php
Classes/Parser/FulltextParser.php
Classes/Parser/QueryParser.php
Classes/Parser/SqlParser.php
Classes/Sample/DataQueryHook.php
Classes/UserFunction/FormEngine.php
Classes/Utility/DatabaseAnalyser.php
Classes/Utility/QueryObject.php
Classes/Utility/SqlUtility.php
Classes/Wizard/QueryCheckWizard.php
Configuration/TCA/tx_dataquery_queries.php
LICENSE.txt [new file with mode: 0644]
Tests/Unit/DataProviderTest.php
Tests/Unit/QueryParserTest.php
Tests/Unit/SqlBuilderDefaultTest.php
Tests/Unit/SqlBuilderLanguageTest.php
Tests/Unit/SqlBuilderTest.php
Tests/Unit/SqlBuilderWorkspaceTest.php
Tests/Unit/SqlParserTest.php
composer.json
ext_localconf.php
ext_tables.php

index 112bb3a..a243b04 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,7 @@
+2016-04-15 Francois Suter (Cobweb)  <typo3@cobweb.ch>
+
+       * Applied PSR-2 formatting and other cleanups, resolves #75678
+
 2015-09-27 Francois Suter (Cobweb)  <typo3@cobweb.ch>
 
        * Fixed crashing query check wizard, resolves #70136
index 27cee40..5058b3b 100644 (file)
@@ -14,8 +14,10 @@ namespace Tesseract\Dataquery\Ajax;
  * The TYPO3 project - inspiring people to share!
  */
 
+use Tesseract\Dataquery\Parser\QueryParser;
 use TYPO3\CMS\Core\Http\AjaxRequestHandler;
 use TYPO3\CMS\Core\Messaging\FlashMessage;
+use TYPO3\CMS\Core\Messaging\FlashMessageQueue;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 
 /**
@@ -25,107 +27,118 @@ use TYPO3\CMS\Core\Utility\GeneralUtility;
  * @package TYPO3
  * @subpackage tx_dataquery
  */
-class AjaxHandler {
+class AjaxHandler
+{
 
-       /**
-        * Returns the parsed query through dataquery parser
-        * or error messages from exceptions should any have been thrown
-        * during query parsing.
-        *
-        * @param array $parameters Empty array (yes, that's weird but true)
-        * @param AjaxRequestHandler $ajaxObj Back-reference to the calling object
-        * @return      void
-        */
-       public function validate($parameters, AjaxRequestHandler $ajaxObj) {
-               $parsingSeverity = FlashMessage::OK;
-               $executionSeverity = FlashMessage::OK;
-               $executionMessage = '';
-               $warningMessage = '';
-               /** @var \TYPO3\CMS\Lang\LanguageService $languageService */
-               $languageService = $GLOBALS['LANG'];
+    /**
+     * Returns the parsed query through dataquery parser
+     * or error messages from exceptions should any have been thrown
+     * during query parsing.
+     *
+     * @param array $parameters Empty array (yes, that's weird but true)
+     * @param AjaxRequestHandler $ajaxObj Back-reference to the calling object
+     * @return    void
+     */
+    public function validate($parameters, AjaxRequestHandler $ajaxObj)
+    {
+        $flashMessageQueue = GeneralUtility::makeInstance(
+                FlashMessageQueue::class,
+                'tx_datafilter_ajax'
+        );
+        $parsingSeverity = FlashMessage::OK;
+        $executionSeverity = FlashMessage::OK;
+        $executionMessage = '';
+        $warningMessage = '';
+        /** @var \TYPO3\CMS\Lang\LanguageService $languageService */
+        $languageService = $GLOBALS['LANG'];
 
-               // Try parsing and building the query
-               try {
-                       // Get the query to parse from the GET/POST parameters
-                       $query = GeneralUtility::_GP('query');
-                       // Create an instance of the parser
-                       // NOTE: NULL is passed for the parent object as there's no controller in this context,
-                       // but that's a bit risky. Maybe extension "tesseract" could provide a dummy controller
-                       // (or some logic should be split: the query parser should not also be a query builder).
-                       /** @var $parser \Tesseract\Dataquery\Parser\QueryParser */
-                       $parser = GeneralUtility::makeInstance('Tesseract\\Dataquery\\Parser\\QueryParser', NULL);
-                       // Clean up and prepare the query string
-                       $query = $parser->prepareQueryString($query);
-                       // Parse the query
-                       // NOTE: if the parsing fails, an exception will be received, which is handled further down
-                       // The parser may return a warning, though
-                       $warningMessage = $parser->parseQuery($query);
-                       // Build the query
-                       $parsedQuery = $parser->buildQuery();
-                       // The query building completed, issue success message
-                       $parsingTitle = $languageService->sL('LLL:EXT:dataquery/locallang.xml:query.success');
-                       $parsingMessage = $parsedQuery;
+        // Try parsing and building the query
+        try {
+            // Get the query to parse from the GET/POST parameters
+            $query = GeneralUtility::_GP('query');
+            // Create an instance of the parser
+            // NOTE: NULL is passed for the parent object as there's no controller in this context,
+            // but that's a bit risky. Maybe extension "tesseract" could provide a dummy controller
+            // (or some logic should be split: the query parser should not also be a query builder).
+            /** @var $parser QueryParser */
+            $parser = GeneralUtility::makeInstance(
+                    QueryParser::class,
+                    null
+            );
+            // Clean up and prepare the query string
+            $query = $parser->prepareQueryString($query);
+            // Parse the query
+            // NOTE: if the parsing fails, an exception will be received, which is handled further down
+            // The parser may return a warning, though
+            $warningMessage = $parser->parseQuery($query);
+            // Build the query
+            $parsedQuery = $parser->buildQuery();
+            // The query building completed, issue success message
+            $parsingTitle = $languageService->sL('LLL:EXT:dataquery/locallang.xml:query.success');
+            $parsingMessage = $parsedQuery;
 
-                       // Force a LIMIT to 1 and try executing the query
-                       $parser->getSQLObject()->structure['LIMIT'] = 1;
-                       // Rebuild the query with the new limit
-                       $executionQuery = $parser->buildQuery();
-                       // Execute query and report outcome
-                       /** @var \TYPO3\CMS\Core\Database\DatabaseConnection $databaseConnection */
-                       $databaseConnection = $GLOBALS['TYPO3_DB'];
-                       $res = $databaseConnection->sql_query($executionQuery);
-                       if ($res === FALSE) {
-                               $executionSeverity = FlashMessage::ERROR;
-                               $errorMessage = $databaseConnection->sql_error();
-                               $executionMessage = sprintf(
-                                       $languageService->sL('LLL:EXT:dataquery/locallang.xml:query.executionFailed'),
-                                       $errorMessage
-                               );
-                       } else {
-                               $executionMessage = $languageService->sL('LLL:EXT:dataquery/locallang.xml:query.executionSuccessful');
-                       }
-               }
-               catch(\Exception $e) {
-                       // The query parsing failed, issue error message
-                       $parsingSeverity = FlashMessage::ERROR;
-                       $parsingTitle = $languageService->sL('LLL:EXT:dataquery/locallang.xml:query.failure');
-                       $exceptionCode = $e->getCode();
-                       // Display "improved" exception message (if available)
-                       $parsingMessage = $languageService->sL('LLL:EXT:dataquery/locallang.xml:query.exception-' . $exceptionCode);
-                       // If some unexpected exception occurred, display original message
-                       if (empty($parsingMessage)) {
-                               $parsingMessage = $e->getMessage();
-                       }
-               }
-               // Render parsing result as flash message
-               /** @var $flashMessage FlashMessage */
-               $flashMessage = GeneralUtility::makeInstance(
-                       'TYPO3\\CMS\\Core\\Messaging\\FlashMessage',
-                       $parsingMessage,
-                       $parsingTitle,
-                       $parsingSeverity
-               );
-               $content = $flashMessage->render();
-               // If a warning was returned by the query parser, display it here
-               if (!empty($warningMessage)) {
-                       $flashMessage = GeneralUtility::makeInstance(
-                               'TYPO3\\CMS\\Core\\Messaging\\FlashMessage',
-                               $warningMessage,
-                                       $languageService->sL('LLL:EXT:dataquery/locallang.xml:query.warning'),
-                               FlashMessage::WARNING
-                       );
-                       $content .= $flashMessage->render();
-               }
-               // If the query was also executed, render execution result
-               if (!empty($executionMessage)) {
-                       $flashMessage = GeneralUtility::makeInstance(
-                               'TYPO3\\CMS\\Core\\Messaging\\FlashMessage',
-                               $executionMessage,
-                               '',
-                               $executionSeverity
-                       );
-                       $content .= $flashMessage->render();
-               }
-               $ajaxObj->addContent('dataquery', $content);
-       }
+            // Force a LIMIT to 1 and try executing the query
+            $parser->getSQLObject()->structure['LIMIT'] = 1;
+            // Rebuild the query with the new limit
+            $executionQuery = $parser->buildQuery();
+            // Execute query and report outcome
+            /** @var \TYPO3\CMS\Core\Database\DatabaseConnection $databaseConnection */
+            $databaseConnection = $GLOBALS['TYPO3_DB'];
+            $res = $databaseConnection->sql_query($executionQuery);
+            if ($res === false) {
+                $executionSeverity = FlashMessage::ERROR;
+                $errorMessage = $databaseConnection->sql_error();
+                $executionMessage = sprintf(
+                        $languageService->sL('LLL:EXT:dataquery/locallang.xml:query.executionFailed'),
+                        $errorMessage
+                );
+            } else {
+                $executionMessage = $languageService->sL('LLL:EXT:dataquery/locallang.xml:query.executionSuccessful');
+            }
+        } catch (\Exception $e) {
+            // The query parsing failed, issue error message
+            $parsingSeverity = FlashMessage::ERROR;
+            $parsingTitle = $languageService->sL('LLL:EXT:dataquery/locallang.xml:query.failure');
+            $exceptionCode = $e->getCode();
+            // Display "improved" exception message (if available)
+            $parsingMessage = $languageService->sL('LLL:EXT:dataquery/locallang.xml:query.exception-' . $exceptionCode);
+            // If some unexpected exception occurred, display original message
+            if (empty($parsingMessage)) {
+                $parsingMessage = $e->getMessage();
+            }
+        }
+        // Render parsing result as flash message
+        /** @var $flashMessage FlashMessage */
+        $flashMessage = GeneralUtility::makeInstance(
+                FlashMessage::class,
+                $parsingMessage,
+                $parsingTitle,
+                $parsingSeverity
+        );
+        $flashMessageQueue->enqueue($flashMessage);
+        // If a warning was returned by the query parser, display it here
+        if (!empty($warningMessage)) {
+            $flashMessage = GeneralUtility::makeInstance(
+                    FlashMessage::class,
+                    $warningMessage,
+                    $languageService->sL('LLL:EXT:dataquery/locallang.xml:query.warning'),
+                    FlashMessage::WARNING
+            );
+            $flashMessageQueue->enqueue($flashMessage);
+        }
+        // If the query was also executed, render execution result
+        if (!empty($executionMessage)) {
+            $flashMessage = GeneralUtility::makeInstance(
+                    FlashMessage::class,
+                    $executionMessage,
+                    '',
+                    $executionSeverity
+            );
+            $flashMessageQueue->enqueue($flashMessage);
+        }
+        $ajaxObj->addContent(
+                'dataquery',
+                $flashMessageQueue->renderFlashMessages()
+        );
+    }
 }
index 3bdd168..6d268d6 100755 (executable)
@@ -21,22 +21,24 @@ namespace Tesseract\Dataquery\Cache;
  * @package TYPO3
  * @subpackage tx_dataquery
  */
-class CacheHandler {
+class CacheHandler
+{
 
-       /**
-        * Clears the dataquery for selected pages only.
-        *
-        * @param array $parameters Parameters passed by DataHandler, including the pages to clear the cache for
-        * @param \TYPO3\CMS\Core\DataHandling\DataHandler $parentObject Reference to the calling DataHandler object
-        * @return void
-        */
-       public function clearCache($parameters, $parentObject) {
-               // Clear the dataquery cache for all the pages passed to this method
-               if (isset($parameters['pageIdArray']) && count($parameters['pageIdArray']) > 0) {
-                       $GLOBALS['TYPO3_DB']->exec_DELETEquery(
-                               'tx_dataquery_cache',
-                               'page_id IN (' . implode(',', $parameters['pageIdArray']) . ')'
-                       );
-               }
-       }
+    /**
+     * Clears the dataquery for selected pages only.
+     *
+     * @param array $parameters Parameters passed by DataHandler, including the pages to clear the cache for
+     * @param \TYPO3\CMS\Core\DataHandling\DataHandler $parentObject Reference to the calling DataHandler object
+     * @return void
+     */
+    public function clearCache($parameters, $parentObject)
+    {
+        // Clear the dataquery cache for all the pages passed to this method
+        if (isset($parameters['pageIdArray']) && count($parameters['pageIdArray']) > 0) {
+            $GLOBALS['TYPO3_DB']->exec_DELETEquery(
+                    'tx_dataquery_cache',
+                    'page_id IN (' . implode(',', $parameters['pageIdArray']) . ')'
+            );
+        }
+    }
 }
index 07d5700..5f74700 100644 (file)
@@ -21,16 +21,17 @@ namespace Tesseract\Dataquery\Cache;
  * @package TYPO3
  * @subpackage tx_dataquery
  */
-interface CacheParametersProcessorInterface {
-       /**
-        * This method must be implemented for processing cache parameters.
-        *
-        * It receives a reference to the current cache parameters and a back-reference to the calling object.
-        * It is expected to return the cache parameters, modified or not.
-        *
-        * @param array $cacheParameters Current cache parameters
-        * @param \Tesseract\Dataquery\Component\DataProvider $parentObject Back-reference to the calling object
-        * @return array Modified cache parameters
-        */
-       public function processCacheParameters($cacheParameters, $parentObject);
+interface CacheParametersProcessorInterface
+{
+    /**
+     * This method must be implemented for processing cache parameters.
+     *
+     * It receives a reference to the current cache parameters and a back-reference to the calling object.
+     * It is expected to return the cache parameters, modified or not.
+     *
+     * @param array $cacheParameters Current cache parameters
+     * @param \Tesseract\Dataquery\Component\DataProvider $parentObject Back-reference to the calling object
+     * @return array Modified cache parameters
+     */
+    public function processCacheParameters($cacheParameters, $parentObject);
 }
index d882c22..9d80205 100644 (file)
@@ -16,6 +16,7 @@ namespace Tesseract\Dataquery\Component;
 
 use Cobweb\Overlays\OverlayEngine;
 use Tesseract\Dataquery\Cache\CacheParametersProcessorInterface;
+use Tesseract\Dataquery\Parser\QueryParser;
 use Tesseract\Tesseract\Service\ProviderBase;
 use Tesseract\Tesseract\Tesseract;
 use Tesseract\Tesseract\Utility\Utilities;
@@ -30,1170 +31,1196 @@ use TYPO3\CMS\Core\Utility\GeneralUtility;
  * @package TYPO3
  * @subpackage tx_dataquery
  */
-class DataProvider extends ProviderBase {
-       public $extKey = 'dataquery';
-
-       /**
-        * @var array Extension configuration
-        */
-       protected $configuration;
-
-       /**
-        * @var string Name of the main table of the query
-        */
-       protected $mainTable;
-
-       /**
-        * Local instance of the SQL parser class
-        *
-        * @var \Tesseract\Dataquery\Parser\QueryParser $sqlParser
-        */
-       protected $sqlParser;
-
-       /**
-        * @var array List of fields used for sorting recordset
-        */
-       static public $sortingFields = array();
-
-       /**
-        * @var int Current level of sorting
-        */
-       static public $sortingLevel = 0;
-
-       /**
-        * @var string Type of data structure to provide (default is idList)
-        */
-       protected $dataStructureType = Tesseract::IDLIST_STRUCTURE_TYPE;
-
-       public function __construct() {
-               $this->initialise();
-       }
-
-       /**
-        * Performs various initializations that are shared between the constructor
-        * and the reset() method inherited from the service interface.
-        *
-        * NOTE: this method is NOT called init() to avoid conflicts with the init() method of the service interface.
-        *
-        * @return      void
-        */
-       public function initialise() {
-               $this->configuration = unserialize($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf'][$this->extKey]);
-               $this->mainTable = '';
-               $this->sqlParser = GeneralUtility::makeInstance(
-                       'Tesseract\\Dataquery\\Parser\\QueryParser',
-                       $this
-               );
-       }
-
-       /**
-        * Gets the data for the current query and return it in a standardised format.
-        *
-        * @return array Resulting data structure
-        */
-       protected function getQueryData() {
-               $dataStructure = array();
-               $returnStructure = array();
-
-               // If the cache duration is not set to 0, try to find a cached query
-               // Avoid that if global no_cache flag is set or if in a workspace
-               $hasStructure = FALSE;
-               if (!empty($this->providerData['cache_duration']) && empty($GLOBALS['TSFE']->no_cache) && empty($GLOBALS['TSFE']->sys_page->versioningPreview)) {
-                       try {
-                               $dataStructure = $this->getCachedStructure();
-//\TYPO3\CMS\Core\Utility\DebugUtility::debug($dataStructure);
-                               $hasStructure = TRUE;
-                       }
-                       // No structure was found, set flag that there's no structure yet
-                       // Add a message for debugging
-                       catch (\Exception $e) {
-                               $hasStructure = FALSE;
-                               $this->controller->addMessage(
-                                       $this->extKey,
-                                       'No cached structure found',
-                                       '',
-                                       FlashMessage::NOTICE
-                               );
-                       }
-               }
-
-               // If there's no structure yet, assemble it
-               if (!$hasStructure) {
-                       try {
-                               // Parse the query and log any warning that may have been raised
-                               $warning = $this->sqlParser->parseQuery($this->providerData['sql_query']);
-                               if (!empty($warning)) {
-                                       $this->controller->addMessage(
-                                               $this->extKey,
-                                               $warning,
-                                               'Problems in the query',
-                                               FlashMessage::WARNING
-                                       );
-                               }
-
-                               // Pass provider data to the parser
-                               $this->sqlParser->setProviderData($this->providerData);
-
-                               // Add the SQL conditions for the selected TYPO3 mechanisms
-                               $this->sqlParser->addTypo3Mechanisms();
-
-                               // Assemble filters, if defined
-                               if (is_array($this->filter) && count($this->filter) > 0) {
-                                       $this->sqlParser->addFilter($this->filter);
-                               }
-
-                               // Use idList from input SDS, if defined
-                               if (is_array($this->structure) && !empty($this->structure['count'])) {
-                                       $this->sqlParser->addIdList($this->structure['uidListWithTable']);
-                               }
-
-                               // Build the complete query and execute it
-                               try {
-                                       $query = $this->sqlParser->buildQuery();
-                                       $this->controller->addMessage(
-                                               $this->extKey,
-                                               sprintf('Query parsed and rebuilt successfully (%s)', $this->providerData['title']),
-                                               '',
-                                               FlashMessage::OK,
-                                               $query
-                                       );
-
-                                       // Execute the query
-                                       $res = $this->getDatabaseConnection()->sql_query($query);
-                                       // If something went wrong, issue an error
-                                       if ($res === FALSE) {
-                                               $dataStructure = array();
-                                               $this->controller->addMessage(
-                                                       $this->extKey,
-                                                               $this->getDatabaseConnection()->sql_error(),
-                                                       'Query execution failed',
-                                                       FlashMessage::ERROR,
-                                                       $query
-                                               );
-
-                                       // Otherwise prepare the full data structure
-                                       } elseif ($this->dataStructureType == Tesseract::IDLIST_STRUCTURE_TYPE) {
-                                               $dataStructure = $this->assembleIdListStructure($res);
-                                       } else {
-                                               $dataStructure = $this->assembleRecordsetStructure($res);
-                                       }
-                               }
-                               catch (\Exception $e) {
-                                       $this->controller->addMessage($this->extKey, $e->getMessage() . ' (' . $e->getCode() . ')', 'Query building failed', FlashMessage::ERROR);
-                               }
-                       }
-                       catch (\Exception $e) {
-                               $this->controller->addMessage($this->extKey, $e->getMessage() . ' (' . $e->getCode() . ')', 'Query parsing error', FlashMessage::ERROR, $this->providerData);
-                       }
-               }
-
-               // Continue only if there were some results
-               if (count($dataStructure) > 0) {
-                       // Apply limit and offset constraints
-                       $returnStructure = $this->applyLimit($dataStructure);
-                       // Free some memory
-                       unset($dataStructure);
-                       // As a last step add the filter to the data structure
-                       // NOTE: not all Data Consumers may be able to handle this data, but at least it's available
-                       $returnStructure['filter'] = $this->filter;
-
-                       // Hook for post-processing the data structure
-                       if (is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF'][$this->extKey]['postProcessDataStructure'])) {
-                               foreach($GLOBALS['TYPO3_CONF_VARS']['EXTCONF'][$this->extKey]['postProcessDataStructure'] as $className) {
-                                       $postProcessor = GeneralUtility::getUserObj($className);
-                                       $returnStructure = $postProcessor->postProcessDataStructure($returnStructure, $this);
-                               }
-                       }
-//\TYPO3\CMS\Core\Utility\DebugUtility::debug($returnStructure);
-               }
-               return $returnStructure;
-       }
-
-       /**
-        * Applies limit and offset constraints, if any and if relevant.
-        *
-        * @param array $dataStructure Full data structure
-        * @return array Data structure with limits applied (if relevant)
-        */
-       protected function applyLimit($dataStructure) {
-               $returnStructure = $dataStructure;
-               // Prepare the limit and offset parameters
-               $limit = (isset($this->filter['limit']['max'])) ? $this->filter['limit']['max'] : 0;
-               $offset = 0;
-               if ($limit > 0) {
-                       // If there's a direct pointer, it takes precedence over the offset
-                       if (isset($this->filter['limit']['pointer']) && $this->filter['limit']['pointer'] > 0) {
-                               $offset = $this->filter['limit']['pointer'];
-
-                       } else {
-                               $offset = $limit * ((isset($this->filter['limit']['offset'])) ? $this->filter['limit']['offset'] : 0);
-                               if ($offset < 0) {
-                                       $offset = 0;
-                               }
-                       }
-               }
-
-               // Take the structure and apply limit and offset, if defined
-               if ($limit > 0 || $offset > 0) {
-                       // Reset offset if beyond total number of records
-                       if ($offset > $dataStructure['totalCount']) {
-                               $offset = 0;
-                       }
-                       // Initialise final structure with data that won't change
-                       $returnStructure = array(
-                                                                       'name' => $dataStructure['name'],
-                                                                       'trueName' => $dataStructure['trueName'],
-                                                                       'totalCount' => $dataStructure['totalCount'],
-                                                                       'header' => $dataStructure['header'],
-                                                                       'records' => array()
-                                                                        );
-                       $counter = 0;
-                       $uidList = array();
-                       foreach ($dataStructure['records'] as $record) {
-                               // Get only those records that are after the offset and within the limit
-                               if ($counter >= $offset && ($limit == 0 || ($limit > 0 && $counter - $offset < $limit))) {
-                                       $counter++;
-                                       $returnStructure['records'][] = $record;
-                                       $uidList[] = $record['uid'];
-                               }
-                               // If the offset has not been reached yet, just increase the counter
-                               elseif ($counter < $offset) {
-                                       $counter++;
-                               }
-                               else {
-                                       break;
-                               }
-                       }
-                       $returnStructure['count'] = count($returnStructure['records']);
-                       $returnStructure['uidList'] = implode(',', $uidList);
-               }
-               return $returnStructure;
-       }
-
-       /**
-        * Prepares a full data structure with overlays if needed but without limits and offset.
-        *
-        * This is the structure that will be cached (at the end of method) to be called again from the cache when appropriate.
-        *
-        * @param mysqli_result $res Database resource from the executed query
-        * @return array The full data structure
-        */
-       protected function prepareFullStructure($res) {
-               // Initialise some variables
-               $this->mainTable = $this->sqlParser->getMainTableName();
-               $subtables = $this->sqlParser->getSubtablesNames();
-               $numSubtables = count($subtables);
-               $allTables = $subtables;
-               array_push($allTables, $this->mainTable);
-               $tableAndFieldLabels = $this->sqlParser->getLocalizedLabels();
-               $uidList = array();
-               $fullRecords = array();
-
-               // Get true table names for all tables
-               $allTablesTrueNames = array();
-               foreach ($allTables as $alias) {
-                       $allTablesTrueNames[$alias] = $this->sqlParser->getTrueTableName($alias);
-               }
-//\TYPO3\CMS\Core\Utility\DebugUtility::debug($allTablesTrueNames, 'True table names');
-
-               // Prepare the header parts for all tables
-               $headers = array();
-               foreach ($allTables as $table) {
-                       if (isset($tableAndFieldLabels[$table]['fields'])) {
-                               $headers[$table] = array();
-                               foreach ($tableAndFieldLabels[$table]['fields'] as $key => $label) {
-                                       $headers[$table][$key] = array('label' => $label);
-                       }
-                       }
-               }
-
-               // Act only if there are records
-               $databaseConnection = $this->getDatabaseConnection();
-               if ($databaseConnection->sql_num_rows($res) > 0) {
-                       // Initialise array for storing records
-                       $rows = array($this->mainTable => array(0 => array()));
-                       if ($numSubtables > 0) {
-                               foreach ($subtables as $table) {
-                                       $rows[$table] = array();
-                               }
-                       }
-
-                       // Loop on all records to assemble the raw recordset
-                       $rawRecordset = array();
-                       while ($row = $databaseConnection->sql_fetch_assoc($res)) {
-                               $rawRecordset[] = $row;
-
-                       }
-//\TYPO3\CMS\Core\Utility\DebugUtility::debug($rawRecordset, 'Raw result');
-
-                       // Analyze the first row of the raw recordset to get which column belongs to which table
-                       // and which aliases are used, if any
-                       $testRow = $rawRecordset[0];
-                       $columnsMappings = array();
-                       $reverseColumnsMappings = array();
-                       foreach ($testRow as $columnName => $value) {
-                               $info = $this->sqlParser->getTrueFieldName($columnName);
-//\TYPO3\CMS\Core\Utility\DebugUtility::debug($info, 'Field info');
-                               $columnsMappings[$columnName] = $info;
-                               $reverseColumnsMappings[$info['aliasTable']][$info['field']] = $columnName;
-                       }
-//\TYPO3\CMS\Core\Utility\DebugUtility::debug($columnsMappings, 'Columns mappings');
-//\TYPO3\CMS\Core\Utility\DebugUtility::debug($reverseColumnsMappings, 'Reversed columns mappings');
-
-                       // Split each row in the recordset into parts for each table in the query
-                       $splitRecordset = $this->splitResultIntoSubparts($rawRecordset, $columnsMappings);
-//\TYPO3\CMS\Core\Utility\DebugUtility::debug($splitRecordset, 'Split result');
-
-                       // If workspace preview is on, records must be overlaid with appropriate version
-                       $versionedRecordset = array();
-                       if ($GLOBALS['TSFE']->sys_page->versioningPreview) {
-                               // Loop on all recordset rows to overlay them
-                               foreach ($splitRecordset as $index => $subParts) {
-                                       $versionedRecord = array();
-                                       $skipRecord = FALSE;
-                                       foreach ($subParts as $alias => $subRow) {
-                                               $table = $allTablesTrueNames[$alias];
-                                               $result = $subRow;
-                                               if ($GLOBALS['TSFE']->sys_page->versioningPreview && $this->sqlParser->mustHandleVersioningOverlay($table)) {
-                                                       $GLOBALS['TSFE']->sys_page->versionOL($table, $result);
-                                               }
-                                               // Include result only if it was not unset during overlaying
-                                               if ($result !== FALSE) {
-                                                       $versionedRecord[$alias] = $result;
-                                               } else {
-                                                       if ($alias == $this->mainTable) {
-                                                               $skipRecord = TRUE;
-                                                       }
-                                               }
-                                       }
-                                       if (!$skipRecord) {
-                                               $versionedRecordset[$index] = $versionedRecord;
-                                       }
-                               }
-                       } else {
-                               $versionedRecordset = $splitRecordset;
-                       }
-//\TYPO3\CMS\Core\Utility\DebugUtility::debug($versionedRecordset, 'Versioned split result');
-
-                       // Get overlays for each table, if language is not default
-                       // Set a general flag about having been through this process or not
-                       $hasBeenThroughOverlayProcess = FALSE;
-                       $finalRecordset = array();
-                       if ($GLOBALS['TSFE']->sys_language_content == 0) {
-                               $finalRecordset = array();
-                               foreach ($versionedRecordset as $subParts) {
-                                               // Reassemble the full record
-                                       $overlaidRecord = array();
-                                       foreach ($subParts as $alias => $subRow) {
-                                               if (isset($subRow)) {
-                                                       foreach ($subRow as $field => $value) {
-                                                               $overlaidRecord[$reverseColumnsMappings[$alias][$field]] = $value;
-                                                       }
-                                               }
-                                       }
-                                       $finalRecordset[] = $overlaidRecord;
-                               }
-//\TYPO3\CMS\Core\Utility\DebugUtility::debug($finalRecordset, 'Reassembled versioned result');
-
-                               // If no sorting is defined at all, perform fixed order sorting, if defined
-                               // Note this will work only if the input structure's id list refers to a single table
-                               if (!$this->sqlParser->hasOrdering() && !empty($this->structure['count'])) {
-                                               // Add fixed order to recordset
-                                       $uidList = GeneralUtility::trimExplode(',', $this->structure['uidList']);
-                                       $fixedOrder = array_flip($uidList);
-                                       foreach ($finalRecordset as $index => $record) {
-                                               $finalRecordset[$index]['tx_dataquery:fixed_order'] = $fixedOrder[$record['uid']];
-                                       }
-                                       unset($fixedOrder);
-//\TYPO3\CMS\Core\Utility\DebugUtility::debug($finalRecordset, 'Recordset with fixed order');
-
-                                       // Sort recordset according to fixed order
-                                       usort($finalRecordset, array('tx_dataquery_wrapper', 'sortUsingFixedOrder'));
-                               }
-//\TYPO3\CMS\Core\Utility\DebugUtility::debug($finalRecordset, 'Recordset after sorting (no overlays)');
-
-                       } else {
-                               // First collect all the uid's for each table
-                               $allUIDs = array();
-                               foreach ($versionedRecordset as $subParts) {
-                                       foreach ($subParts as $alias => $subRow) {
-                                               $table = $allTablesTrueNames[$alias];
-                                               foreach ($subRow as $fieldName => $fieldValue) {
-                                                       if ($fieldName == 'uid' && isset($fieldValue)) {
-                                                               if (!isset($allUIDs[$table])) {
-                                                                       $allUIDs[$table] = array();
-                                                               }
-                                                               $allUIDs[$table][] = $fieldValue;
-                                                       }
-                                               }
-                                       }
-                               }
-                               // Get overlays for all tables
-                               $overlays = array();
-                               $doOverlays = array();
-                               foreach ($allUIDs as $table => $uidList) {
-                                       // Make sure the uid's are unique
-                                       $allUIDs[$table] = array_unique($uidList);
-                                       $doOverlays[$table] = $this->sqlParser->mustHandleLanguageOverlay($table);
-                                       // Get overlays only if needed/possible
-                                       if ($doOverlays[$table] && count($allUIDs[$table]) > 0) {
-                                               $overlays[$table] = OverlayEngine::getOverlayRecords($table, $allUIDs[$table], $GLOBALS['TSFE']->sys_language_content, $this->sqlParser->mustHandleVersioningOverlay($table));
-                                               // Set global overlay process flag to true
-                                               $hasBeenThroughOverlayProcess |= TRUE;
-                                       }
-                               }
-//\TYPO3\CMS\Core\Utility\DebugUtility::debug($allUIDs, 'Unique IDs per table');
-//\TYPO3\CMS\Core\Utility\DebugUtility::debug($doOverlays, 'Do overlays?');
-//\TYPO3\CMS\Core\Utility\DebugUtility::debug($overlays, 'Overlays');
-
-                               // Loop on all recordset rows to overlay them
-                               foreach ($versionedRecordset as $subParts) {
-                                               // Overlay each part
-                                       foreach ($subParts as $alias => $subRow) {
-                                               $table = $allTablesTrueNames[$alias];
-                                               $tableCtrl = $GLOBALS['TCA'][$table]['ctrl'];
-                                               $result = $subRow;
-                                               if ($GLOBALS['TSFE']->sys_page->versioningPreview && $this->sqlParser->mustHandleVersioningOverlay($table)) {
-                                                       $GLOBALS['TSFE']->sys_page->versionOL($table, $result);
-                                               }
-                                               if ($doOverlays[$table] && $subRow[$tableCtrl['languageField']] != $GLOBALS['TSFE']->sys_language_content) {
-                                                       // Overlay with record from foreign table
-                                                       if (isset($tableCtrl['transForeignTable']) && isset($overlays[$table][$subRow['uid']])) {
-                                                               $result = OverlayEngine::overlaySingleRecord($table, $subRow, $overlays[$table][$subRow['uid']]);
-
-                                                       // Overlay with record from same table
-                                                       } elseif (isset($overlays[$table][$subRow['uid']][$subRow['pid']])) {
-                                                               $result = OverlayEngine::overlaySingleRecord($table, $subRow, $overlays[$table][$subRow['uid']][$subRow['pid']]);
-
-                                                       // No overlay exists
-                                                       } else {
-                                                               // Take original record, only if non-translated are not hidden, or if language is [All]
-                                                               if ($GLOBALS['TSFE']->sys_language_contentOL == 'hideNonTranslated' && $subRow[$tableCtrl['languageField']] != -1) {
-                                                                       // Skip record
-                                                                       unset($result);
-                                                               } else {
-                                                                       $result = $subRow;
-                                                               }
-                                                       }
-                                               }
-                                               // Include result only if it was not unset during overlaying
-                                               if (isset($result)) {
-                                                       $subParts[$alias] = $result;
-                                               } else {
-                                                       unset($subParts[$alias]);
-                                               }
-                                       }
-                                       // Reassemble the full record
-                                       $overlaidRecord = array();
-                                       foreach ($subParts as $alias => $subRow) {
-                                               if (isset($subRow)) {
-                                                       foreach ($subRow as $field => $value) {
-                                                               // Remap fields, except for special key "_LOCALIZED_UID"
-                                                               if ($field === '_LOCALIZED_UID') {
-                                                                       // Differentiate main table and sub-tables
-                                                                       if ($alias == $this->mainTable) {
-                                                                               $key = '_LOCALIZED_UID';
-                                                                       } else {
-                                                                               $key = $alias . '$_LOCALIZED_UID';
-                                                                       }
-                                                                       $overlaidRecord[$key] = $value;
-                                                               } else {
-                                                                       $overlaidRecord[$reverseColumnsMappings[$alias][$field]] = $value;
-                                                               }
-                                                       }
-                                               // If the record for main table does not exist anymore, remove it entirely
-                                               } else {
-                                                       if ($alias == $this->mainTable) {
-                                                               unset($overlaidRecord);
-                                                               break;
-                                                       }
-                                               }
-                                       }
-//\TYPO3\CMS\Core\Utility\DebugUtility::debug($overlaidRecord, 'Overlaid record');
-                                       if (isset($overlaidRecord['uid'])) {
-                                               $finalRecordset[] = $overlaidRecord;
-                                       }
-                               }
-                               // Clean up (potentially large) arrays that are not used anymore
-                               unset($rawRecordset);
-                               unset($overlays);
-                               unset($subParts);
-//\TYPO3\CMS\Core\Utility\DebugUtility::debug($finalRecordset, 'Overlaid recordset');
-
-                               // If the dataquery was provided with a structure,
-                               // use the list of uid's to define a fixed order of records
-                               if (isset($this->structure['uidList'])) {
-                                       $uidList = GeneralUtility::trimExplode(',', $this->structure['uidList']);
-                                       $fixedOrder = array_flip($uidList);
-                                       foreach ($finalRecordset as $index => $record) {
-                                               $finalRecordset[$index]['tx_dataquery:fixed_order'] = $fixedOrder[$record['uid']];
-                                       }
-                                       unset($fixedOrder);
-                               }
-//\TYPO3\CMS\Core\Utility\DebugUtility::debug($finalRecordset, 'Final recordset before sorting');
-
-                               // Perform sorting if not handled by SQL
-                               if (!$this->sqlParser->isSqlUsedForOrdering()) {
-                                       self::$sortingFields = $this->sqlParser->getOrderByFields();
-                                       // The names of the fields as stored in the sorting fields configuration
-                                       // match the names used in the SQL query, but not the aliases
-                                       // So they will not match the column names in the full recordset
-                                       // Use the reverse mapping information (if available) to get the aliases
-                                       // (if defined, otherwise stick to field name)
-                                       foreach (self::$sortingFields as $index => $orderInfo) {
-                                               $alias = $this->mainTable;
-                                               // Field may have a special alias which is also not the colum name
-                                               // found in the recordset, but should be used to find that name
-                                               $fieldName = (isset($orderInfo['alias'])) ? $orderInfo['alias'] : $orderInfo['field'];
-                                               $fieldParts = explode('.', $fieldName);
-                                               if (count($fieldParts) == 1) {
-                                                       $field = $fieldParts[0];
-                                               } else {
-                                                       $alias = $fieldParts[0];
-                                                       $field = $fieldParts[1];
-                                               }
-                                               if (isset($reverseColumnsMappings[$alias][$field])) {
-                                                       self::$sortingFields[$index]['field'] = $reverseColumnsMappings[$alias][$field];
-                                               } else {
-                                                       self::$sortingFields[$index]['field'] = $field;
-                                               }
-                                       }
-//\TYPO3\CMS\Core\Utility\DebugUtility::debug(self::$sortingFields, 'Sorting fields');
-                                       self::$sortingLevel = 0;
-                                       usort($finalRecordset, array('tx_dataquery_wrapper', 'sortRecordset'));
-//\TYPO3\CMS\Core\Utility\DebugUtility::debug($finalRecordset, 'Sorted, overlaid recordset');
-
-                               // If no sorting is defined at all, perform fixed order sorting, if defined
-                               } elseif (!$this->sqlParser->hasOrdering() && isset($this->structure['uidList'])) {
-                                       usort($finalRecordset, array('tx_dataquery_wrapper', 'sortUsingFixedOrder'));
-                               }
-                       } // End of translation handling
-
-                       // Loop on all records to sort them by table. This can be seen as "de-JOINing" the tables.
-                       // This is necessary for such operations as overlays. When overlays are done, tables will be joined again
-                       // but within the format of Standardised Data Structure
-                       $handledUids = array();
-                       foreach ($finalRecordset as $row) {
-                               $currentUID = $row['uid'];
-                               // If we're handling a record we haven't handled before, perform some initialisations
-                               if (!array_key_exists($currentUID, $handledUids)) {
-                                       if ($numSubtables > 0) {
-                                               foreach ($subtables as $table) {
-                                                       $rows[$table][$currentUID] = array();
-                                               }
-                                       }
-                               }
-                               $recordsPerTable = array();
-                               foreach ($row as $fieldName => $fieldValue) {
-                                       // The query contains no joined table
-                                       // All fields belong to the main table
-                                       if ($numSubtables == 0) {
-                                               // If field is special field "_LOCALIZED_UID", keep its name as is
-                                               if ($fieldName === '_LOCALIZED_UID') {
-                                                       $finalFieldName = '_LOCALIZED_UID';
-
-                                               // Otherwise, use mapped name
-                                               } else {
-                                                       $finalFieldName = $columnsMappings[$fieldName]['mapping']['field'];
-                                               }
-                                               $recordsPerTable[$this->mainTable][$finalFieldName] = $fieldValue;
-
-                                       // There are multiple tables
-                                       } else {
-                                               // If field is special field "_LOCALIZED_UID" (possibly prepended with table name),
-                                               // keep it as is
-                                               if (strpos($fieldName, '_LOCALIZED_UID') !== FALSE) {
-                                                       // Handle main table and sub-tables separately
-                                                       if ($fieldName === '_LOCALIZED_UID') {
-                                                               $recordsPerTable[$this->mainTable]['_LOCALIZED_UID'] = $fieldValue;
-                                                       } else {
-                                                               list($table, $finalFieldName) = explode('$', $fieldName);
-                                                               $recordsPerTable[$table]['_LOCALIZED_UID'] = $fieldValue;
-                                                       }
-                                               // For other fields, get the field's true name
-                                               } else {
-                                                       $finalFieldName = $columnsMappings[$fieldName]['mapping']['field'];
-                                                       // However, if the field had an explicit alias, use that alias
-                                                       if (isset($columnsMappings[$fieldName]['mapping']['alias'])) {
-                                                               $finalFieldName = $columnsMappings[$fieldName]['mapping']['alias'];
-                                                       }
-
-                                                       // Field belongs to a subtable
-                                                       if (in_array($columnsMappings[$fieldName]['mapping']['table'], $subtables)) {
-                                                               $subtableName = $columnsMappings[$fieldName]['mapping']['table'];
-                                                               if (isset($fieldValue)) {
-                                                                       $recordsPerTable[$subtableName][$finalFieldName] = $fieldValue;
-                                                               }
-                                                       // Else assume the field belongs to the main table
-                                                       } else {
-                                                               $recordsPerTable[$this->mainTable][$finalFieldName] = $fieldValue;
-                                                       }
-                                               }
-                                       }
-                               }
-                               // If we're handling a record we haven't handled before, store the current information for the main table
-                               if (!array_key_exists($currentUID, $handledUids)) {
-                                       $rows[$this->mainTable][0][] = $recordsPerTable[$this->mainTable];
-                                       $handledUids[$currentUID] = $currentUID;
-                               }
-                               // Store information for each subtable
-                               if ($numSubtables > 0) {
-                                       foreach ($subtables as $table) {
-                                               if (isset($recordsPerTable[$table]) && count($recordsPerTable[$table]) > 0) {
-                                                       $rows[$table][$currentUID][] = $recordsPerTable[$table];
-                                               }
-                                       }
-                               }
-                       }
-                       // Clean up a potentially large array that is not used anymore
-                       unset($finalRecordset);
-                       unset($recordsPerTable);
-//\TYPO3\CMS\Core\Utility\DebugUtility::debug($rows, 'De-JOINed tables');
-
-                               // Now loop on all the records of the main table and join them to their subtables
-                       $hasInnerJoin = $this->sqlParser->hasInnerJoinOnFirstSubtable();
-                       $uidList = array();
-                       foreach ($rows[$this->mainTable][0] as $aRecord) {
-                               $uidList[] = $aRecord['uid'];
-                               $theFullRecord = $aRecord;
-                               $theFullRecord['__substructure'] = array();
-                               // Check if there are any subtables in the query
-                               $recordsPerSubtable = array();
-                               if ($numSubtables > 0) {
-                                       foreach ($subtables as $table) {
-                                               // Check if there are any subrecords for this record
-                                               if (isset($rows[$table][$aRecord['uid']])) {
-                                                       $numSubrecords = count($rows[$table][$aRecord['uid']]);
-                                                       if ($numSubrecords > 0) {
-                                                               $sublimit = $this->sqlParser->getSubTableLimit($table);
-                                                               $subcounter = 0;
-                                                               // Perform overlays only if language is not default and if necessary for table
-                                                               $subRecords = array();
-                                                               $subUidList = array();
-                                                               // Loop on all subrecords and apply limit, if any
-                                                               foreach ($rows[$table][$aRecord['uid']] as $subRow) {
-                                                                       if (!isset($subRow['uid'])) continue;
-                                                                       // Add the subrecord to the subtable only if it hasn't been included yet
-                                                                       // Multiple identical subrecords may happen when joining several tables together
-                                                                       // Take into account any limit that may have been placed on the number of subrecords in the query
-                                                                       // (using the non-SQL standard keyword MAX)
-                                                                       if (!in_array($subRow['uid'], $subUidList)) {
-                                                                               if ($sublimit == 0 || $subcounter < $sublimit) {
-                                                                                       $subRecords[] = $subRow;
-                                                                                       $subUidList[] = $subRow['uid'];
-                                                                               } elseif ($sublimit != 0 || $subcounter >= $sublimit) {
-                                                                                       break;
-                                                                               }
-                                                                               $subcounter++;
-                                                                       }
-                                                               }
-                                                               // If there are indeed items, add the subtable to the record
-                                                               $numItems = count($subUidList);
-                                                               $recordsPerSubtable[$table] = $numItems;
-                                                               if ($numItems > 0) {
-                                                                       $theFullRecord['__substructure'][$table] = array(
-                                                                                                                                                       'name' => $table,
-                                                                                                                                                       'trueName' => $allTablesTrueNames[$table],
-                                                                                                                                                       'count' => $numItems,
-                                                                                                                                                       'uidList' => implode(',' , $subUidList),
-                                                                                                                                                       'header' => $headers[$table],
-                                                                                                                                                       'records' => $subRecords
-                                                                                                                                               );
-                                                               }
-                                                       }
-                                               }
-                                       }
-                               }
-                               // If the query used INNER JOINs and went through the overlay process,
-                               // preform additional checks
-                               if ($numSubtables > 0 && !empty($hasInnerJoin) && $hasBeenThroughOverlayProcess) {
-                                       // If there are no subrecords after the overlay process, but the query
-                                       // used an INNER JOIN, the record must be removed, so that the end result
-                                       // will look like what it would have been in the default language
-                                       if (!empty($recordsPerSubtable[$hasInnerJoin])) {
-                                               $fullRecords[] = $theFullRecord;
-                                       }
-
-                               // Otherwise just take the record as is
-                               } else {
-                                       $fullRecords[] = $theFullRecord;
-                               }
-                       }
-                       // Clean up a potentially large array that is not used anymore
-                       unset($rows);
-               }
-               $databaseConnection->sql_free_result($res);
-
-                       // Assemble the full structure
-               $numRecords = count($fullRecords);
-               $dataStructure = array(
-                                                       'name' => $this->mainTable,
-                                                       'trueName' => $allTablesTrueNames[$this->mainTable],
-                                                       'count' => $numRecords,
-                                                       'totalCount' => $numRecords,
-                                                       'uidList' => implode(',', $uidList),
-                                                       'header' => $headers[$this->mainTable],
-                                                       'records' => $fullRecords
-                                               );
-               // Clean up a potentially large array that is not used anymore
-               unset($fullRecords);
-//\TYPO3\CMS\Core\Utility\DebugUtility::debug($dataStructure, 'Finished data structure');
-
-               // Hook for post-processing the data structure before it is stored into cache
-               if (is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF'][$this->extKey]['postProcessDataStructureBeforeCache'])) {
-                       foreach($GLOBALS['TYPO3_CONF_VARS']['EXTCONF'][$this->extKey]['postProcessDataStructureBeforeCache'] as $className) {
-                               $postProcessor = GeneralUtility::getUserObj($className);
-                               $dataStructure = $postProcessor->postProcessDataStructureBeforeCache($dataStructure, $this);
-                       }
-               }
-
-               // Store the structure in the cache table,
-               $this->writeStructureToCache($dataStructure);
-
-               // Finally return the assembled structure
-               return $dataStructure;
-       }
-
-       /**
-        * Wrapper around prepareFullStructure() for harmonizing naming between data providers
-        *
-        * @param mysqli_result $res Database resource from the executed query
-        * @return array A recordset-type data structure
-        * @see tx_dataquery_wrapper::prepareFullStructure()
-        */
-       protected function assembleRecordsetStructure($res) {
-               return $this->prepareFullStructure($res);
-       }
-
-       /**
-        * Prepares an id-list type data structure based on the results of the SQL query
-        *
-        * @param mysqli_result $res Database resource from the executed query
-        * @return array An id-list type data structure
-        */
-       protected function assembleIdListStructure($res) {
-               $databaseConnection = $this->getDatabaseConnection();
-               $uidList = array();
-               // Act only if there are records
-               if ($databaseConnection->sql_num_rows($res) > 0) {
-                       $this->mainTable = $this->sqlParser->getMainTableName();
-
-                       // Loop on all records to assemble the list of all uid's
-                       // Note that all overlay mechanisms (language and version) are ignored when assembling
-                       // an id-list type structure. Indeed the uid's returned by such a structure will always
-                       // be fed into some other mechanism that will query the database for a more complete structure
-                       // and overlays will be taken into account at that point
-                       $uidColumns = array();
-                       $firstPass = TRUE;
-                       while ($row = $databaseConnection->sql_fetch_assoc($res)) {
-                               // On the first pass, assemble a list of all the columns containing uid's
-                               // and their relative tables
-                               if ($firstPass) {
-                                       foreach ($row as $columnName => $value) {
-                                               if ($columnName == 'uid' || substr($columnName, -4) == '$uid') {
-                                                       if ($columnName == 'uid') {
-                                                               $table = $this->mainTable;
-                                                       } else {
-                                                               $table = substr($columnName, 0, -4);
-                                                       }
-                                                       $uidColumns[$table] = $columnName;
-                                                       $uidList[$table] = array();
-                                               }
-                                       }
-                                       $firstPass = FALSE;
-                               }
-                               // Loop on all uid columns and store them relative to their related table
-                               foreach ($uidColumns as $table => $columnName) {
-                                       if (isset($row[$columnName])) {
-                                               $uidList[$table][] = $row[$columnName];
-                                       }
-                               }
-                       }
-                       $databaseConnection->sql_free_result($res);
-
-                       // Remove duplicates for every table
-                       foreach ($uidList as $table => $uidListForTable) {
-                               $uidList[$table] = array_unique($uidListForTable);
-                       }
-               }
-
-               // Assemble the full structure
-               $numRecords = (isset($uidList[$this->mainTable])) ? count($uidList[$this->mainTable]) : 0;
-               $uidListString = (isset($uidList[$this->mainTable])) ? implode(',', $uidList[$this->mainTable]) : '';
-               $uidListWithTable = '';
-               foreach ($uidList as $table => $uidListForTable) {
-                       if (!empty($uidListWithTable)) {
-                               $uidListWithTable .= ',';
-                       }
-                       $uidListWithTable .= $table . '_' . implode(',' . $table . '_', $uidListForTable);
-               }
-               $dataStructure = array (
-                       'uniqueTable' => (count($uidList) == 1) ? $this->mainTable : '',
-                       'uidList' => $uidListString,
-                       'uidListWithTable' => $uidListWithTable,
-                       'count' => $numRecords,
-                       'totalCount' => $numRecords
-               );
-
-               // Hook for post-processing the data structure before it is stored into cache
-               if (is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF'][$this->extKey]['postProcessDataStructureBeforeCache'])) {
-                       foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF'][$this->extKey]['postProcessDataStructureBeforeCache'] as $className) {
-                               $postProcessor = GeneralUtility::getUserObj($className);
-                               $dataStructure = $postProcessor->postProcessDataStructureBeforeCache($dataStructure, $this);
-                       }
-               }
-
-               // Store the structure in the cache table,
-               $this->writeStructureToCache($dataStructure);
-
-               // Finally return the assembled structure
-               return $dataStructure;
-       }
-
-       /**
-        * Splits the rows in the recordset returned by the database query into several parts,
-        * one for each (aliased) table in the row.
-        *
-        * Example:
-        * Consider the following query:
-        *
-        * SELECT pages.title, tt_content.header FROM pages,tt_content WHERE tt_content.pid = pages.uid
-        *
-        * A typical result row might look like:
-        *
-        * array('title' => 'foo', 'header' => 'bar');
-        *
-        * This method will turn such an entry into:
-        *
-        * array(
-        *              'pages' => array('title' => 'foo'),
-        *              'tt_content' => array('header' => 'bar'),
-        * );
-        *
-        * This is necessary so that each of these sub-parts can be overlaid (for language or version) properly.
-        *
-        * @param array $result Raw entry from the DB recordset
-        * @param array $columnsMappings Information about each column in the recordset
-        * @return array The result, split into its constituent tables
-        */
-       protected function splitResultIntoSubparts(array $result, array $columnsMappings) {
-               $splitResults = array();
-               foreach ($result as $row) {
-                       $subParts = array();
-                       // Split record into parts related to a single table
-                       foreach ($columnsMappings as $columnName => $columnInfo) {
-                               if (!isset($subParts[$columnInfo['aliasTable']])) {
-                                       $subParts[$columnInfo['aliasTable']] = array();
-                               }
-                               $subParts[$columnInfo['aliasTable']][$columnInfo['field']] = $row[$columnName];
-                       }
-                       $splitResults[] = $subParts;
-               }
-               return $splitResults;
-       }
-
-       /**
-        * Retrieves a data structure stored in cache provided it fits all parameters.
-        *
-        * If no appropriate cache is found, it throws an exception.
-        *
-        * @throws \Tesseract\Tesseract\Exception\Exception
-        * @return array
-        */
-       protected function getCachedStructure() {
-               // Assemble condition for finding correct cache
-               // This means matching the dataquery's primary key, the current language, the filter's hash (without the limit)
-               // and that it has not expired
-               $where = 'query_id = ' . intval($this->providerData['uid']) . ' AND page_id = ' . intval($GLOBALS['TSFE']->id);
-               $where .= ' AND cache_hash = \'' . $this->calculateCacheHash(array()) . '\'';
-               $where .= ' AND expires > \'' . time() . '\'';
-               $row = $this->getDatabaseConnection()->exec_SELECTgetSingleRow(
-                       'structure_cache',
-                       'tx_dataquery_cache',
-                       $where
-               );
-               if (empty($row)) {
-                       throw new \Tesseract\Tesseract\Exception\Exception('No cached structure');
-               } else {
-                       return unserialize($row['structure_cache']);
-               }
-       }
-
-       /**
-        * This method write the standard data structure to cache,
-        * provided some conditions are met
-        *
-        * @param array $structure A standard data structure
-        * @return void
-        */
-       protected function writeStructureToCache($structure) {
-               // Write only if cache is active, i.e.
-               // if cache duration is not empty, and we're not in no_cache mode or in a workspace
-               if (!empty($this->providerData['cache_duration']) && empty($GLOBALS['TSFE']->no_cache) && empty($GLOBALS['TSFE']->sys_page->versioningPreview)) {
-                       $cacheHash = $this->calculateCacheHash(array());
-                       $serializedStructure = serialize($structure);
-                       // Write only if serialized data is not too large
-                       if (empty($this->configuration['cacheLimit']) || strlen($serializedStructure) <= $this->configuration['cacheLimit']) {
-                               $fields = array(
-                                       'query_id' => $this->providerData['uid'],
-                                       'page_id' => $GLOBALS['TSFE']->id,
-                                       'cache_hash' => $cacheHash,
-                                       'structure_cache' => $serializedStructure,
-                                       'expires' => time() + $this->providerData['cache_duration']
-                               );
-                               $this->getDatabaseConnection()->exec_INSERTquery('tx_dataquery_cache', $fields);
-                       }
-                       // If data is too large for caching, make sure no other cache is left over
-                       else {
-                               $where = 'query_id = ' . intval($this->providerData['uid']);
-                               $where .= ' AND page_id = ' . intval($GLOBALS['TSFE']->id);
-                               $where .= ' AND cache_hash = \'' . $cacheHash . '\'';
-                               $this->getDatabaseConnection()->exec_DELETEquery('tx_dataquery_cache', $where);
-                       }
-               }
-       }
-
-       /**
-        * Assembles a hash parameter depending on a variety of parameters, including
-        * the current FE language and the groups of the current FE user, if any.
-        *
-        * @param array $parameters Additional parameters to add to the hash calculation
-        * @return string A md5 hash
-        */
-       protected function calculateCacheHash(array $parameters) {
-               // The base of the hash parameters is the current filter
-               // To this we add the uidList (if it exists)
-               // This makes it possible to vary the cache as a function of the idList provided by the input structure
-               $filterForCache = $this->filter;
-               if (is_array($this->structure) && isset($this->structure['uidListWithTable'])) {
-                       $filterForCache['uidListWithTable'] = $this->structure['uidListWithTable'];
-               }
-               // If some parameters were given, add them to the base cache parameters
-               $cacheParameters = $filterForCache;
-               if (is_array($parameters) && count($parameters) > 0) {
-                       $cacheParameters = array_merge($cacheParameters, $parameters);
-               }
-               // Finally we add other parameters of uniqueness:
-               //      - the current FE language
-               //      - the groups of the currently logged in FE user (if any)
-               //      - the type of structure
-               $cacheParameters['sys_language_uid'] = $GLOBALS['TSFE']->sys_language_content;
-               if (is_array($GLOBALS['TSFE']->fe_user->user) && count($GLOBALS['TSFE']->fe_user->groupData['uid']) > 0) {
-                       $cacheParameters['fe_groups'] = $GLOBALS['TSFE']->fe_user->groupData['uid'];
-               }
-               $cacheParameters['structure_type'] = $this->dataStructureType;
-               // Go through a hook for manipulating cache parameters
-               if (is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['dataquery']['processCacheHashParameters'])) {
-                       foreach($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['dataquery']['processCacheHashParameters'] as $className) {
-                               /** @var $processor CacheParametersProcessorInterface */
-                               $processor = GeneralUtility::getUserObj($className);
-                               if ($processor instanceof CacheParametersProcessorInterface) {
-                                       $cacheParameters = $processor->processCacheParameters($cacheParameters, $this);
-                               }
-                       }
-               }
-               // Calculate the hash using the method provided by the base controller,
-               // which filters out the "limit" part of the filter
-               return Utilities::calculateFilterCacheHash($cacheParameters);
-       }
+class DataProvider extends ProviderBase
+{
+    public $extKey = 'dataquery';
 
     /**
-        * Returns the name of the main table of the query,
-        * which is the table name that appears in the FROM clause, or the alias, if any.
-        *
-        * @return string Main table name
-        */
-       public function getMainTableName() {
-               return $this->mainTable;
-       }
-
-       /**
-        * Performs a special sorting of the recordset.
-        *
-        * @param mixed $a First element to sort
-        * @param mixed $b Second element to sort
-        * @return integer -1 if first argument is smaller than second argument, 1 if first is greater than second and 0 if both are equal
-        *
-        * @see \Tesseract\Dataquery\Component\DataProvider::prepareFullStructure()
-        */
-       static public function sortRecordset($a, $b) {
-               // Get the sorting information from static variables
-               // The level is a pointer to the current field being used for sorting
-               $level = self::$sortingLevel;
-               $field = self::$sortingFields[$level]['field'];
-               $order = (empty(self::$sortingFields[$level]['order'])) ? 'ASC' : strtoupper(self::$sortingFields[$level]['order']);
-               $result = strnatcasecmp($a[$field], $b[$field]);
-               if ($result == 0) {
-                       // If results are equal on the current level, check if there's a next level of sorting
-                       // for differentiating the records
-                       // If yes, call sorting method recursively
-                       if (isset(self::$sortingFields[$level + 1])) {
-                               self::$sortingLevel++;
-                               $result = self::sortRecordset($a, $b);
-                               self::$sortingLevel--;
-                       }
-               }
-               else {
-                       if ($order == 'DESC') {
-                               $result = -$result;
-                       }
-               }
-               return $result;
-       }
-
-       /**
-        * Sorts records using a special fixed order value.
-        *
-        * @param mixed $a First element to sort
-        * @param mixed $b Second element to sort
-        * @return integer -1 if first argument is smaller than second argument, 1 if first is greater than second and 0 if both are equal
-        *
-        * @see \Tesseract\Dataquery\Component\DataProvider::prepareFullStructure()
-        */
-       static public function sortUsingFixedOrder($a, $b) {
-               $result = 1;
-               if ($a['tx_dataquery:fixed_order'] == $b['tx_dataquery:fixed_order']) {
-                       $result = 0;
-               } elseif ($a['tx_dataquery:fixed_order'] < $b['tx_dataquery:fixed_order']) {
-                       $result = -1;
-               }
-               return $result;
-       }
-
-// Data Provider interface methods
-
-       /**
-        * Returns the type of data structure that the Data Provider can prepare.
-        *
-        * @return string Type of the provided data structure
-        */
-       public function getProvidedDataStructure() {
-               return $this->dataStructureType;
-       }
-
-       /**
-        * Indicates whether the Data Provider can create the type of data structure requested or not.
-        *
-        * @param string $type: Type of data structure
-        * @return boolean TRUE if it can handle the requested type, FALSE otherwise
-        */
-       public function providesDataStructure($type) {
-
-               // Check which type was requested and return true if type can be provided
-               // Store requested type internally for later processing
-               if ($type == Tesseract::IDLIST_STRUCTURE_TYPE) {
-                       $this->dataStructureType = Tesseract::IDLIST_STRUCTURE_TYPE;
-                       $result = TRUE;
-               } elseif ($type == Tesseract::RECORDSET_STRUCTURE_TYPE) {
-                       $this->dataStructureType = Tesseract::RECORDSET_STRUCTURE_TYPE;
-                       $result = TRUE;
-               } else {
-                       $result = FALSE;
-               }
-               return $result;
-       }
-
-       /**
-        * Returns the type of data structure that the Data Provider can receive as input.
-        *
-        * @return string Type of used data structures
-        */
-       public function getAcceptedDataStructure() {
-               return Tesseract::IDLIST_STRUCTURE_TYPE;
-       }
-
-       /**
-        * Indicates whether the Data Provider can use as input the type of data structure requested or not.
-        *
-        * @param string $type Type of data structure
-        * @return boolean True if it can use the requested type, false otherwise
-        */
-       public function acceptsDataStructure($type) {
-               return $type == Tesseract::IDLIST_STRUCTURE_TYPE;
-       }
-
-       /**
-        * Assembles the data structure and returns it.
-        *
-        * If the empty structure flag has been set, a dummy empty structure is returned instead.
-        *
-        * @return array Standardised data structure
-        */
-       public function getDataStructure() {
-               // If the empty output structure flag was raised, prepare a proper structure devoid of data
-               if ($this->hasEmptyOutputStructure) {
-                       try {
-                               // Parse the query to get the main table's name
-                               $this->sqlParser->parseQuery($this->providerData['sql_query']);
-                               $this->initEmptyDataStructure(
-                                       $this->sqlParser->getMainTableName(),
-                                       $this->dataStructureType
-                               );
-                       }
-                       catch (\Exception $e) {
-                               $this->controller->addMessage($this->extKey, $e->getMessage() . ' (' . $e->getCode() . ')', 'Query parsing error', FlashMessage::ERROR, $this->providerData);
-                       }
-               } else {
-                       return $this->getQueryData();
-               }
-               return $this->outputStructure;
-       }
-
-       /**
-        * Passes a data structure to the Data Provider.
-        *
-        * @param array $structure Standardised data structure
-        * @return void
-        */
-       public function setDataStructure($structure) {
-               if (is_array($structure)) {
-                       $this->structure = $structure;
-               }
-       }
-
-       /**
-        * Loads the query and gets the list of tables and fields,
-        * complete with localized labels.
-        *
-        * @param string $language 2-letter iso code for language
-        * @return array List of tables and fields
-        */
-       public function getTablesAndFields($language = '') {
-               $this->sqlParser->parseQuery($this->providerData['sql_query']);
-               $tablesAndFields = $this->sqlParser->getLocalizedLabels($language);
-
-               // Hook for post-processing the tables and fields information
-               if (is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF'][$this->extKey]['postProcessFieldInformation'])) {
-                       foreach($GLOBALS['TYPO3_CONF_VARS']['EXTCONF'][$this->extKey]['postProcessFieldInformation'] as $className) {
-                               $postProcessor = GeneralUtility::getUserObj($className);
-                               $tablesAndFields = $postProcessor->postProcessFieldInformation($tablesAndFields, $this);
-                       }
-               }
-               return $tablesAndFields;
+     * @var array Extension configuration
+     */
+    protected $configuration;
+
+    /**
+     * @var string Name of the main table of the query
+     */
+    protected $mainTable;
+
+    /**
+     * Local instance of the SQL parser class
+     *
+     * @var QueryParser $sqlParser
+     */
+    protected $sqlParser;
+
+    /**
+     * @var array List of fields used for sorting recordset
+     */
+    static public $sortingFields = array();
+
+    /**
+     * @var int Current level of sorting
+     */
+    static public $sortingLevel = 0;
+
+    /**
+     * @var string Type of data structure to provide (default is idList)
+     */
+    protected $dataStructureType = Tesseract::IDLIST_STRUCTURE_TYPE;
+
+    public function __construct()
+    {
+        $this->initialise();
+    }
+
+    /**
+     * Performs various initializations that are shared between the constructor
+     * and the reset() method inherited from the service interface.
+     *
+     * NOTE: this method is NOT called init() to avoid conflicts with the init() method of the service interface.
+     *
+     * @return    void
+     */
+    public function initialise()
+    {
+        $this->configuration = unserialize($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf'][$this->extKey]);
+        $this->mainTable = '';
+        $this->sqlParser = GeneralUtility::makeInstance(
+                QueryParser::class,
+                $this
+        );
+    }
+
+    /**
+     * Gets the data for the current query and return it in a standardised format.
+     *
+     * @return array Resulting data structure
+     */
+    protected function getQueryData()
+    {
+        $dataStructure = array();
+        $returnStructure = array();
+
+        // If the cache duration is not set to 0, try to find a cached query
+        // Avoid that if global no_cache flag is set or if in a workspace
+        $hasStructure = false;
+        if (!empty($this->providerData['cache_duration']) && empty($GLOBALS['TSFE']->no_cache) && empty($GLOBALS['TSFE']->sys_page->versioningPreview)) {
+            try {
+                $dataStructure = $this->getCachedStructure();
+                //\TYPO3\CMS\Core\Utility\DebugUtility::debug($dataStructure);
+                $hasStructure = true;
+            }
+                // No structure was found, set flag that there's no structure yet
+                // Add a message for debugging
+            catch (\Exception $e) {
+                $hasStructure = false;
+                $this->controller->addMessage(
+                        $this->extKey,
+                        'No cached structure found',
+                        '',
+                        FlashMessage::NOTICE
+                );
+            }
+        }
+
+        // If there's no structure yet, assemble it
+        if (!$hasStructure) {
+            try {
+                // Parse the query and log any warning that may have been raised
+                $warning = $this->sqlParser->parseQuery($this->providerData['sql_query']);
+                if (!empty($warning)) {
+                    $this->controller->addMessage(
+                            $this->extKey,
+                            $warning,
+                            'Problems in the query',
+                            FlashMessage::WARNING
+                    );
+                }
+
+                // Pass provider data to the parser
+                $this->sqlParser->setProviderData($this->providerData);
+
+                // Add the SQL conditions for the selected TYPO3 mechanisms
+                $this->sqlParser->addTypo3Mechanisms();
+
+                // Assemble filters, if defined
+                if (is_array($this->filter) && count($this->filter) > 0) {
+                    $this->sqlParser->addFilter($this->filter);
+                }
+
+                // Use idList from input SDS, if defined
+                if (is_array($this->structure) && !empty($this->structure['count'])) {
+                    $this->sqlParser->addIdList($this->structure['uidListWithTable']);
+                }
+
+                // Build the complete query and execute it
+                try {
+                    $query = $this->sqlParser->buildQuery();
+                    $this->controller->addMessage(
+                            $this->extKey,
+                            sprintf('Query parsed and rebuilt successfully (%s)', $this->providerData['title']),
+                            '',
+                            FlashMessage::OK,
+                            $query
+                    );
+
+                    // Execute the query
+                    $res = $this->getDatabaseConnection()->sql_query($query);
+                    // If something went wrong, issue an error
+                    if ($res === false) {
+                        $dataStructure = array();
+                        $this->controller->addMessage(
+                                $this->extKey,
+                                $this->getDatabaseConnection()->sql_error(),
+                                'Query execution failed',
+                                FlashMessage::ERROR,
+                                $query
+                        );
+
+                        // Otherwise prepare the full data structure
+                    } elseif ($this->dataStructureType === Tesseract::IDLIST_STRUCTURE_TYPE) {
+                        $dataStructure = $this->assembleIdListStructure($res);
+                    } else {
+                        $dataStructure = $this->assembleRecordsetStructure($res);
+                    }
+                } catch (\Exception $e) {
+                    $this->controller->addMessage($this->extKey, $e->getMessage() . ' (' . $e->getCode() . ')',
+                            'Query building failed', FlashMessage::ERROR);
+                }
+            } catch (\Exception $e) {
+                $this->controller->addMessage($this->extKey, $e->getMessage() . ' (' . $e->getCode() . ')',
+                        'Query parsing error', FlashMessage::ERROR, $this->providerData);
+            }
+        }
+
+        // Continue only if there were some results
+        if (count($dataStructure) > 0) {
+            // Apply limit and offset constraints
+            $returnStructure = $this->applyLimit($dataStructure);
+            // Free some memory
+            unset($dataStructure);
+            // As a last step add the filter to the data structure
+            // NOTE: not all Data Consumers may be able to handle this data, but at least it's available
+            $returnStructure['filter'] = $this->filter;
+
+            // Hook for post-processing the data structure
+            if (is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF'][$this->extKey]['postProcessDataStructure'])) {
+                foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF'][$this->extKey]['postProcessDataStructure'] as $className) {
+                    $postProcessor = GeneralUtility::getUserObj($className);
+                    $returnStructure = $postProcessor->postProcessDataStructure($returnStructure, $this);
+                }
+            }
+            //\TYPO3\CMS\Core\Utility\DebugUtility::debug($returnStructure);
+        }
+        return $returnStructure;
+    }
+
+    /**
+     * Applies limit and offset constraints, if any and if relevant.
+     *
+     * @param array $dataStructure Full data structure
+     * @return array Data structure with limits applied (if relevant)
+     */
+    protected function applyLimit($dataStructure)
+    {
+        $returnStructure = $dataStructure;
+        // Prepare the limit and offset parameters
+        $limit = (isset($this->filter['limit']['max'])) ? $this->filter['limit']['max'] : 0;
+        $offset = 0;
+        if ($limit > 0) {
+            // If there's a direct pointer, it takes precedence over the offset
+            if (isset($this->filter['limit']['pointer']) && $this->filter['limit']['pointer'] > 0) {
+                $offset = $this->filter['limit']['pointer'];
+
+            } else {
+                $offset = $limit * ((isset($this->filter['limit']['offset'])) ? $this->filter['limit']['offset'] : 0);
+                if ($offset < 0) {
+                    $offset = 0;
+                }
+            }
+        }
+
+        // Take the structure and apply limit and offset, if defined
+        if ($limit > 0 || $offset > 0) {
+            // Reset offset if beyond total number of records
+            if ($offset > $dataStructure['totalCount']) {
+                $offset = 0;
+            }
+            // Initialise final structure with data that won't change
+            $returnStructure = array(
+                    'name' => $dataStructure['name'],
+                    'trueName' => $dataStructure['trueName'],
+                    'totalCount' => $dataStructure['totalCount'],
+                    'header' => $dataStructure['header'],
+                    'records' => array()
+            );
+            $counter = 0;
+            $uidList = array();
+            foreach ($dataStructure['records'] as $record) {
+                // Get only those records that are after the offset and within the limit
+                if ($counter >= $offset && ($limit == 0 || ($limit > 0 && $counter - $offset < $limit))) {
+                    $counter++;
+                    $returnStructure['records'][] = $record;
+                    $uidList[] = $record['uid'];
+                } // If the offset has not been reached yet, just increase the counter
+                elseif ($counter < $offset) {
+                    $counter++;
+                } else {
+                    break;
+                }
+            }
+            $returnStructure['count'] = count($returnStructure['records']);
+            $returnStructure['uidList'] = implode(',', $uidList);
+        }
+        return $returnStructure;
+    }
+
+    /**
+     * Prepares a full data structure with overlays if needed but without limits and offset.
+     *
+     * This is the structure that will be cached (at the end of method) to be called again from the cache when appropriate.
+     *
+     * @param mysqli_result $res Database resource from the executed query
+     * @return array The full data structure
+     */
+    protected function prepareFullStructure($res)
+    {
+        // Initialise some variables
+        $this->mainTable = $this->sqlParser->getMainTableName();
+        $subtables = $this->sqlParser->getSubtablesNames();
+        $numSubtables = count($subtables);
+        $allTables = $subtables;
+        array_push($allTables, $this->mainTable);
+        $tableAndFieldLabels = $this->sqlParser->getLocalizedLabels();
+        $uidList = array();
+        $fullRecords = array();
+
+        // Get true table names for all tables
+        $allTablesTrueNames = array();
+        foreach ($allTables as $alias) {
+            $allTablesTrueNames[$alias] = $this->sqlParser->getTrueTableName($alias);
+        }
+        //\TYPO3\CMS\Core\Utility\DebugUtility::debug($allTablesTrueNames, 'True table names');
+
+        // Prepare the header parts for all tables
+        $headers = array();
+        foreach ($allTables as $table) {
+            if (isset($tableAndFieldLabels[$table]['fields'])) {
+                $headers[$table] = array();
+                foreach ($tableAndFieldLabels[$table]['fields'] as $key => $label) {
+                    $headers[$table][$key] = array('label' => $label);
+                }
+            }
+        }
+
+        // Act only if there are records
+        $databaseConnection = $this->getDatabaseConnection();
+        if ($databaseConnection->sql_num_rows($res) > 0) {
+            // Initialise array for storing records
+            $rows = array($this->mainTable => array(0 => array()));
+            if ($numSubtables > 0) {
+                foreach ($subtables as $table) {
+                    $rows[$table] = array();
+                }
+            }
+
+            // Loop on all records to assemble the raw recordset
+            $rawRecordset = array();
+            while ($row = $databaseConnection->sql_fetch_assoc($res)) {
+                $rawRecordset[] = $row;
+
+            }
+            //\TYPO3\CMS\Core\Utility\DebugUtility::debug($rawRecordset, 'Raw result');
+
+            // Analyze the first row of the raw recordset to get which column belongs to which table
+            // and which aliases are used, if any
+            $testRow = $rawRecordset[0];
+            $columnsMappings = array();
+            $reverseColumnsMappings = array();
+            foreach ($testRow as $columnName => $value) {
+                $info = $this->sqlParser->getTrueFieldName($columnName);
+                //\TYPO3\CMS\Core\Utility\DebugUtility::debug($info, 'Field info');
+                $columnsMappings[$columnName] = $info;
+                $reverseColumnsMappings[$info['aliasTable']][$info['field']] = $columnName;
+            }
+            //\TYPO3\CMS\Core\Utility\DebugUtility::debug($columnsMappings, 'Columns mappings');
+            //\TYPO3\CMS\Core\Utility\DebugUtility::debug($reverseColumnsMappings, 'Reversed columns mappings');
+
+            // Split each row in the recordset into parts for each table in the query
+            $splitRecordset = $this->splitResultIntoSubparts($rawRecordset, $columnsMappings);
+            //\TYPO3\CMS\Core\Utility\DebugUtility::debug($splitRecordset, 'Split result');
+
+            // If workspace preview is on, records must be overlaid with appropriate version
+            $versionedRecordset = array();
+            if ($GLOBALS['TSFE']->sys_page->versioningPreview) {
+                // Loop on all recordset rows to overlay them
+                foreach ($splitRecordset as $index => $subParts) {
+                    $versionedRecord = array();
+                    $skipRecord = false;
+                    foreach ($subParts as $alias => $subRow) {
+                        $table = $allTablesTrueNames[$alias];
+                        $result = $subRow;
+                        if ($GLOBALS['TSFE']->sys_page->versioningPreview && $this->sqlParser->mustHandleVersioningOverlay($table)) {
+                            $GLOBALS['TSFE']->sys_page->versionOL($table, $result);
+                        }
+                        // Include result only if it was not unset during overlaying
+                        if ($result !== false) {
+                            $versionedRecord[$alias] = $result;
+                        } else {
+                            if ($alias == $this->mainTable) {
+                                $skipRecord = true;
+                            }
+                        }
+                    }
+                    if (!$skipRecord) {
+                        $versionedRecordset[$index] = $versionedRecord;
+                    }
+                }
+            } else {
+                $versionedRecordset = $splitRecordset;
+            }
+            //\TYPO3\CMS\Core\Utility\DebugUtility::debug($versionedRecordset, 'Versioned split result');
+
+            // Get overlays for each table, if language is not default
+            // Set a general flag about having been through this process or not
+            $hasBeenThroughOverlayProcess = false;
+            $finalRecordset = array();
+            if ($GLOBALS['TSFE']->sys_language_content == 0) {
+                $finalRecordset = array();
+                foreach ($versionedRecordset as $subParts) {
+                    // Reassemble the full record
+                    $overlaidRecord = array();
+                    foreach ($subParts as $alias => $subRow) {
+                        if (isset($subRow)) {
+                            foreach ($subRow as $field => $value) {
+                                $overlaidRecord[$reverseColumnsMappings[$alias][$field]] = $value;
+                            }
+                        }
+                    }
+                    $finalRecordset[] = $overlaidRecord;
+                }
+                //\TYPO3\CMS\Core\Utility\DebugUtility::debug($finalRecordset, 'Reassembled versioned result');
+
+                // If no sorting is defined at all, perform fixed order sorting, if defined
+                // Note this will work only if the input structure's id list refers to a single table
+                if (!$this->sqlParser->hasOrdering() && !empty($this->structure['count'])) {
+                    // Add fixed order to recordset
+                    $uidList = GeneralUtility::trimExplode(',', $this->structure['uidList']);
+                    $fixedOrder = array_flip($uidList);
+                    foreach ($finalRecordset as $index => $record) {
+                        $finalRecordset[$index]['tx_dataquery:fixed_order'] = $fixedOrder[$record['uid']];
+                    }
+                    unset($fixedOrder);
+                    //\TYPO3\CMS\Core\Utility\DebugUtility::debug($finalRecordset, 'Recordset with fixed order');
+
+                    // Sort recordset according to fixed order
+                    usort($finalRecordset, array('tx_dataquery_wrapper', 'sortUsingFixedOrder'));
+                }
+                //\TYPO3\CMS\Core\Utility\DebugUtility::debug($finalRecordset, 'Recordset after sorting (no overlays)');
+
+            } else {
+                // First collect all the uid's for each table
+                $allUIDs = array();
+                foreach ($versionedRecordset as $subParts) {
+                    foreach ($subParts as $alias => $subRow) {
+                        $table = $allTablesTrueNames[$alias];
+                        foreach ($subRow as $fieldName => $fieldValue) {
+                            if ($fieldName === 'uid' && isset($fieldValue)) {
+                                if (!isset($allUIDs[$table])) {
+                                    $allUIDs[$table] = array();
+                                }
+                                $allUIDs[$table][] = $fieldValue;
+                            }
+                        }
+                    }
+                }
+                // Get overlays for all tables
+                $overlays = array();
+                $doOverlays = array();
+                foreach ($allUIDs as $table => $uidList) {
+                    // Make sure the uid's are unique
+                    $allUIDs[$table] = array_unique($uidList);
+                    $doOverlays[$table] = $this->sqlParser->mustHandleLanguageOverlay($table);
+                    // Get overlays only if needed/possible
+                    if ($doOverlays[$table] && count($allUIDs[$table]) > 0) {
+                        $overlays[$table] = OverlayEngine::getOverlayRecords($table, $allUIDs[$table],
+                                $GLOBALS['TSFE']->sys_language_content,
+                                $this->sqlParser->mustHandleVersioningOverlay($table));
+                        // Set global overlay process flag to true
+                        $hasBeenThroughOverlayProcess |= true;
+                    }
+                }
+                //\TYPO3\CMS\Core\Utility\DebugUtility::debug($allUIDs, 'Unique IDs per table');
+                //\TYPO3\CMS\Core\Utility\DebugUtility::debug($doOverlays, 'Do overlays?');
+                //\TYPO3\CMS\Core\Utility\DebugUtility::debug($overlays, 'Overlays');
+
+                // Loop on all recordset rows to overlay them
+                foreach ($versionedRecordset as $subParts) {
+                    // Overlay each part
+                    foreach ($subParts as $alias => $subRow) {
+                        $table = $allTablesTrueNames[$alias];
+                        $tableCtrl = $GLOBALS['TCA'][$table]['ctrl'];
+                        $result = $subRow;
+                        if ($GLOBALS['TSFE']->sys_page->versioningPreview && $this->sqlParser->mustHandleVersioningOverlay($table)) {
+                            $GLOBALS['TSFE']->sys_page->versionOL($table, $result);
+                        }
+                        if ($doOverlays[$table] && $subRow[$tableCtrl['languageField']] != $GLOBALS['TSFE']->sys_language_content) {
+                            // Overlay with record from foreign table
+                            if (isset($tableCtrl['transForeignTable']) && isset($overlays[$table][$subRow['uid']])) {
+                                $result = OverlayEngine::overlaySingleRecord($table, $subRow,
+                                        $overlays[$table][$subRow['uid']]);
+
+                                // Overlay with record from same table
+                            } elseif (isset($overlays[$table][$subRow['uid']][$subRow['pid']])) {
+                                $result = OverlayEngine::overlaySingleRecord($table, $subRow,
+                                        $overlays[$table][$subRow['uid']][$subRow['pid']]);
+
+                                // No overlay exists
+                            } else {
+                                // Take original record, only if non-translated are not hidden, or if language is [All]
+                                if ($GLOBALS['TSFE']->sys_language_contentOL == 'hideNonTranslated' && $subRow[$tableCtrl['languageField']] != -1) {
+                                    // Skip record
+                                    unset($result);
+                                } else {
+                                    $result = $subRow;
+                                }
+                            }
+                        }
+                        // Include result only if it was not unset during overlaying
+                        if (isset($result)) {
+                            $subParts[$alias] = $result;
+                        } else {
+                            unset($subParts[$alias]);
+                        }
+                    }
+                    // Reassemble the full record
+                    $overlaidRecord = array();
+                    foreach ($subParts as $alias => $subRow) {
+                        if (isset($subRow)) {
+                            foreach ($subRow as $field => $value) {
+                                // Remap fields, except for special key "_LOCALIZED_UID"
+                                if ($field === '_LOCALIZED_UID') {
+                                    // Differentiate main table and sub-tables
+                                    if ($alias == $this->mainTable) {
+                                        $key = '_LOCALIZED_UID';
+                                    } else {
+                                        $key = $alias . '$_LOCALIZED_UID';
+                                    }
+                                    $overlaidRecord[$key] = $value;
+                                } else {
+                                    $overlaidRecord[$reverseColumnsMappings[$alias][$field]] = $value;
+                                }
+                            }
+                            // If the record for main table does not exist anymore, remove it entirely
+                        } else {
+                            if ($alias == $this->mainTable) {
+                                unset($overlaidRecord);
+                                break;
+                            }
+                        }
+                    }
+                    //\TYPO3\CMS\Core\Utility\DebugUtility::debug($overlaidRecord, 'Overlaid record');
+                    if (isset($overlaidRecord['uid'])) {
+                        $finalRecordset[] = $overlaidRecord;
+                    }
+                }
+                // Clean up (potentially large) arrays that are not used anymore
+                unset($rawRecordset);
+                unset($overlays);
+                unset($subParts);
+                //\TYPO3\CMS\Core\Utility\DebugUtility::debug($finalRecordset, 'Overlaid recordset');
+
+                // If the dataquery was provided with a structure,
+                // use the list of uid's to define a fixed order of records
+                if (isset($this->structure['uidList'])) {
+                    $uidList = GeneralUtility::trimExplode(',', $this->structure['uidList']);
+                    $fixedOrder = array_flip($uidList);
+                    foreach ($finalRecordset as $index => $record) {
+                        $finalRecordset[$index]['tx_dataquery:fixed_order'] = $fixedOrder[$record['uid']];
+                    }
+                    unset($fixedOrder);
+                }
+                //\TYPO3\CMS\Core\Utility\DebugUtility::debug($finalRecordset, 'Final recordset before sorting');
+
+                // Perform sorting if not handled by SQL
+                if (!$this->sqlParser->isSqlUsedForOrdering()) {
+                    self::$sortingFields = $this->sqlParser->getOrderByFields();
+                    // The names of the fields as stored in the sorting fields configuration
+                    // match the names used in the SQL query, but not the aliases
+                    // So they will not match the column names in the full recordset
+                    // Use the reverse mapping information (if available) to get the aliases
+                    // (if defined, otherwise stick to field name)
+                    foreach (self::$sortingFields as $index => $orderInfo) {
+                        $alias = $this->mainTable;
+                        // Field may have a special alias which is also not the colum name
+                        // found in the recordset, but should be used to find that name
+                        $fieldName = (isset($orderInfo['alias'])) ? $orderInfo['alias'] : $orderInfo['field'];
+                        $fieldParts = explode('.', $fieldName);
+                        if (count($fieldParts) == 1) {
+                            $field = $fieldParts[0];
+                        } else {
+                            $alias = $fieldParts[0];
+                            $field = $fieldParts[1];
+                        }
+                        if (isset($reverseColumnsMappings[$alias][$field])) {
+                            self::$sortingFields[$index]['field'] = $reverseColumnsMappings[$alias][$field];
+                        } else {
+                            self::$sortingFields[$index]['field'] = $field;
+                        }
+                    }
+                    //\TYPO3\CMS\Core\Utility\DebugUtility::debug(self::$sortingFields, 'Sorting fields');
+                    self::$sortingLevel = 0;
+                    usort($finalRecordset, array('tx_dataquery_wrapper', 'sortRecordset'));
+                    //\TYPO3\CMS\Core\Utility\DebugUtility::debug($finalRecordset, 'Sorted, overlaid recordset');
+
+                    // If no sorting is defined at all, perform fixed order sorting, if defined
+                } elseif (!$this->sqlParser->hasOrdering() && isset($this->structure['uidList'])) {
+                    usort($finalRecordset, array('tx_dataquery_wrapper', 'sortUsingFixedOrder'));
+                }
+            } // End of translation handling
+
+            // Loop on all records to sort them by table. This can be seen as "de-JOINing" the tables.
+            // This is necessary for such operations as overlays. When overlays are done, tables will be joined again
+            // but within the format of Standardised Data Structure
+            $handledUids = array();
+            foreach ($finalRecordset as $row) {
+                $currentUID = $row['uid'];
+                // If we're handling a record we haven't handled before, perform some initialisations
+                if (!array_key_exists($currentUID, $handledUids)) {
+                    if ($numSubtables > 0) {
+                        foreach ($subtables as $table) {
+                            $rows[$table][$currentUID] = array();
+                        }
+                    }
+                }
+                $recordsPerTable = array();
+                foreach ($row as $fieldName => $fieldValue) {
+                    // The query contains no joined table
+                    // All fields belong to the main table
+                    if ($numSubtables === 0) {
+                        // If field is special field "_LOCALIZED_UID", keep its name as is
+                        if ($fieldName === '_LOCALIZED_UID') {
+                            $finalFieldName = '_LOCALIZED_UID';
+
+                            // Otherwise, use mapped name
+                        } else {
+                            $finalFieldName = $columnsMappings[$fieldName]['mapping']['field'];
+                        }
+                        $recordsPerTable[$this->mainTable][$finalFieldName] = $fieldValue;
+
+                        // There are multiple tables
+                    } else {
+                        // If field is special field "_LOCALIZED_UID" (possibly prepended with table name),
+                        // keep it as is
+                        if (strpos($fieldName, '_LOCALIZED_UID') !== false) {
+                            // Handle main table and sub-tables separately
+                            if ($fieldName === '_LOCALIZED_UID') {
+                                $recordsPerTable[$this->mainTable]['_LOCALIZED_UID'] = $fieldValue;
+                            } else {
+                                list($table, $finalFieldName) = explode('$', $fieldName);
+                                $recordsPerTable[$table]['_LOCALIZED_UID'] = $fieldValue;
+                            }
+                            // For other fields, get the field's true name
+                        } else {
+                            $finalFieldName = $columnsMappings[$fieldName]['mapping']['field'];
+                            // However, if the field had an explicit alias, use that alias
+                            if (isset($columnsMappings[$fieldName]['mapping']['alias'])) {
+                                $finalFieldName = $columnsMappings[$fieldName]['mapping']['alias'];
+                            }
+
+                            // Field belongs to a subtable
+                            if (in_array($columnsMappings[$fieldName]['mapping']['table'], $subtables)) {
+                                $subtableName = $columnsMappings[$fieldName]['mapping']['table'];
+                                if (isset($fieldValue)) {
+                                    $recordsPerTable[$subtableName][$finalFieldName] = $fieldValue;
+                                }
+                                // Else assume the field belongs to the main table
+                            } else {
+                                $recordsPerTable[$this->mainTable][$finalFieldName] = $fieldValue;
+                            }
+                        }
+                    }
+                }
+                // If we're handling a record we haven't handled before, store the current information for the main table
+                if (!array_key_exists($currentUID, $handledUids)) {
+                    $rows[$this->mainTable][0][] = $recordsPerTable[$this->mainTable];
+                    $handledUids[$currentUID] = $currentUID;
+                }
+                // Store information for each subtable
+                if ($numSubtables > 0) {
+                    foreach ($subtables as $table) {
+                        if (isset($recordsPerTable[$table]) && count($recordsPerTable[$table]) > 0) {
+                            $rows[$table][$currentUID][] = $recordsPerTable[$table];
+                        }
+                    }
+                }
+            }
+            // Clean up a potentially large array that is not used anymore
+            unset($finalRecordset);
+            unset($recordsPerTable);
+            //\TYPO3\CMS\Core\Utility\DebugUtility::debug($rows, 'De-JOINed tables');
+
+            // Now loop on all the records of the main table and join them to their subtables
+            $hasInnerJoin = $this->sqlParser->hasInnerJoinOnFirstSubtable();
+            $uidList = array();
+            foreach ($rows[$this->mainTable][0] as $aRecord) {
+                $uidList[] = $aRecord['uid'];
+                $theFullRecord = $aRecord;
+                $theFullRecord['__substructure'] = array();
+                // Check if there are any subtables in the query
+                $recordsPerSubtable = array();
+                if ($numSubtables > 0) {
+                    foreach ($subtables as $table) {
+                        // Check if there are any subrecords for this record
+                        if (isset($rows[$table][$aRecord['uid']])) {
+                            $numSubrecords = count($rows[$table][$aRecord['uid']]);
+                            if ($numSubrecords > 0) {
+                                $sublimit = $this->sqlParser->getSubTableLimit($table);
+                                $subcounter = 0;
+                                // Perform overlays only if language is not default and if necessary for table
+                                $subRecords = array();
+                                $subUidList = array();
+                                // Loop on all subrecords and apply limit, if any
+                                foreach ($rows[$table][$aRecord['uid']] as $subRow) {
+                                    if (!isset($subRow['uid'])) {
+                                        continue;
+                                    }
+                                    // Add the subrecord to the subtable only if it hasn't been included yet
+                                    // Multiple identical subrecords may happen when joining several tables together
+                                    // Take into account any limit that may have been placed on the number of subrecords in the query
+                                    // (using the non-SQL standard keyword MAX)
+                                    if (!in_array($subRow['uid'], $subUidList)) {
+                                        if ($sublimit == 0 || $subcounter < $sublimit) {
+                                            $subRecords[] = $subRow;
+                                            $subUidList[] = $subRow['uid'];
+                                        } elseif ($sublimit != 0 || $subcounter >= $sublimit) {
+                                            break;
+                                        }
+                                        $subcounter++;
+                                    }
+                                }
+                                // If there are indeed items, add the subtable to the record
+                                $numItems = count($subUidList);
+                                $recordsPerSubtable[$table] = $numItems;
+                                if ($numItems > 0) {
+                                    $theFullRecord['__substructure'][$table] = array(
+                                            'name' => $table,
+                                            'trueName' => $allTablesTrueNames[$table],
+                                            'count' => $numItems,
+                                            'uidList' => implode(',', $subUidList),
+                                            'header' => $headers[$table],
+                                            'records' => $subRecords
+                                    );
+                                }
+                            }
+                        }
+                    }
+                }
+                // If the query used INNER JOINs and went through the overlay process,
+                // preform additional checks
+                if ($numSubtables > 0 && !empty($hasInnerJoin) && $hasBeenThroughOverlayProcess) {
+                    // If there are no subrecords after the overlay process, but the query
+                    // used an INNER JOIN, the record must be removed, so that the end result
+                    // will look like what it would have been in the default language
+                    if (!empty($recordsPerSubtable[$hasInnerJoin])) {
+                        $fullRecords[] = $theFullRecord;
+                    }
+
+                    // Otherwise just take the record as is
+                } else {
+                    $fullRecords[] = $theFullRecord;
+                }
+            }
+            // Clean up a potentially large array that is not used anymore
+            unset($rows);
+        }
+        $databaseConnection->sql_free_result($res);
+
+        // Assemble the full structure
+        $numRecords = count($fullRecords);
+        $dataStructure = array(
+                'name' => $this->mainTable,
+                'trueName' => $allTablesTrueNames[$this->mainTable],
+                'count' => $numRecords,
+                'totalCount' => $numRecords,
+                'uidList' => implode(',', $uidList),
+                'header' => $headers[$this->mainTable],
+                'records' => $fullRecords
+        );
+        // Clean up a potentially large array that is not used anymore
+        unset($fullRecords);
+        //\TYPO3\CMS\Core\Utility\DebugUtility::debug($dataStructure, 'Finished data structure');
+
+        // Hook for post-processing the data structure before it is stored into cache
+        if (is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF'][$this->extKey]['postProcessDataStructureBeforeCache'])) {
+            foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF'][$this->extKey]['postProcessDataStructureBeforeCache'] as $className) {
+                $postProcessor = GeneralUtility::getUserObj($className);
+                $dataStructure = $postProcessor->postProcessDataStructureBeforeCache($dataStructure, $this);
+            }
+        }
+
+        // Store the structure in the cache table,
+        $this->writeStructureToCache($dataStructure);
+
+        // Finally return the assembled structure
+        return $dataStructure;
+    }
+
+    /**
+     * Wrapper around prepareFullStructure() for harmonizing naming between data providers
+     *
+     * @param mysqli_result $res Database resource from the executed query
+     * @return array A recordset-type data structure
+     * @see tx_dataquery_wrapper::prepareFullStructure()
+     */
+    protected function assembleRecordsetStructure($res)
+    {
+        return $this->prepareFullStructure($res);
+    }
+
+    /**
+     * Prepares an id-list type data structure based on the results of the SQL query
+     *
+     * @param mysqli_result $res Database resource from the executed query
+     * @return array An id-list type data structure
+     */
+    protected function assembleIdListStructure($res)
+    {
+        $databaseConnection = $this->getDatabaseConnection();
+        $uidList = array();
+        // Act only if there are records
+        if ($databaseConnection->sql_num_rows($res) > 0) {
+            $this->mainTable = $this->sqlParser->getMainTableName();
+
+            // Loop on all records to assemble the list of all uid's
+            // Note that all overlay mechanisms (language and version) are ignored when assembling
+            // an id-list type structure. Indeed the uid's returned by such a structure will always
+            // be fed into some other mechanism that will query the database for a more complete structure
+            // and overlays will be taken into account at that point
+            $uidColumns = array();
+            $firstPass = true;
+            while ($row = $databaseConnection->sql_fetch_assoc($res)) {
+                // On the first pass, assemble a list of all the columns containing uid's
+                // and their relative tables
+                if ($firstPass) {
+                    foreach ($row as $columnName => $value) {
+                        if ($columnName === 'uid' || substr($columnName, -4) === '$uid') {
+                            if ($columnName === 'uid') {
+                                $table = $this->mainTable;
+                            } else {
+                                $table = substr($columnName, 0, -4);
+                            }
+                            $uidColumns[$table] = $columnName;
+                            $uidList[$table] = array();
+                        }
+                    }
+                    $firstPass = false;
+                }
+                // Loop on all uid columns and store them relative to their related table
+                foreach ($uidColumns as $table => $columnName) {
+                    if (isset($row[$columnName])) {
+                        $uidList[$table][] = $row[$columnName];
+                    }
+                }
+            }
+            $databaseConnection->sql_free_result($res);
+
+            // Remove duplicates for every table
+            foreach ($uidList as $table => $uidListForTable) {
+                $uidList[$table] = array_unique($uidListForTable);
+            }
+        }
+
+        // Assemble the full structure
+        $numRecords = (isset($uidList[$this->mainTable])) ? count($uidList[$this->mainTable]) : 0;
+        $uidListString = (isset($uidList[$this->mainTable])) ? implode(',', $uidList[$this->mainTable]) : '';
+        $uidListWithTable = '';
+        foreach ($uidList as $table => $uidListForTable) {
+            if (!empty($uidListWithTable)) {
+                $uidListWithTable .= ',';
+            }
+            $uidListWithTable .= $table . '_' . implode(',' . $table . '_', $uidListForTable);
+        }
+        $dataStructure = array(
+                'uniqueTable' => (count($uidList) == 1) ? $this->mainTable : '',
+                'uidList' => $uidListString,
+                'uidListWithTable' => $uidListWithTable,
+                'count' => $numRecords,
+                'totalCount' => $numRecords
+        );
+
+        // Hook for post-processing the data structure before it is stored into cache
+        if (is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF'][$this->extKey]['postProcessDataStructureBeforeCache'])) {
+            foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF'][$this->extKey]['postProcessDataStructureBeforeCache'] as $className) {
+                $postProcessor = GeneralUtility::getUserObj($className);
+                $dataStructure = $postProcessor->postProcessDataStructureBeforeCache($dataStructure, $this);
+            }
+        }
+
+        // Store the structure in the cache table,
+        $this->writeStructureToCache($dataStructure);
+
+        // Finally return the assembled structure
+        return $dataStructure;
+    }
+
+    /**
+     * Splits the rows in the recordset returned by the database query into several parts,
+     * one for each (aliased) table in the row.
+     *
+     * Example:
+     * Consider the following query:
+     *
+     * SELECT pages.title, tt_content.header FROM pages,tt_content WHERE tt_content.pid = pages.uid
+     *
+     * A typical result row might look like:
+     *
+     * array('title' => 'foo', 'header' => 'bar');
+     *
+     * This method will turn such an entry into:
+     *
+     * array(
+     *        'pages' => array('title' => 'foo'),
+     *        'tt_content' => array('header' => 'bar'),
+     * );
+     *
+     * This is necessary so that each of these sub-parts can be overlaid (for language or version) properly.
+     *
+     * @param array $result Raw entry from the DB recordset
+     * @param array $columnsMappings Information about each column in the recordset
+     * @return array The result, split into its constituent tables
+     */
+    protected function splitResultIntoSubparts(array $result, array $columnsMappings)
+    {
+        $splitResults = array();
+        foreach ($result as $row) {
+            $subParts = array();
+            // Split record into parts related to a single table
+            foreach ($columnsMappings as $columnName => $columnInfo) {
+                if (!isset($subParts[$columnInfo['aliasTable']])) {
+                    $subParts[$columnInfo['aliasTable']] = array();
+                }
+                $subParts[$columnInfo['aliasTable']][$columnInfo['field']] = $row[$columnName];
+            }
+            $splitResults[] = $subParts;
+        }
+        return $splitResults;
+    }
+
+    /**
+     * Retrieves a data structure stored in cache provided it fits all parameters.
+     *
+     * If no appropriate cache is found, it throws an exception.
+     *
+     * @throws \Tesseract\Tesseract\Exception\Exception
+     * @return array
+     */
+    protected function getCachedStructure()
+    {
+        // Assemble condition for finding correct cache
+        // This means matching the dataquery's primary key, the current language, the filter's hash (without the limit)
+        // and that it has not expired
+        $where = 'query_id = ' . (int)$this->providerData['uid'] . ' AND page_id = ' . (int)$GLOBALS['TSFE']->id;
+        $where .= ' AND cache_hash = \'' . $this->calculateCacheHash(array()) . '\'';
+        $where .= ' AND expires > \'' . time() . '\'';
+        $row = $this->getDatabaseConnection()->exec_SELECTgetSingleRow(
+                'structure_cache',
+                'tx_dataquery_cache',
+                $where
+        );
+        if (empty($row)) {
+            throw new \Tesseract\Tesseract\Exception\Exception('No cached structure');
+        } else {
+            return unserialize($row['structure_cache']);
+        }
+    }
+
+    /**
+     * This method write the standard data structure to cache,
+     * provided some conditions are met
+     *
+     * @param array $structure A standard data structure
+     * @return void
+     */
+    protected function writeStructureToCache($structure)
+    {
+        // Write only if cache is active, i.e.
+        // if cache duration is not empty, and we're not in no_cache mode or in a workspace
+        if (!empty($this->providerData['cache_duration']) && empty($GLOBALS['TSFE']->no_cache) && empty($GLOBALS['TSFE']->sys_page->versioningPreview)) {
+            $cacheHash = $this->calculateCacheHash(array());
+            $serializedStructure = serialize($structure);
+            // Write only if serialized data is not too large
+            if (empty($this->configuration['cacheLimit']) || strlen($serializedStructure) <= $this->configuration['cacheLimit']) {
+                $fields = array(
+                        'query_id' => $this->providerData['uid'],
+                        'page_id' => $GLOBALS['TSFE']->id,
+                        'cache_hash' => $cacheHash,
+                        'structure_cache' => $serializedStructure,
+                        'expires' => time() + $this->providerData['cache_duration']
+                );
+                $this->getDatabaseConnection()->exec_INSERTquery('tx_dataquery_cache', $fields);
+            } // If data is too large for caching, make sure no other cache is left over
+            else {
+                $where = 'query_id = ' . (int)$this->providerData['uid'];
+                $where .= ' AND page_id = ' . (int)$GLOBALS['TSFE']->id;
+                $where .= ' AND cache_hash = \'' . $cacheHash . '\'';
+                $this->getDatabaseConnection()->exec_DELETEquery('tx_dataquery_cache', $where);
+            }
+        }
+    }
+
+    /**
+     * Assembles a hash parameter depending on a variety of parameters, including
+     * the current FE language and the groups of the current FE user, if any.
+     *
+     * @param array $parameters Additional parameters to add to the hash calculation
+     * @return string A md5 hash
+     */
+    protected function calculateCacheHash(array $parameters)
+    {
+        // The base of the hash parameters is the current filter
+        // To this we add the uidList (if it exists)
+        // This makes it possible to vary the cache as a function of the idList provided by the input structure
+        $filterForCache = $this->filter;
+        if (is_array($this->structure) && isset($this->structure['uidListWithTable'])) {
+            $filterForCache['uidListWithTable'] = $this->structure['uidListWithTable'];
+        }
+        // If some parameters were given, add them to the base cache parameters
+        $cacheParameters = $filterForCache;
+        if (is_array($parameters) && count($parameters) > 0) {
+            $cacheParameters = array_merge($cacheParameters, $parameters);
+        }
+        // Finally we add other parameters of uniqueness:
+        //     - the current FE language
+        //     - the groups of the currently logged in FE user (if any)
+        //     - the type of structure
+        $cacheParameters['sys_language_uid'] = $GLOBALS['TSFE']->sys_language_content;
+        if (is_array($GLOBALS['TSFE']->fe_user->user) && count($GLOBALS['TSFE']->fe_user->groupData['uid']) > 0) {
+            $cacheParameters['fe_groups'] = $GLOBALS['TSFE']->fe_user->groupData['uid'];
+        }
+        $cacheParameters['structure_type'] = $this->dataStructureType;
+        // Go through a hook for manipulating cache parameters
+        if (is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['dataquery']['processCacheHashParameters'])) {
+            foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['dataquery']['processCacheHashParameters'] as $className) {
+                /** @var $processor CacheParametersProcessorInterface */
+                $processor = GeneralUtility::getUserObj($className);
+                if ($processor instanceof CacheParametersProcessorInterface) {
+                    $cacheParameters = $processor->processCacheParameters($cacheParameters, $this);
+                }
+            }
+        }
+        // Calculate the hash using the method provided by the base controller,
+        // which filters out the "limit" part of the filter
+        return Utilities::calculateFilterCacheHash($cacheParameters);
+    }
+
+    /**
+     * Returns the name of the main table of the query,
+     * which is the table name that appears in the FROM clause, or the alias, if any.
+     *
+     * @return string Main table name
+     */
+    public function getMainTableName()
+    {
+        return $this->mainTable;
+    }
+
+    /**
+     * Performs a special sorting of the recordset.
+     *
+     * @param mixed $a First element to sort
+     * @param mixed $b Second element to sort
+     * @return integer -1 if first argument is smaller than second argument, 1 if first is greater than second and 0 if both are equal
+     *
+     * @see    \Tesseract\Dataquery\Component\DataProvider::prepareFullStructure()
+     */
+    static public function sortRecordset($a, $b)
+    {
+        // Get the sorting information from static variables
+        // The level is a pointer to the current field being used for sorting
+        $level = self::$sortingLevel;
+        $field = self::$sortingFields[$level]['field'];
+        $order = (empty(self::$sortingFields[$level]['order'])) ? 'ASC' : strtoupper(self::$sortingFields[$level]['order']);
+        $result = strnatcasecmp($a[$field], $b[$field]);
+        if ($result == 0) {
+            // If results are equal on the current level, check if there's a next level of sorting
+            // for differentiating the records
+            // If yes, call sorting method recursively
+            if (isset(self::$sortingFields[$level + 1])) {
+                self::$sortingLevel++;
+                $result = self::sortRecordset($a, $b);
+                self::$sortingLevel--;
+            }
+        } else {
+            if ($order === 'DESC') {
+                $result = -$result;
+            }
+        }
+        return $result;
+    }
+
+    /**
+     * Sorts records using a special fixed order value.
+     *
+     * @param mixed $a First element to sort
+     * @param mixed $b Second element to sort
+     * @return integer -1 if first argument is smaller than second argument, 1 if first is greater than second and 0 if both are equal
+     *
+     * @see    \Tesseract\Dataquery\Component\DataProvider::prepareFullStructure()
+     */
+    static public function sortUsingFixedOrder($a, $b)
+    {
+        $result = 1;
+        if ($a['tx_dataquery:fixed_order'] == $b['tx_dataquery:fixed_order']) {
+            $result = 0;
+        } elseif ($a['tx_dataquery:fixed_order'] < $b['tx_dataquery:fixed_order']) {
+            $result = -1;
+        }
+        return $result;
+    }
+
+    // Data Provider interface methods
+
+    /**
+     * Returns the type of data structure that the Data Provider can prepare.
+     *
+     * @return string Type of the provided data structure
+     */
+    public function getProvidedDataStructure()
+    {
+        return $this->dataStructureType;
     }
 
-// t3lib_svbase methods
-
-       /**
-        * Resets values for a number of properties.
-        *
-        * This is necessary because services are managed as singletons
-        *
-        * NOTE: If you make your own implementation of reset in your DataProvider class, don't forget to call parent::reset()
-        *
-        * @return      void
-        */
-       public function reset() {
-               parent::reset();
-               $this->initialise();
-       }
-
-       /**
-        * Returns the global database object.
-        *
-        * @return \TYPO3\CMS\Core\Database\DatabaseConnection
-        */
-       protected function getDatabaseConnection() {
-               return $GLOBALS['TYPO3_DB'];
-       }
+    /**
+     * Indicates whether the Data Provider can create the type of data structure requested or not.
+     *
+     * @param string $type : Type of data structure
+     * @return boolean TRUE if it can handle the requested type, FALSE otherwise
+     */
+    public function providesDataStructure($type)
+    {
+
+        // Check which type was requested and return true if type can be provided
+        // Store requested type internally for later processing
+        if ($type == Tesseract::IDLIST_STRUCTURE_TYPE) {
+            $this->dataStructureType = Tesseract::IDLIST_STRUCTURE_TYPE;
+            $result = true;
+        } elseif ($type == Tesseract::RECORDSET_STRUCTURE_TYPE) {
+            $this->dataStructureType = Tesseract::RECORDSET_STRUCTURE_TYPE;
+            $result = true;
+        } else {
+            $result = false;
+        }
+        return $result;
+    }
+
+    /**
+     * Returns the type of data structure that the Data Provider can receive as input.
+     *
+     * @return string Type of used data structures
+     */
+    public function getAcceptedDataStructure()
+    {
+        return Tesseract::IDLIST_STRUCTURE_TYPE;
+    }
+
+    /**
+     * Indicates whether the Data Provider can use as input the type of data structure requested or not.
+     *
+     * @param string $type Type of data structure
+     * @return boolean True if it can use the requested type, false otherwise
+     */
+    public function acceptsDataStructure($type)
+    {
+        return $type == Tesseract::IDLIST_STRUCTURE_TYPE;
+    }
+
+    /**
+     * Assembles the data structure and returns it.
+     *
+     * If the empty structure flag has been set, a dummy empty structure is returned instead.
+     *
+     * @return array Standardised data structure
+     */
+    public function getDataStructure()
+    {
+        // If the empty output structure flag was raised, prepare a proper structure devoid of data
+        if ($this->hasEmptyOutputStructure) {
+            try {
+                // Parse the query to get the main table's name
+                $this->sqlParser->parseQuery($this->providerData['sql_query']);
+                $this->initEmptyDataStructure(
+                        $this->sqlParser->getMainTableName(),
+                        $this->dataStructureType
+                );
+            } catch (\Exception $e) {
+                $this->controller->addMessage($this->extKey, $e->getMessage() . ' (' . $e->getCode() . ')',
+                        'Query parsing error', FlashMessage::ERROR, $this->providerData);
+            }
+        } else {
+            return $this->getQueryData();
+        }
+        return $this->outputStructure;
+    }
+
+    /**
+     * Passes a data structure to the Data Provider.
+     *
+     * @param array $structure Standardised data structure
+     * @return void
+     */
+    public function setDataStructure($structure)
+    {
+        if (is_array($structure)) {
+            $this->structure = $structure;
+        }
+    }
+
+    /**
+     * Loads the query and gets the list of tables and fields,
+     * complete with localized labels.
+     *
+     * @param string $language 2-letter iso code for language
+     * @return array List of tables and fields
+     */
+    public function getTablesAndFields($language = '')
+    {
+        $this->sqlParser->parseQuery($this->providerData['sql_query']);
+        $tablesAndFields = $this->sqlParser->getLocalizedLabels($language);
+
+        // Hook for post-processing the tables and fields information
+        if (is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF'][$this->extKey]['postProcessFieldInformation'])) {
+            foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF'][$this->extKey]['postProcessFieldInformation'] as $className) {
+                $postProcessor = GeneralUtility::getUserObj($className);
+                $tablesAndFields = $postProcessor->postProcessFieldInformation($tablesAndFields, $this);
+            }
+        }
+        return $tablesAndFields;
+    }
+
+    // t3lib_svbase methods
+
+    /**
+     * Resets values for a number of properties.
+     *
+     * This is necessary because services are managed as singletons
+     *
+     * NOTE: If you make your own implementation of reset in your DataProvider class, don't forget to call parent::reset()
+     *
+     * @return    void
+     */
+    public function reset()
+    {
+        parent::reset();
+        $this->initialise();
+    }
+
+    /**
+     * Returns the global database object.
+     *
+     * @return \TYPO3\CMS\Core\Database\DatabaseConnection
+     */
+    protected function getDatabaseConnection()
+    {
+        return $GLOBALS['TYPO3_DB'];
+    }
 }
index deb19b7..268340d 100644 (file)
@@ -21,5 +21,6 @@ namespace Tesseract\Dataquery\Exception;
  * @package TYPO3
  * @subpackage tx_tesseract
  */
-class InvalidQueryException extends \Exception {
+class InvalidQueryException extends \Exception
+{
 }
index 3986e10..dd2fe9e 100644 (file)
@@ -26,35 +26,38 @@ use Tesseract\Datafilter\PostprocessEmptyFilterCheckInterface;
  * @package TYPO3
  * @subpackage tx_dataquery
  */
-class DataFilterHook implements PostprocessFilterInterface, PostprocessEmptyFilterCheckInterface {
+class DataFilterHook implements PostprocessFilterInterface, PostprocessEmptyFilterCheckInterface
+{
 
-       /**
-        * Handles the tx_dataquery_sql field and adds it
-        * to the filter itself.
-        *
-        * @param DataFilter $filter A datafilter object
-        * @return void
-        */
-       public function postprocessFilter(DataFilter $filter) {
-               $filterData = $filter->getData();
-               if (!empty($filterData['tx_dataquery_sql'])) {
-                       // Parse any expressions inside the additional sql field
-                       $additionalSQL = ExpressionParser::evaluateString($filterData['tx_dataquery_sql'], FALSE);
-                       $filterArray = $filter->getFilter();
-                       $filterArray['rawSQL'] = $additionalSQL;
-                       $filter->setFilter($filterArray);
-               }
-       }
+    /**
+     * Handles the tx_dataquery_sql field and adds it
+     * to the filter itself.
+     *
+     * @param DataFilter $filter A datafilter object
+     * @return void
+     */
+    public function postprocessFilter(DataFilter $filter)
+    {
+        $filterData = $filter->getData();
+        if (!empty($filterData['tx_dataquery_sql'])) {
+            // Parse any expressions inside the additional sql field
+            $additionalSQL = ExpressionParser::evaluateString($filterData['tx_dataquery_sql'], false);
+            $filterArray = $filter->getFilter();
+            $filterArray['rawSQL'] = $additionalSQL;
+            $filter->setFilter($filterArray);
+        }
+    }
 
-       /**
-        * Modified the empty filter check to take the tx_dataquery_sql field into account.
-        *
-        * @param boolean $isEmpty Current value of the is filter empty flag
-        * @param DataFilter $filter The calling filter object
-        * @return boolean
-        */
-       public function postprocessEmptyFilterCheck($isEmpty, DataFilter $filter) {
-               $filterStructure = $filter->getFilter();
-               return $isEmpty && empty($filterStructure['rawSQL']);
-       }
+    /**
+     * Modified the empty filter check to take the tx_dataquery_sql field into account.
+     *
+     * @param boolean $isEmpty Current value of the is filter empty flag
+     * @param DataFilter $filter The calling filter object
+     * @return boolean
+     */
+    public function postprocessEmptyFilterCheck($isEmpty, DataFilter $filter)
+    {
+        $filterStructure = $filter->getFilter();
+        return $isEmpty && empty($filterStructure['rawSQL']);
+    }
 }
index 7bc7415..caac927 100644 (file)
@@ -15,6 +15,7 @@ namespace Tesseract\Dataquery\Parser;
  */
 
 use Tesseract\Dataquery\Exception\InvalidQueryException;
+use Tesseract\Dataquery\Utility\DatabaseAnalyser;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 
 /**
@@ -25,159 +26,168 @@ use TYPO3\CMS\Core\Utility\GeneralUtility;
  * @package TYPO3
  * @subpackage dataquery
  */
-class FulltextParser {
-
-       /**
-        * @var string
-        */
-       protected $searchTerms = array();
-
-       /**
-        * @var \Tesseract\Dataquery\Utility\DatabaseAnalyser
-        */
-       protected $analyser;
-
-       /**
-        * @var array
-        */
-       protected $indexedFields = array();
-
-       /**
-        * Unserialized extension configuration
-        * @var array
-        */
-       protected $configuration;
-
-       /**
-        * Constructor
-        *
-        * @return FulltextParser
-        */
-       public function __construct() {
-
-               /** @var $analyser \Tesseract\Dataquery\Utility\DatabaseAnalyser */
-               $analyser = GeneralUtility::makeInstance('Tesseract\\Dataquery\\Utility\\DatabaseAnalyser');
-               $this->setAnalyser($analyser);
-               $this->configuration = unserialize($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf']['dataquery']);
-       }
-
-       /**
-        * Retrieves full-text index fields for a given table.
-        *
-        * @param string $tableName
-        */
-       protected function retrieveIndexedFields($tableName) {
-               $this->indexedFields = $this->analyser->getFields($tableName);
-       }
-
-       /**
-        * Sets the analyser.
-        *
-        * Useful for unit tests.
-        *
-        * @param \Tesseract\Dataquery\Utility\DatabaseAnalyser $analyser
-        */
-       public function setAnalyser($analyser) {
-               $this->analyser = $analyser;
-       }
-
-       /**
-        * Sets the extension configuration.
-        *
-        * Useful for unit tests.
-        *
-        * @param array $configuration Extension configuration
-        * @return void
-        */
-       public function setConfiguration($configuration) {
-               $this->configuration = $configuration;
-       }
-
-       /**
-        * Parses the query. If a placeholder "fulltext:foo" is found, then replace with a MATCH / AGAINST expression.
-        *
-        * @param string $table Name of the table to search
-        * @param string $index Name of the fulltext index to use
-        * @param string $search Search string
-        * @param boolean $isNaturalSearch TRUE if fulltext search should be in natural mode
-        * @param boolean $isNegated TRUE if condition should be negated
-        * @return string SQL MATCH() condition
-        * @throws InvalidQueryException
-        */
-       public function parse($table, $index, $search, $isNaturalSearch, $isNegated) {
-               $this->retrieveIndexedFields($table);
-               if (isset($this->indexedFields[$index])) {
-                       $indexFields = $this->indexedFields[$index];
-               } else {
-                       throw new InvalidQueryException(
-                               sprintf('Table %s has no index "%s"', $table, $index),
-                               1421769189
-                       );
-               }
-               // Search terms from a query string will be urlencode'd
-               $processedSearchTerms = urldecode($search);
-               $booleanMode = '';
-               if (!$isNaturalSearch) {
-                       $processedSearchTerms = $this->processSearchTerm($processedSearchTerms);
-                       $booleanMode = ' IN BOOLEAN MODE';
-               }
-               if (empty($processedSearchTerms)) {
-                       throw new InvalidQueryException(
-                               'Empty fulltext search condition',
-                               1423068811
-                       );
-               }
-               $baseCondition = "MATCH(%s) AGAINST('%s'%s)";
-               if ($isNegated) {
-                       $baseCondition = 'NOT ' . $baseCondition;
-               }
-               $condition = sprintf($baseCondition, $indexFields, $processedSearchTerms, $booleanMode);
-               return $condition;
-       }
-
-       /**
-        * Processes the search term.
-        *
-        * @param string $term Search term
-        * @return string
-        */
-       public function processSearchTerm($term) {
-
-               $termsProcessed = array();
-
-               // Handle double quote wrapping
-               if (preg_match_all('/".+"/isU', $term, $matches)) {
-
-                       foreach ($matches as $match) {
-                               $searchedCharacters = array(
-                                       '"',
-                                       ' '
-                               );
-                               $replacedCharacters = array(
-                                       '',
-                                       '###'
-                               );
-                               $search = $match;
-                               $replace = str_replace($searchedCharacters, $replacedCharacters, $match);
-                               $term = str_replace($search, $replace, $term);
-                       }
-               }
-
-               $terms = explode(' ', $term);
-               foreach ($terms as $term) {
-                       if (!empty($term)) {
-                               // Handle exclusion of term
-                               $logic = '+';
-                               if (substr($term, 0, 1) == '-') {
-                                       $term = substr($term, 1);
-                                       $logic = '-';
-                               }
-                               if (strlen($term) >= $this->configuration['fullTextMinimumWordLength']) {
-                                       $termProcessed = str_replace('###', ' ', addslashes($term));
-                                       $termsProcessed[] = sprintf('%s"%s"', $logic, $termProcessed);
-                               }
-                       }
-               }
-               return implode(' ', $termsProcessed);
-       }
+class FulltextParser
+{
+
+    /**
+     * @var string
+     */
+    protected $searchTerms = array();
+
+    /**
+     * @var DatabaseAnalyser
+     */
+    protected $analyser;
+
+    /**
+     * @var array
+     */
+    protected $indexedFields = array();
+
+    /**
+     * Unserialized extension configuration
+     * @var array
+     */
+    protected $configuration;
+
+    /**
+     * Constructor
+     *
+     * @return FulltextParser
+     */
+    public function __construct()
+    {
+
+        /** @var $analyser DatabaseAnalyser */
+        $analyser = GeneralUtility::makeInstance(
+                DatabaseAnalyser::class
+    );
+        $this->setAnalyser($analyser);
+        $this->configuration = unserialize($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf']['dataquery']);
+    }
+
+    /**
+     * Retrieves full-text index fields for a given table.
+     *
+     * @param string $tableName
+     */
+    protected function retrieveIndexedFields($tableName)
+    {
+        $this->indexedFields = $this->analyser->getFields($tableName);
+    }
+
+    /**
+     * Sets the analyser.
+     *
+     * Useful for unit tests.
+     *
+     * @param DatabaseAnalyser $analyser
+     */
+    public function setAnalyser($analyser)
+    {
+        $this->analyser = $analyser;
+    }
+
+    /**
+     * Sets the extension configuration.
+     *
+     * Useful for unit tests.
+     *
+     * @param array $configuration Extension configuration
+     * @return void
+     */
+    public function setConfiguration($configuration)
+    {
+        $this->configuration = $configuration;
+    }
+
+    /**
+     * Parses the query. If a placeholder "fulltext:foo" is found, then replace with a MATCH / AGAINST expression.
+     *
+     * @param string $table Name of the table to search
+     * @param string $index Name of the fulltext index to use
+     * @param string $search Search string
+     * @param boolean $isNaturalSearch TRUE if fulltext search should be in natural mode
+     * @param boolean $isNegated TRUE if condition should be negated
+     * @return string SQL MATCH() condition
+     * @throws InvalidQueryException
+     */
+    public function parse($table, $index, $search, $isNaturalSearch, $isNegated)
+    {
+        $this->retrieveIndexedFields($table);
+        if (isset($this->indexedFields[$index])) {
+            $indexFields = $this->indexedFields[$index];
+        } else {
+            throw new InvalidQueryException(
+                    sprintf('Table %s has no index "%s"', $table, $index),
+                    1421769189
+            );
+        }
+        // Search terms from a query string will be urlencode'd
+        $processedSearchTerms = urldecode($search);
+        $booleanMode = '';
+        if (!$isNaturalSearch) {
+            $processedSearchTerms = $this->processSearchTerm($processedSearchTerms);
+            $booleanMode = ' IN BOOLEAN MODE';
+        }
+        if (empty($processedSearchTerms)) {
+            throw new InvalidQueryException(
+                    'Empty fulltext search condition',
+                    1423068811
+            );
+        }
+        $baseCondition = "MATCH(%s) AGAINST('%s'%s)";
+        if ($isNegated) {
+            $baseCondition = 'NOT ' . $baseCondition;
+        }
+        $condition = sprintf($baseCondition, $indexFields, $processedSearchTerms, $booleanMode);
+        return $condition;
+    }
+
+    /**
+     * Processes the search term.
+     *
+     * @param string $term Search term
+     * @return string
+     */
+    public function processSearchTerm($term)
+    {
+
+        $termsProcessed = array();
+
+        // Handle double quote wrapping
+        if (preg_match_all('/".+"/isU', $term, $matches)) {
+
+            foreach ($matches as $match) {
+                $searchedCharacters = array(
+                        '"',
+                        ' '
+                );
+                $replacedCharacters = array(
+                        '',
+                        '###'
+                );
+                $search = $match;
+                $replace = str_replace($searchedCharacters, $replacedCharacters, $match);
+                $term = str_replace($search, $replace, $term);
+            }
+        }
+
+        $terms = explode(' ', $term);
+        foreach ($terms as $aTerm) {
+            if (!empty($aTerm)) {
+                // Handle exclusion of term
+                $logic = '+';
+                if (strpos($aTerm, '-') === 0) {
+                    $aTerm = substr($aTerm, 1);
+                    $logic = '-';
+                }
+                if (strlen($aTerm) >= $this->configuration['fullTextMinimumWordLength']) {
+                    $termProcessed = str_replace('###', ' ', addslashes($aTerm));
+                    $termsProcessed[] = sprintf('%s"%s"', $logic, $termProcessed);
+                }
+            }
+        }
+        return implode(' ', $termsProcessed);
+    }
 }
index dc748ad..5dbdb0e 100644 (file)
@@ -33,1427 +33,1491 @@ use TYPO3\CMS\Core\Utility\GeneralUtility;
  * @package TYPO3
  * @subpackage tx_dataquery
  */
-class QueryParser {
-       static public $extKey = 'dataquery';
-
-       /**
-        * List of eval types which indicate non-text fields
-        * @var array   $notTextTypes
-        */
-       static protected $notTextTypes = array('date', 'datetime', 'time', 'timesec', 'year', 'num', 'md5', 'int', 'double2');
-
-       /**
-        * Reference to the calling object
-        * @var \Tesseract\Dataquery\Component\DataProvider $parentObject
-        */
-       protected $parentObject;
-
-       /**
-        * Unserialized extension configuration
-        * @var array $configuration
-        */
-       protected $configuration;
-
-       /**
-        * Structured type containing the parts of the parsed query
-        * @var \Tesseract\Dataquery\Utility\QueryObject $queryObject
-        */
-       protected $queryObject;
-
-       /**
-        * True names for all the fields. The key is the actual alias used in the query.
-        * @var array $fieldTrueNames
-        */
-       protected $fieldTrueNames = array();
-
-       /**
-        * List of all fields being queried, arranged per table (aliased)
-        * @var array $queryFields
-        */
-       protected $queryFields = array();
-
-       /**
-        * Flag for each table whether to perform overlays or not
-        * @var array
-        */
-       protected $doOverlays = array();
-
-       /**
-        * Flag for each table whether to perform versioning overlays or not
-        * @var array
-        */
-       protected $doVersioning = array();
-
-       /**
-        * True if order by is processed using SQL, false otherwise (see preprocessOrderByFields())
-        * @var boolean
-        */
-       protected $processOrderBy = TRUE;
-
-       /**
-        * Cache array to store table name matches
-        * @var array
-        * @see matchAliasOrTableNeme()
-        */
-       protected $tableMatches = array();
-
-       /**
-        * @var array Database record corresponding to the current Data Query
-        */
-       protected $providerData;
-
-       public function  __construct($parentObject) {
-               $this->parentObject = $parentObject;
-               $this->configuration = unserialize($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf']['dataquery']);
-       }
-
-       /**
-        * This method is used to parse a SELECT SQL query.
-        * It is a simple parser and no way generic. It expects queries to be written a certain way.
-        *
-        * @param string $query The query to be parsed
-        * @throws InvalidQueryException
-        * @return string A warning message, if any (fatal errors throw exceptions)
-        */
-       public function parseQuery($query) {
-               $warning = '';
-               // Clean up and prepare the query string
-               $query = $this->prepareQueryString($query);
-
-               // Parse the SQL query
-               /** @var $sqlParser \Tesseract\Dataquery\Parser\SqlParser */
-               $sqlParser = GeneralUtility::makeInstance('Tesseract\\Dataquery\\Parser\\SqlParser');
-               // NOTE: the following call may throw exceptions,
-               // but we let them bubble up
-               $this->queryObject = $sqlParser->parseSQL($query);
-               // Perform some further analysis on the query components
-               $this->analyzeQuery();
-               // Make sure the list of selected fields contains base fields
-               // like uid and pid (if available)
-               // Don't do this for queries using the DISTINCT keyword, as it may mess it up
-               if (!$this->queryObject->structure['DISTINCT']) {
-                       $this->addBaseFields();
-
-               // If the query uses the DISTINCT keyword, check if a "uid" field has been defined manually
-               // If not, issue warning
-               } else {
-                       if (!$this->checkUidForDistinctUsage()) {
-                               throw new InvalidQueryException('"uid" field missing with DISTINCT usage', 1313354033);
-                       }
-               }
-               // Check if the query selects the same field multiple times
-               // Issue a warning if yes, since the results may be unpredictable
-               $duplicates = $this->checkForDuplicateFields();
-               if (count($duplicates) > 0) {
-                       $warning = 'Duplicate fields in query: ' . implode(' / ', $duplicates);
-               }
-
-//\TYPO3\CMS\Core\Utility\DebugUtility::debug($this->queryObject->aliases, 'Table aliases');
-//\TYPO3\CMS\Core\Utility\DebugUtility::debug($this->fieldAliases, 'Field aliases');
-//\TYPO3\CMS\Core\Utility\DebugUtility::debug($this->fieldTrueNames, 'Field true names');
-//\TYPO3\CMS\Core\Utility\DebugUtility::debug($this->queryFields, 'Query fields');
-//\TYPO3\CMS\Core\Utility\DebugUtility::debug($this->queryObject->structure, 'Structure');
-               return $warning;
-       }
-
-       /**
-        * This method performs a number of operations on a given string,
-        * supposed to be a SQL query
-        * It is meant to be called before the query is actually parsed
-        *
-        * @param       string  $string: a SQL query
-        * @return      string  Cleaned up SQL query
-        */
-       public function prepareQueryString($string) {
-               // Put the query through the field parser to filter out commented lines
-               $queryLines = Utilities::parseConfigurationField($string);
-               // Put the query into a single string
-               $query = implode(' ', $queryLines);
-               // Strip backquotes
-               $query = str_replace('`', '', $query);
-               // Strip trailing semi-colon if any
-               if (strrpos($query, ';') == strlen($query) - 1) {
-                       $query = substr($query, 0, -1);
-               }
-               // Parse query for subexpressions
-               $query = ExpressionParser::evaluateString($query, FALSE);
-               return $query;
-       }
-
-       /**
-        * Analyzes the query in more depth.
-        *
-        * In particular, it loops on all SELECT field and makes sure every field
-        * has a proper alias.
-        *
-        * @return void
-        */
-       protected function analyzeQuery() {
-               // Loop on all query fields to assemble additional information structures
-               foreach ($this->queryObject->structure['SELECT'] as $index => $fieldInfo) {
-                       // Assemble list of fields per table
-                       // The name of the field is used both as key and value, but the value will be replaced by the fields' labels in getLocalizedLabels()
-                       if (!isset($this->queryFields[$fieldInfo['tableAlias']])) {
-                               $this->queryFields[$fieldInfo['tableAlias']] = array('name' => $fieldInfo['table'], 'table' => $fieldInfo['tableAlias'], 'fields' => array());
-                       }
-                       $this->queryFields[$fieldInfo['tableAlias']]['fields'][] = array('name' => $fieldInfo['field'], 'function' => $fieldInfo['function']);
-
-                       // Assemble full names for each field
-                       // The full name is:
-                       //      1) the name of the table or its alias
-                       //      2) a dot
-                       //      3) the name of the field
-                       //
-                       // => If it's the main table and there's an alias for the field
-                       //
-                       //      4a) AS and the field alias
-                       //
-                       //      4a-2)   if the alias contains a dot (.) it means it contains a table name (or alias)
-                       //                      and a field name. So we use this information
-                       //
-                       // This means something like foo.bar AS hey.you will get transformed into foo.bar AS hey$you
-                       //
-                       // In effect this means that you can "reassign" a value from one table (foo) to another (hey)
-                       //
-                       // => If it's not the main table, all fields get an alias using either their own name or the given field alias
-                       //
-                       //      4b) AS and $ and the field or its alias
-                       //
-                       // So something like foo.bar AS boink will get transformed into foo.bar AS foo$boink
-                       //
-                       //      4b-2)   like 4a-2) above, but for subtables
-                       //
-                       // The $ sign is used in class tx_dataquery_wrapper for building the data structure
-                       // Initialize values
-                       $mappedField = '';
-                       $mappedTable = '';
-                       $fullField = $fieldInfo['tableAlias'] . '.' . $fieldInfo['field'];
-                       if ($fieldInfo['function']) {
-                               $fullField = $fieldInfo['field'];
-                       }
-                       $theField = $fieldInfo['field'];
-                       // Case 4a
-                       if ($fieldInfo['tableAlias'] == $this->queryObject->mainTable) {
-                               if (empty($fieldInfo['fieldAlias'])) {
-                                       $theAlias = $theField;
-                               } else {
-                                       $fullField .= ' AS ';
-                                       if (strpos($fieldInfo['fieldAlias'], '.') === FALSE) {
-                                               $theAlias = $fieldInfo['fieldAlias'];
-                                               $mappedTable = $fieldInfo['tableAlias'];
-                                               $mappedField = $fieldInfo['fieldAlias'];
-                                       }
-                                       // Case 4a-2
-                                       else {
-                                               list($mappedTable, $mappedField) = explode('.', $fieldInfo['fieldAlias']);
-                                               $theAlias = str_replace('.', '$', $fieldInfo['fieldAlias']);
-                                       }
-                                       $fullField .= $theAlias;
-                               }
-                       } else {
-                               $fullField .= ' AS ';
-                               if (empty($fieldInfo['fieldAlias'])) {
-                                       $theAlias = $fieldInfo['tableAlias'] . '$' . $fieldInfo['field'];
-                               }
-                               else {
-                                       // Case 4b
-                                       if (strpos($fieldInfo['fieldAlias'], '.') === FALSE) {
-                                               $theAlias = $fieldInfo['tableAlias'] . '$' . $fieldInfo['fieldAlias'];
-                                       }
-                                       // Case 4b-2
-                                       else {
-                                               list($mappedTable, $mappedField) = explode('.', $fieldInfo['fieldAlias']);
-                                               $theAlias = str_replace('.', '$', $fieldInfo['fieldAlias']);
-                                       }
-                               }
-                               $fullField .= $theAlias;
-                       }
-                       if (empty($mappedTable)) {
-                               $mappedTable = $fieldInfo['tableAlias'];
-                               $mappedField = $theField;
-                       }
-                       $this->fieldTrueNames[$theAlias] = array(
-                               'table' => $fieldInfo['table'],
-                               'aliasTable' => $fieldInfo['tableAlias'],
-                               'field' => $theField,
-                               'mapping' => array('table' => $mappedTable, 'field' => $mappedField)
-                       );
-                       $this->queryObject->structure['SELECT'][$index] = $fullField;
+class QueryParser
+{
+    static public $extKey = 'dataquery';
+
+    /**
+     * List of eval types which indicate non-text fields
+     * @var    array $notTextTypes
+     */
+    static protected $notTextTypes = array(
+            'date',
+            'datetime',
+            'time',
+            'timesec',
+            'year',
+            'num',
+            'md5',
+            'int',
+            'double2'
+    );
+
+    /**
+     * Reference to the calling object
+     * @var \Tesseract\Dataquery\Component\DataProvider $parentObject
+     */
+    protected $parentObject;
+
+    /**
+     * Unserialized extension configuration
+     * @var array $configuration
+     */
+    protected $configuration;
+
+    /**
+     * Structured type containing the parts of the parsed query
+     * @var \Tesseract\Dataquery\Utility\QueryObject $queryObject
+     */
+    protected $queryObject;
+
+    /**
+     * True names for all the fields. The key is the actual alias used in the query.
+     * @var array $fieldTrueNames
+     */
+    protected $fieldTrueNames = array();
+
+    /**
+     * List of all fields being queried, arranged per table (aliased)
+     * @var array $queryFields
+     */
+    protected $queryFields = array();
+
+    /**
+     * Flag for each table whether to perform overlays or not
+     * @var    array
+     */
+    protected $doOverlays = array();
+
+    /**
+     * Flag for each table whether to perform versioning overlays or not
+     * @var    array
+     */
+    protected $doVersioning = array();
+
+    /**
+     * True if order by is processed using SQL, false otherwise (see preprocessOrderByFields())
+     * @var    boolean
+     */
+    protected $processOrderBy = true;
+
+    /**
+     * Cache array to store table name matches
+     * @var array
+     * @see matchAliasOrTableNeme()
+     */
+    protected $tableMatches = array();
+
+    /**
+     * @var array Database record corresponding to the current Data Query
+     */
+    protected $providerData;
+
+    public function __construct($parentObject)
+    {
+        $this->parentObject = $parentObject;
+        $this->configuration = unserialize($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf']['dataquery']);
+    }
+
+    /**
+     * This method is used to parse a SELECT SQL query.
+     * It is a simple parser and no way generic. It expects queries to be written a certain way.
+     *
+     * @param string $query The query to be parsed
+     * @throws InvalidQueryException
+     * @return string A warning message, if any (fatal errors throw exceptions)
+     */
+    public function parseQuery($query)
+    {
+        $warning = '';
+        // Clean up and prepare the query string
+        $query = $this->prepareQueryString($query);
+
+        // Parse the SQL query
+        /** @var $sqlParser SqlParser */
+        $sqlParser = GeneralUtility::makeInstance(
+                SqlParser::class
+        );
+        // NOTE: the following call may throw exceptions,
+        // but we let them bubble up
+        $this->queryObject = $sqlParser->parseSQL($query);
+        // Perform some further analysis on the query components
+        $this->analyzeQuery();
+        // Make sure the list of selected fields contains base fields
+        // like uid and pid (if available)
+        // Don't do this for queries using the DISTINCT keyword, as it may mess it up
+        if (!$this->queryObject->structure['DISTINCT']) {
+            $this->addBaseFields();
+
+            // If the query uses the DISTINCT keyword, check if a "uid" field has been defined manually
+            // If not, issue warning
+        } else {
+            if (!$this->checkUidForDistinctUsage()) {
+                throw new InvalidQueryException('"uid" field missing with DISTINCT usage', 1313354033);
+            }
+        }
+        // Check if the query selects the same field multiple times
+        // Issue a warning if yes, since the results may be unpredictable
+        $duplicates = $this->checkForDuplicateFields();
+        if (count($duplicates) > 0) {
+            $warning = 'Duplicate fields in query: ' . implode(' / ', $duplicates);
+        }
+
+        //\TYPO3\CMS\Core\Utility\DebugUtility::debug($this->queryObject->aliases, 'Table aliases');
+        //\TYPO3\CMS\Core\Utility\DebugUtility::debug($this->fieldAliases, 'Field aliases');
+        //\TYPO3\CMS\Core\Utility\DebugUtility::debug($this->fieldTrueNames, 'Field true names');
+        //\TYPO3\CMS\Core\Utility\DebugUtility::debug($this->queryFields, 'Query fields');
+        //\TYPO3\CMS\Core\Utility\DebugUtility::debug($this->queryObject->structure, 'Structure');
+        return $warning;
+    }
+
+    /**
+     * This method performs a number of operations on a given string,
+     * supposed to be a SQL query
+     * It is meant to be called before the query is actually parsed
+     *
+     * @param    string $string : a SQL query
+     * @return    string    Cleaned up SQL query
+     */
+    public function prepareQueryString($string)
+    {
+        // Put the query through the field parser to filter out commented lines
+        $queryLines = Utilities::parseConfigurationField($string);
+        // Put the query into a single string
+        $query = implode(' ', $queryLines);
+        // Strip backquotes
+        $query = str_replace('`', '', $query);
+        // Strip trailing semi-colon if any
+        if (strrpos($query, ';') == strlen($query) - 1) {
+            $query = substr($query, 0, -1);
         }
-       }
-
-       /**
-        * Checks every table that doesn't have a uid or pid field and tries to add it
-        * to the list of fields to select.
-        *
-        * @return void
-        */
-       protected function addBaseFields() {
-               // Loop on the tables that don't have a uid field
+        // Parse query for subexpressions
+        $query = ExpressionParser::evaluateString($query, false);
+        return $query;
+    }
+
+    /**
+     * Analyzes the query in more depth.
+     *
+     * In particular, it loops on all SELECT field and makes sure every field
+     * has a proper alias.
+     *
+     * @return void
+     */
+    protected function analyzeQuery()
+    {
+        // Loop on all query fields to assemble additional information structures
+        foreach ($this->queryObject->structure['SELECT'] as $index => $fieldInfo) {
+            // Assemble list of fields per table
+            // The name of the field is used both as key and value, but the value will be replaced by the fields' labels in getLocalizedLabels()
+            if (!isset($this->queryFields[$fieldInfo['tableAlias']])) {
+                $this->queryFields[$fieldInfo['tableAlias']] = array(
+                        'name' => $fieldInfo['table'],
+                        'table' => $fieldInfo['tableAlias'],
+                        'fields' => array()
+                );
+            }
+            $this->queryFields[$fieldInfo['tableAlias']]['fields'][] = array(
+                    'name' => $fieldInfo['field'],
+                    'function' => $fieldInfo['function']
+            );
+
+            // Assemble full names for each field
+            // The full name is:
+            // 1) the name of the table or its alias
+            // 2) a dot
+            // 3) the name of the field
+            //
+            // => If it's the main table and there's an alias for the field
+            //
+            // 4a) AS and the field alias
+            //
+            // 4a-2)   if the alias contains a dot (.) it means it contains a table name (or alias)
+            //                 and a field name. So we use this information
+            //
+            // This means something like foo.bar AS hey.you will get transformed into foo.bar AS hey$you
+            //
+            // In effect this means that you can "reassign" a value from one table (foo) to another (hey)
+            //
+            // => If it's not the main table, all fields get an alias using either their own name or the given field alias
+            //
+            // 4b) AS and $ and the field or its alias
+            //
+            // So something like foo.bar AS boink will get transformed into foo.bar AS foo$boink
+            //
+            // 4b-2)   like 4a-2) above, but for subtables
+            //
+            // The $ sign is used in class tx_dataquery_wrapper for building the data structure
+            // Initialize values
+            $mappedField = '';
+            $mappedTable = '';
+            $fullField = $fieldInfo['tableAlias'] . '.' . $fieldInfo['field'];
+            if ($fieldInfo['function']) {
+                $fullField = $fieldInfo['field'];
+            }
+            $theField = $fieldInfo['field'];
+            // Case 4a
+            if ($fieldInfo['tableAlias'] == $this->queryObject->mainTable) {
+                if (empty($fieldInfo['fieldAlias'])) {
+                    $theAlias = $theField;
+                } else {
+                    $fullField .= ' AS ';
+                    if (strpos($fieldInfo['fieldAlias'], '.') === false) {
+                        $theAlias = $fieldInfo['fieldAlias'];
+                        $mappedTable = $fieldInfo['tableAlias'];
+                        $mappedField = $fieldInfo['fieldAlias'];
+                    } // Case 4a-2
+                    else {
+                        list($mappedTable, $mappedField) = explode('.', $fieldInfo['fieldAlias']);
+                        $theAlias = str_replace('.', '$', $fieldInfo['fieldAlias']);
+                    }
+                    $fullField .= $theAlias;
+                }
+            } else {
+                $fullField .= ' AS ';
+                if (empty($fieldInfo['fieldAlias'])) {
+                    $theAlias = $fieldInfo['tableAlias'] . '$' . $fieldInfo['field'];
+                } else {
+                    // Case 4b
+                    if (strpos($fieldInfo['fieldAlias'], '.') === false) {
+                        $theAlias = $fieldInfo['tableAlias'] . '$' . $fieldInfo['fieldAlias'];
+                    } // Case 4b-2
+                    else {
+                        list($mappedTable, $mappedField) = explode('.', $fieldInfo['fieldAlias']);
+                        $theAlias = str_replace('.', '$', $fieldInfo['fieldAlias']);
+                    }
+                }
+                $fullField .= $theAlias;
+            }
+            if (empty($mappedTable)) {
+                $mappedTable = $fieldInfo['tableAlias'];
+                $mappedField = $theField;
+            }
+            $this->fieldTrueNames[$theAlias] = array(
+                    'table' => $fieldInfo['table'],
+                    'aliasTable' => $fieldInfo['tableAlias'],
+                    'field' => $theField,
+                    'mapping' => array('table' => $mappedTable, 'field' => $mappedField)
+            );
+            $this->queryObject->structure['SELECT'][$index] = $fullField;
+        }
+    }
+
+    /**
+     * Checks every table that doesn't have a uid or pid field and tries to add it
+     * to the list of fields to select.
+     *
+     * @return void
+     */
+    protected function addBaseFields()
+    {
+        // Loop on the tables that don't have a uid field
         foreach ($this->queryObject->hasBaseFields as $alias => $listOfFields) {
-                       // Get all fields for the given table
-                       $fieldsInfo = OverlayEngine::getAllFieldsForTable($this->queryObject->aliases[$alias]);
-                       foreach ($listOfFields as $baseField => $flag) {
-                               if (!$flag) {
-                                       // Add the uid field only if it exists
-                                       if (isset($fieldsInfo[$baseField])) {
-                                               $this->addExtraField($baseField, $alias, $this->getTrueTableName($alias));
-                                       }
-                               }
-                       }
+            // Get all fields for the given table
+            $fieldsInfo = OverlayEngine::getAllFieldsForTable($this->queryObject->aliases[$alias]);
+            foreach ($listOfFields as $baseField => $flag) {
+                if (!$flag) {
+                    // Add the uid field only if it exists
+                    if (isset($fieldsInfo[$baseField])) {
+                        $this->addExtraField($baseField, $alias, $this->getTrueTableName($alias));
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * Gets the localized labels for all tables and fields in the query in the given language.
+     *
+     * @param string $language Two-letter ISO code of a language
+     * @return array List of all localized labels
+     */
+    public function getLocalizedLabels($language = '')
+    {
+        $lang = Utilities::getLanguageObject($language);
+
+        // Now that we have a properly initialised language object,
+        // loop on all labels and get any existing localised string
+        $localizedStructure = array();
+        foreach ($this->queryFields as $alias => $tableData) {
+            $table = $tableData['name'];
+            // Initialize structure for table, if not already done
+            if (!isset($localizedStructure[$alias])) {
+                $localizedStructure[$alias] = array('table' => $table, 'fields' => array());
+            }
+            // Get the labels for the tables
+            if (isset($GLOBALS['TCA'][$table]['ctrl']['title'])) {
+                $tableName = $lang->sL($GLOBALS['TCA'][$table]['ctrl']['title']);
+                $localizedStructure[$alias]['name'] = $tableName;
+            } else {
+                $localizedStructure[$alias]['name'] = $table;
+            }
+            // Get the labels for the fields
+            foreach ($tableData['fields'] as $fieldData) {
+                // Set default values
+                $tableAlias = $alias;
+                $field = $fieldData['name'];
+                // Get the localized label, if it exists, otherwise use field name
+                // Skip if it's a function (it will have no TCA definition anyway)
+                $fieldName = $field;
+                if (!$fieldData['function'] && isset($GLOBALS['TCA'][$table]['columns'][$fieldData['name']]['label'])) {
+                    $fieldName = $lang->sL($GLOBALS['TCA'][$table]['columns'][$fieldData['name']]['label']);
+                }
+                // Check if the field has an alias, if yes use it
+                // Otherwise use the field name itself as an alias
+                $fieldAlias = $field;
+                if (isset($this->queryObject->fieldAliases[$alias][$field])) {
+                    $fieldAlias = $this->queryObject->fieldAliases[$alias][$field];
+                    // If the alias contains a dot (.), it means it contains the alias of a table name
+                    // Explode the name on the dot and use the parts as a new table alias and field name
+                    if (strpos($fieldAlias, '.') !== false) {
+                        list($tableAlias, $fieldAlias) = GeneralUtility::trimExplode('.', $fieldAlias);
+                        // Initialize structure for table, if not already done
+                        if (!isset($localizedStructure[$tableAlias])) {
+                            $localizedStructure[$tableAlias] = array('table' => $tableAlias, 'fields' => array());
+                        }
+                    }
+                }
+                // Store the localized label
+                $localizedStructure[$tableAlias]['fields'][$fieldAlias] = $fieldName;
+            }
         }
-       }
-
-       /**
-        * Gets the localized labels for all tables and fields in the query in the given language.
-        *
-        * @param string $language Two-letter ISO code of a language
-        * @return array List of all localized labels
-        */
-       public function getLocalizedLabels($language = '') {
-               $lang = Utilities::getLanguageObject($language);
-
-               // Now that we have a properly initialised language object,
-               // loop on all labels and get any existing localised string
-               $localizedStructure = array();
-               foreach ($this->queryFields as $alias => $tableData) {
-                       $table = $tableData['name'];
-                               // Initialize structure for table, if not already done
-                       if (!isset($localizedStructure[$alias])) {
-                               $localizedStructure[$alias] = array('table' => $table, 'fields' => array());
-                       }
-                       // Get the labels for the tables
-                       if (isset($GLOBALS['TCA'][$table]['ctrl']['title'])) {
-                               $tableName = $lang->sL($GLOBALS['TCA'][$table]['ctrl']['title']);
-                               $localizedStructure[$alias]['name'] = $tableName;
-                       }
-                       else {
-                               $localizedStructure[$alias]['name'] = $table;
-                       }
-                       // Get the labels for the fields
-                       foreach ($tableData['fields'] as $fieldData) {
-                               // Set default values
-                               $tableAlias = $alias;
-                               $field = $fieldData['name'];
-                               // Get the localized label, if it exists, otherwise use field name
-                               // Skip if it's a function (it will have no TCA definition anyway)
-                               $fieldName = $field;
-                               if (!$fieldData['function'] && isset($GLOBALS['TCA'][$table]['columns'][$fieldData['name']]['label'])) {
-                                       $fieldName = $lang->sL($GLOBALS['TCA'][$table]['columns'][$fieldData['name']]['label']);
-                               }
-                               // Check if the field has an alias, if yes use it
-                               // Otherwise use the field name itself as an alias
-                               $fieldAlias = $field;
-                               if (isset($this->queryObject->fieldAliases[$alias][$field])) {
-                                       $fieldAlias = $this->queryObject->fieldAliases[$alias][$field];
-                                       // If the alias contains a dot (.), it means it contains the alias of a table name
-                                       // Explode the name on the dot and use the parts as a new table alias and field name
-                                       if (strpos($fieldAlias, '.') !== false) {
-                                               list($tableAlias, $fieldAlias) = GeneralUtility::trimExplode('.', $fieldAlias);
-                                               // Initialize structure for table, if not already done
-                                               if (!isset($localizedStructure[$tableAlias])) $localizedStructure[$tableAlias] = array('table' => $tableAlias, 'fields' => array());
-                                       }
-                               }
-                               // Store the localized label
-                               $localizedStructure[$tableAlias]['fields'][$fieldAlias] = $fieldName;
+        //             \TYPO3\CMS\Core\Utility\DebugUtility::debug($localizedStructure, 'Localized structure');
+        return $localizedStructure;
+    }
+
+    /**
+     * Sets the data coming from the Data Provider class.
+     *
+     * @param array $providerData Database record corresponding to the current Data Query record
+     * @return void
+     */
+    public function setProviderData($providerData)
+    {
+        $this->providerData = $providerData;
+        // Perform some processing on some fields
+        // Mostly this is about turning into arrays the fields containing comma-separated values
+        $this->providerData['ignore_time_for_tables_exploded'] = GeneralUtility::trimExplode(',',
+                $this->providerData['ignore_time_for_tables']);
+        $this->providerData['ignore_disabled_for_tables_exploded'] = GeneralUtility::trimExplode(',',
+                $this->providerData['ignore_disabled_for_tables']);
+        $this->providerData['ignore_fegroup_for_tables_exploded'] = GeneralUtility::trimExplode(',',
+                $this->providerData['ignore_fegroup_for_tables']);
+        $this->providerData['get_versions_directly_exploded'] = GeneralUtility::trimExplode(',',
+                $this->providerData['get_versions_directly']);
+    }
+
+    /**
+     * Returns an associative array containing information for method enableFields.
+     *
+     * enableFields() will skip each enable field condition from the returned array.
+     *
+     * @param string $tableName The name of the table
+     * @return array The array containing the keys to be ignored
+     */
+    protected function getIgnoreArray($tableName)
+    {
+        $ignoreArray = array();
+        // Handle case when some fields should be partially excluded from enableFields()
+        if ($this->providerData['ignore_enable_fields'] == '2') {
+
+            // starttime / endtime field
+            if (in_array($tableName, $this->providerData['ignore_time_for_tables_exploded']) ||
+                    $this->providerData['ignore_time_for_tables'] === '*'
+            ) {
+                $ignoreArray['starttime'] = true;
+                $ignoreArray['endtime'] = true;
+            }
+
+            // disabled field
+            if (in_array($tableName, $this->providerData['ignore_disabled_for_tables_exploded']) ||
+                    $this->providerData['ignore_disabled_for_tables'] === '*'
+            ) {
+                $ignoreArray['disabled'] = true;
+            }
+
+            // fe_group field
+            if (in_array($tableName, $this->providerData['ignore_fegroup_for_tables_exploded']) ||
+                    $this->providerData['ignore_fegroup_for_tables'] === '*'
+            ) {
+                $ignoreArray['fe_group'] = true;
             }
         }
-//             \TYPO3\CMS\Core\Utility\DebugUtility::debug($localizedStructure, 'Localized structure');
-               return $localizedStructure;
+        return $ignoreArray;
     }
 
-       /**
-        * Sets the data coming from the Data Provider class.
-        *
-        * @param array $providerData Database record corresponding to the current Data Query record
-        * @return void
-        */
-       public function setProviderData($providerData) {
-               $this->providerData = $providerData;
-               // Perform some processing on some fields
-               // Mostly this is about turning into arrays the fields containing comma-separated values
-               $this->providerData['ignore_time_for_tables_exploded'] = GeneralUtility::trimExplode(',', $this->providerData['ignore_time_for_tables']);
-               $this->providerData['ignore_disabled_for_tables_exploded'] = GeneralUtility::trimExplode(',', $this->providerData['ignore_disabled_for_tables']);
-               $this->providerData['ignore_fegroup_for_tables_exploded'] = GeneralUtility::trimExplode(',', $this->providerData['ignore_fegroup_for_tables']);
-               $this->providerData['get_versions_directly_exploded'] = GeneralUtility::trimExplode(',', $this->providerData['get_versions_directly']);
-       }
-
-       /**
-        * Returns an associative array containing information for method enableFields.
-        *
-        * enableFields() will skip each enable field condition from the returned array.
-        *
-        * @param string $tableName The name of the table
-        * @return array The array containing the keys to be ignored
-        */
-       protected function getIgnoreArray($tableName) {
-               $ignoreArray = array();
-               // Handle case when some fields should be partially excluded from enableFields()
-               if ($this->providerData['ignore_enable_fields'] == '2') {
-
-                       // starttime / endtime field
-                       if (in_array($tableName, $this->providerData['ignore_time_for_tables_exploded']) ||
-                                       $this->providerData['ignore_time_for_tables'] == '*') {
-                               $ignoreArray['starttime'] = TRUE;
-                               $ignoreArray['endtime'] = TRUE;
-                       }
-
-                       // disabled field
-                       if (in_array($tableName, $this->providerData['ignore_disabled_for_tables_exploded']) ||
-                                       $this->providerData['ignore_disabled_for_tables'] == '*') {
-                               $ignoreArray['disabled'] = TRUE;
-                       }
-
-                       // fe_group field
-                       if (in_array($tableName, $this->providerData['ignore_fegroup_for_tables_exploded']) ||
-                                       $this->providerData['ignore_fegroup_for_tables'] == '*') {
-                               $ignoreArray['fe_group'] = TRUE;
-                       }
-               }
-               return $ignoreArray;
-       }
-
-       /**
-        * Adds where clause elements related to typical TYPO3 control parameters.
-        *
-        * The parameters are:
-        *
-        *      - the enable fields
-        *      - the language handling
-        *      - the versioning system
-        *
-        * @return      void
-        */
-       public function addTypo3Mechanisms() {
-               // Add enable fields conditions
-               $this->addEnableFieldsCondition();
-               // Assemble a list of all currently selected fields for each table,
-               // skipping function calls (which can't be overlayed anyway)
-               // This is used by the next two methods, which may add some necessary fields,
-               // if not present already
-               $fieldsPerTable = array();
-               foreach ($this->queryFields as $alias => $tableData) {
-                       $fieldsPerTable[$alias] = array();
-                       foreach ($tableData['fields'] as $fieldData) {
-                               if (!$fieldData['function']) {
-                                       $fieldsPerTable[$alias][] = $fieldData['name'];
-                               }
-                       }
-               }
-               // Add language-related conditions
-               $this->addLanguageCondition($fieldsPerTable);
-               // Add versioning-related conditions
-               $this->addVersioningCondition($fieldsPerTable);
-       }
-
-       /**
-        * This method adds all SQL conditions needed to enforce the enable fields for
-        * all tables involved
-        *
-        * @return void
-        */
-       protected function addEnableFieldsCondition() {
-               // First check if enable fields must really be added or should be ignored
-               if ($this->providerData['ignore_enable_fields'] == '0' || $this->providerData['ignore_enable_fields'] == '2') {
-
-                       // Start with main table
-                       // Define parameters for enable fields condition
-                       $trueTableName = $this->queryObject->aliases[$this->queryObject->mainTable];
-                       $showHidden = ($trueTableName == 'pages') ? $GLOBALS['TSFE']->showHiddenPage : $GLOBALS['TSFE']->showHiddenRecords;
-                       $ignoreArray = $this->getIgnoreArray($this->queryObject->mainTable);
-
-                       $enableClause = OverlayEngine::getEnableFieldsCondition(
-                               $trueTableName,
-                               $showHidden,
-                               $ignoreArray
-                       );
-                       // Replace the true table name by its alias if necessary
-                       // NOTE: there's a risk that a field containing the table name might be modified abusively
-                       // There's no real way around it except changing tx_overlays::getEnableFieldsCondition()
-                       // to re-implement a better t3lib_page::enableFields()
-                       // Adding the "." in the replacement reduces the risks
-                       if ($this->queryObject->mainTable != $trueTableName) {
-                               $enableClause = str_replace($trueTableName . '.', $this->queryObject->mainTable . '.', $enableClause);
-                       }
-                       $this->addWhereClause($enableClause);
-
-                       // Add enable fields to JOINed tables
-                       if (isset($this->queryObject->structure['JOIN']) && is_array($this->queryObject->structure['JOIN'])) {
-                               foreach ($this->queryObject->structure['JOIN'] as $joinData) {
-
-                                       // Define parameters for enable fields condition
-                                       $table = $joinData['table'];
-                                       $showHidden = ($table == 'pages') ? $GLOBALS['TSFE']->showHiddenPage : $GLOBALS['TSFE']->showHiddenRecords;
-                                       $ignoreArray = $this->getIgnoreArray($joinData['alias']);
-
-                                       $enableClause = OverlayEngine::getEnableFieldsCondition(
-                                               $table,
-                                               $showHidden,
-                                               $ignoreArray
-                                       );
-                                       if (!empty($enableClause)) {
-                                               if ($table != $joinData['alias']) {
-                                                       $enableClause = str_replace($table . '.', $joinData['alias'] . '.', $enableClause);
-                                               }
-                                               $this->addOnClause($enableClause, $joinData['alias']);
-                                       }
-                               }
-                       }
-               }
-       }
-
-       /**
-        * Adds SQL conditions related to language handling.
-        *
-        * Also adds the necessary fields to the list of SELECTed fields
-        *
-        * @param array $fieldsPerTable List of all fields already SELECTed, per table
-        *
-        * @return void
-        */
-       protected function addLanguageCondition($fieldsPerTable) {
-               $skippedTablesForLanguageOverlays = GeneralUtility::trimExplode(',', $this->providerData['skip_overlays_for_tables'], TRUE);
-
-               // Add the language condition, if necessary
-               if (empty($this->providerData['ignore_language_handling']) && !$this->queryObject->structure['DISTINCT']) {
-
-                       // Add the DB fields and the SQL conditions necessary for having everything ready to handle overlays
-                       // as per the standard TYPO3 mechanism
-                       // Loop on all tables involved
-                       foreach ($this->queryFields as $alias => $tableData) {
-                               $table = $tableData['name'];
-
-                               // First entirely skip tables which are defined in the skip list
-                               if (in_array($table, $skippedTablesForLanguageOverlays)) {
-                                       $this->doOverlays[$table] = FALSE;
-
-                               // Check which handling applies, based on existing TCA structure
-                               // The table must at least have a language field or point to a foreign table for translation
-                               } elseif (isset($GLOBALS['TCA'][$table]['ctrl']['languageField']) || isset($GLOBALS['TCA'][$table]['ctrl']['transForeignTable'])) {
-
-                                       // The table uses translations in the same table (transOrigPointerField) or in a foreign table (transForeignTable)
-                                       // Prepare for overlays
-                                       if (isset($GLOBALS['TCA'][$table]['ctrl']['transOrigPointerField']) || isset($GLOBALS['TCA'][$table]['ctrl']['transForeignTable'])) {
-                                               // For each table, make sure that the fields necessary for handling the language overlay are included in the list of selected fields
-                                               try {
-                                                       $fieldsForOverlayArray = OverlayEngine::selectOverlayFieldsArray($table, implode(',', $fieldsPerTable[$alias]));
-                                                       // Extract which fields were added and add them to the list of fields to select
-                                                       $addedFields = array_diff($fieldsForOverlayArray, $fieldsPerTable[$alias]);
-                                                       if (count($addedFields) > 0) {
-                                                               foreach ($addedFields as $aField) {
-                                                                       $this->addExtraField($aField, $alias, $table);
-                                                               }
-                                                       }
-                                                       $this->doOverlays[$table] = TRUE;
-                                                       // Add the language condition for the given table (only for tables containing their own translations)
-                                                       if (isset($GLOBALS['TCA'][$table]['ctrl']['transOrigPointerField'])) {
-                                                               $languageCondition = OverlayEngine::getLanguageCondition($table, $alias);
-                                                               if ($alias == $this->queryObject->mainTable) {
-                                                                       $this->addWhereClause($languageCondition);
-                                                               } else {
-                                                                       $this->addOnClause($languageCondition, $alias);
-                                                               }
-                                                       }
-                                               }
-                                               catch (\Exception $e) {
-                                                       $this->doOverlays[$table] = FALSE;
-                                               }
-                                       }
-
-                               // The table simply contains a language flag.
-                               // This is just about adding the proper condition on the language field and nothing more
-                               // No overlays will be handled at a later time
-                               } else {
-                                       if (isset($GLOBALS['TCA'][$table]['ctrl']['languageField'])) {
-                                               // Take language that corresponds to current language or [All]
-                                               $languageCondition = $alias . '.' . $GLOBALS['TCA'][$table]['ctrl']['languageField'] . ' IN (' . $GLOBALS['TSFE']->sys_language_content . ', -1)';
-                                               if ($alias == $this->queryObject->mainTable) {
-                                                       $this->addWhereClause($languageCondition);
-                                               } else {
-                                                       $this->addOnClause($languageCondition, $alias);
-                                               }
-                                       }
-                               }
-                       }
-               }
-//\TYPO3\CMS\Core\Utility\DebugUtility::debug($this->doOverlays);
-       }
-
-       /**
-        * Adds SQL conditions related version handling.
-        *
-        * Also add the necessary fields to the list of SELECTed fields.
-        * Contrary to the other conditions, versioning conditions are always added,
-        * if only to make sure that only LIVE records are selected.
-        *
-        * @param array $fieldsPerTable List of all fields already SELECTed, per table
-        *
-        * @return void
-        */
-       protected function addVersioningCondition($fieldsPerTable) {
-               foreach ($this->queryFields as $alias => $tableData) {
-                       $table = $tableData['name'];
-                       $this->doVersioning[$table] = FALSE;
-
-                       // Continue if table indeed supports versioning
-                       if (!empty($GLOBALS['TCA'][$table]['ctrl']['versioningWS'])) {
-                               // By default make sure to take only LIVE version
-                               $workspaceCondition = $alias . ".t3ver_oid = '0'";
-                               // If in preview mode, assemble condition according to current workspace
-                               if ($GLOBALS['TSFE']->sys_page->versioningPreview) {
-                                       // For each table, make sure that the fields necessary for handling the language overlay are included in the list of selected fields
-                                       try {
-                                               $fieldsForOverlayArray = OverlayEngine::selectVersioningFieldsArray(
-                                                       $table,
-                                                       implode(',', $fieldsPerTable[$alias])
-                                               );
-                                               // Extract which fields were added and add them to the list of fields to select
-                                               $addedFields = array_diff($fieldsForOverlayArray, $fieldsPerTable[$alias]);
-                                               if (count($addedFields) > 0) {
-                                                       foreach ($addedFields as $aField) {
-                                                               $this->addExtraField($aField, $alias, $table);
-                                                       }
-                                               }
-                                               $this->doVersioning[$table] = TRUE;
-                                               $getVersionsDirectly = FALSE;
-                                               if ($this->providerData['get_versions_directly'] == '*' || in_array($alias, $this->providerData['get_versions_directly_exploded'])) {
-                                                       $getVersionsDirectly = TRUE;
-                                               }
-                                               $workspaceCondition = OverlayEngine::getVersioningCondition(
-                                                       $table,
-                                                       $alias,
-                                                       $getVersionsDirectly
-                                               );
-                                       }
-                                       catch (\Exception $e) {
-                                               $this->doVersioning[$table] = FALSE;
-                                               $this->parentObject->getController()->addMessage(
-                                                       self::$extKey,
-                                                       'A problem happened with versioning: ' . $e->getMessage() . ' (' . $e->getCode() . ')',
-                                                       'Falling back to LIVE records for table ' . $table,
-                                                       FlashMessage::WARNING
-                                               );
-                                       }
-                               }
-                               if ($alias == $this->queryObject->mainTable) {
-                                       $this->addWhereClause($workspaceCondition);
-                               } else {
-                                       $this->addOnClause($workspaceCondition, $alias);
-                               }
-                       }
-               }
-//\TYPO3\CMS\Core\Utility\DebugUtility::debug($this->doVersioning);
-       }
-
-       /**
-        * Takes a Data Filter structure and processes its instructions.
-        *
-        * @param array $filter Data Filter structure
-        * @return void
-        */
-       public function addFilter($filter) {
-               // First handle the "filter" part, which will be turned into part of a SQL WHERE clause
-               $completeFilters = array();
-               $logicalOperator = (empty($filter['logicalOperator'])) ? 'AND' : $filter['logicalOperator'];
-               if (isset($filter['filters']) && is_array($filter['filters'])) {
-                       foreach ($filter['filters'] as $index => $filterData) {
-                               $table = '';
-                               // Check if the condition must be explicitly ignored
-                               // (i.e. it is transmitted by the filter only for information)
-                               // If not, resolve the table name, if possible
-                               if ($filterData['void']) {
-                                       $ignoreCondition = TRUE;
-                               } else {
-                                       $ignoreCondition = FALSE;
-                                       $table = (empty($filterData['table'])) ? $this->queryObject->mainTable : $filterData['table'];
-                                       // Check if the table is available in the query
-                                       try {
-                                               $table = $this->matchAliasOrTableName($table, 'Filter - ' . ((empty($filterData['string'])) ? $index : $filterData['string']));
-                                       }
-                                       catch (InvalidQueryException $e) {
-                                               $ignoreCondition = TRUE;
-                                               $this->parentObject->getController()->addMessage(
-                                                       self::$extKey,
-                                                       'The condition did not apply to a table used in the query.',
-                                                       'Condition ignored',
-                                                       FlashMessage::NOTICE,
-                                                       $filterData
-                                               );
-                                       }
-                               }
-                               // If the table is not in the query, ignore the condition
-                               if (!$ignoreCondition) {
-                                       $field = $filterData['field'];
-                                       $fullField = $table . '.' . $field;
-                                       // If the field is an alias, override full field definition
-                                       // to whatever the alias is mapped to
-                                       if (isset($this->queryObject->fieldAliasMappings[$field])) {
-                                               $fullField = $this->queryObject->fieldAliasMappings[$field];
-                                       }
-                                       $condition = '';
-                                       // Define table on which to apply the condition
-                                       // Conditions will normally be applied in the WHERE clause
-                                       // if the table is the main one, otherwise it is applied
-                                       // in the ON clause of the relevant JOIN statement
-                                       // However the application of the condition may be forced to be in the WHERE clause,
-                                       // no matter which table it targets
-                                       $tableForApplication = $table;
-                                       if ($filterData['main']) {
-                                               $tableForApplication = $this->queryObject->mainTable;
-                                       }
-                                       foreach ($filterData['conditions'] as $conditionData) {
-                                               // If the value is special value "\all", all values must be taken,
-                                               // so the condition is simply ignored
-                                               if ($conditionData['value'] != '\all') {
-                                                       try {
-                                                               $parsedCondition = SqlUtility::conditionToSql($fullField, $table, $conditionData);
-                                                               if (!empty($condition)) {
-                                                                       $condition .= ' AND ';
-                                                               }
-                                                               $condition .= '(' . $parsedCondition . ')';
-                                                               // If the operator was a full text search, store resulting condition which will be used
-                                                               // later to replace the placeholder in the SELECT part of the statement
-                                                               if ($conditionData['operator'] == 'fulltext' || $conditionData['operator'] == 'fulltext_natural') {
-                                                                       $fullFieldParts = explode('.', $fullField);
-                                                                       $placeholderKey = $table . '.fulltext.' . $fullFieldParts[2];
-                                                                       if (isset($this->queryObject->fulltextSearchPlaceholders[$placeholderKey])) {
-                                                                               $this->queryObject->fulltextSearchPlaceholders[$placeholderKey] = $condition;
-                                                                       }
-                                                               }
-                                                       }
-                                                       catch (InvalidQueryException $e) {
-                                                               $this->parentObject->getController()->addMessage(
-                                                                       self::$extKey,
-                                                                       $e->getMessage(),
-                                                                       'Condition ignored',
-                                                                       FlashMessage::WARNING,
-                                                                       $filterData
-                                                               );
-                                                       }
-                                               }
-                                       }
-                                       // Add the condition only if it wasn't empty
-                                       if (!empty($condition)) {
-                                               if (empty($completeFilters[$tableForApplication])) {
-                                                       $completeFilters[$tableForApplication] = '';
-                                               } else {
-                                                       $completeFilters[$tableForApplication] .= ' ' . $logicalOperator . ' ';
-                                               }
-                                               $completeFilters[$tableForApplication] .= '(' . $condition . ')';
-                                       }
-                               }
-                       }
-                       foreach ($completeFilters as $table => $whereClause) {
-                               if ($table == $this->queryObject->mainTable) {
-                                       $this->addWhereClause($whereClause);
-                               } elseif (in_array($table, $this->queryObject->subtables)) {
-                                       $this->addOnClause($whereClause, $table);
-                               }
-                       }
-                       // Free some memory
-                       unset($completeFilters);
-               }
-               // Add the eventual raw SQL in the filter
-               // Raw SQL is always added to the main where clause
-               if (!empty($filter['rawSQL'])) {
-                       $this->addWhereClause($filter['rawSQL']);
-               }
-               // Handle the order by clauses
-               if (count($filter['orderby']) > 0) {
-                       foreach ($filter['orderby'] as $orderData) {
-                               // Special case if ordering is random
-                               if ($orderData['order'] == 'RAND') {
-                                       $this->queryObject->structure['ORDER BY'][] = 'RAND()';
-
-                               // Handle normal configuration
-                               } else {
-                                       $table = ((empty($orderData['table'])) ? $this->queryObject->mainTable : $orderData['table']);
-                                       // Try applying the order clause to an existing table
-                                       try {
-                                               $table = $this->matchAliasOrTableName($table, 'Order clause - ' . $table . ' - ' . $orderData['field'] . ' - ' . $orderData['order']);
-                                               $completeField = $table . '.' . $orderData['field'];
-                                               $orderbyClause = $completeField . ' ' . $orderData['order'];
-                                               $this->queryObject->structure['ORDER BY'][] = $orderbyClause;
-                                               $this->queryObject->orderFields[] = array(
-                                                       'field' => $completeField,
-                                                       'order' => $orderData['order'],
-                                                       'engine' => isset($orderData['engine']) ? $orderData['engine'] : ''
-                                               );
-                                       }
-                                       // Table was not matched
-                                       catch (InvalidQueryException $e) {
-                                               $this->parentObject->getController()->addMessage(
-                                                       self::$extKey,
-                                                       'The ordering clause did not apply to a table used in the query.',
-                                                       'Ordering ignored',
-                                                       FlashMessage::NOTICE,
-                                                       $orderData
-                                               );
-                                       }
-                               }
-                       }
-               }
-       }
-
-       /**
-        * Takes a list of uid's prepended by their table name,
-        * as returned in the "uidListWithTable" property of a idList-type SDS,
-        * and makes it into appropriate SQL IN conditions for every table that matches those used in the query.
-        *
-        * @param string $idList Comma-separated list of uid's prepended by their table name
-        * @return void
-        */
-       public function addIdList($idList) {
-               if (!empty($idList)) {
-                       $idArray = GeneralUtility::trimExplode(',', $idList);
-                       $idlistsPerTable = array();
-                       // First assemble a list of all uid's for each table
-                       foreach ($idArray as $item) {
-                               // Code inspired from \TYPO3\CMS\Core\Database\RelationHandler
-                               // String is reversed before exploding, to get uid first
-                               list($uid, $table) = explode('_', strrev($item), 2);
-                               // Exploded parts are reversed back
-                               $uid = strrev($uid);
-                               // If table is not defined, assume it's the main table
-                               if (empty($table)) {
-                                       $table = $this->queryObject->mainTable;
-                                       if (!isset($idlistsPerTable[$table])) {
-                                               $idlistsPerTable[$table] = array();
-                                       }
-                                       $idlistsPerTable[$table][] = $uid;
-                               } else {
-                                       $table = strrev($table);
-                                       // Make sure the table name matches one used in the query
-                                       try {
-                                               $table = $this->matchAliasOrTableName($table, 'Id list - ' . $item);
-                                               if (!isset($idlistsPerTable[$table])) {
-                                                       $idlistsPerTable[$table] = array();
-                                               }
-                                               $idlistsPerTable[$table][] = $uid;
-                                       }
-                                       catch (InvalidQueryException $e) {
-                                               $this->parentObject->getController()->addMessage(
-                                                       self::$extKey,
-                                                       'An item from the id list did not apply, because table ' . $table . ' is not used in the query.',
-                                                       'Id ignored',
-                                                       FlashMessage::NOTICE,
-                                                       $item
-                                               );
-                                       }
-                               }
-                       }
-                       // Loop on all tables and add test on list of uid's, if table is indeed in query
-                       foreach ($idlistsPerTable as $table => $uidArray) {
-                               $condition = $table . '.uid IN (' . implode(',', $uidArray) . ')';
-                               if ($table == $this->queryObject->mainTable) {
-                                       $this->addWhereClause($condition);
-                               } elseif (in_array($table, $this->queryObject->subtables)) {
-                                       if (!empty($this->queryObject->structure['JOIN'][$table]['on'])) {
-                                               $this->queryObject->structure['JOIN'][$table]['on'] .= ' AND ';
-                                       }
-                                       $this->queryObject->structure['JOIN'][$table]['on'] .= $condition;
-                               }
-                       }
-                       // Free some memory
-                       unset($idlistsPerTable);
-               }
-       }
-
-       /**
-        * Builds up the query with all the data stored in the structure.
-        *
-        * @return string The assembled SQL query
-        */
-       public function buildQuery() {
-               // First check what to do with ORDER BY fields
-               $this->preprocessOrderByFields();
-               // Start assembling the query
-               $query  = 'SELECT ';
-               if ($this->queryObject->structure['DISTINCT']) {
-                       $query .= 'DISTINCT ';
-               }
-               $query .= implode(', ', $this->queryObject->structure['SELECT']) . ' ';
-
-               // Process fulltext replacements, if any
-               foreach ($this->queryObject->fulltextSearchPlaceholders as $placeholder => $replacement) {
-                       $query = str_replace($placeholder, $replacement, $query);
-               }
-
-               $query .= 'FROM ' . $this->queryObject->structure['FROM']['table'];
-               if (!empty($this->queryObject->structure['FROM']['alias'])) {
-                       $query .= ' AS ' . $this->queryObject->structure['FROM']['alias'];
-               }
-               $query .= ' ';
-               if (isset($this->queryObject->structure['JOIN'])) {
-                       foreach ($this->queryObject->structure['JOIN'] as $theJoin) {
-                               $query .= strtoupper($theJoin['type']) . ' JOIN ' . $theJoin['table'];
-                               if (!empty($theJoin['alias'])) {
-                                       $query .= ' AS ' . $theJoin['alias'];
-                               }
-                               if (!empty($theJoin['on'])) {
-                                       $query .= ' ON ' . $theJoin['on'];
-                               }
-                               $query .= ' ';
-                       }
-               }
-               if (count($this->queryObject->structure['WHERE']) > 0) {
-                       $whereClause = '';
-                       foreach ($this->queryObject->structure['WHERE'] as $clause) {
-                               if (!empty($whereClause)) {
-                                       $whereClause .= ' AND ';
-                               }
-                               $whereClause .= '(' . $clause . ')';
-                       }
-                       $query .= 'WHERE ' . $whereClause . ' ';
-               }
-               if (count($this->queryObject->structure['GROUP BY']) > 0) {
-                       $query .= 'GROUP BY ' . implode(', ', $this->queryObject->structure['GROUP BY']) . ' ';
-               }
-                       // Add order by clause if defined and if applicable (see preprocessOrderByFields())
-               if ($this->processOrderBy && count($this->queryObject->structure['ORDER BY']) > 0) {
-                       $query .= 'ORDER BY ' . implode(', ', $this->queryObject->structure['ORDER BY']) . ' ';
-               }
-               if (isset($this->queryObject->structure['LIMIT'])) {
-                       $query .= 'LIMIT ' . $this->queryObject->structure['LIMIT'];
-                       if (isset($this->queryObject->structure['OFFSET'])) {
-                               $query .= ' OFFSET ' . $this->queryObject->structure['OFFSET'];
-                       }
-               }
-
-//\TYPO3\CMS\Core\Utility\DebugUtility::debug($query);
-               return $query;
-       }
-
-       /**
-        * Performs some operations on the fields used for ordering the query, if any.
-        *
-        * If the language is not the default one, order may not be desirable in SQL
-        * As translations are handled using overlays in TYPO3, it is not possible
-        * to sort the records alphabetically in the SQL statement, because the SQL
-        * statement gets only the records in original language.
-        *
-        * @return boolean True if order by must be processed by the SQL query, false otherwise
-        */
-       protected function preprocessOrderByFields() {
-/*
-\TYPO3\CMS\Core\Utility\DebugUtility::debug($this->queryObject->orderFields, 'Order fields');
-\TYPO3\CMS\Core\Utility\DebugUtility::debug($this->queryObject->fieldAliases, 'Field aliases');
-\TYPO3\CMS\Core\Utility\DebugUtility::debug($this->fieldTrueNames, 'Field true names');
-\TYPO3\CMS\Core\Utility\DebugUtility::debug($this->queryFields, 'Query fields');
-\TYPO3\CMS\Core\Utility\DebugUtility::debug($this->queryObject->structure['SELECT'], 'Select structure');
-*/
-               if (count($this->queryObject->orderFields) > 0) {
-                       // If in the FE context and not the default language, start checking for possible use of SQL or not
-                       if (isset($GLOBALS['TSFE']) && $GLOBALS['TSFE']->sys_language_content > 0) {
-                               // Initialise sorting mode flag
-                               $cannotUseSQLForSorting = FALSE;
-                               // Initialise various arrays
-                               $newQueryFields = array();
-                               $newSelectFields = array();
-                               $newTrueNames = array();
-                               $countNewFields = 0;
-                               foreach ($this->queryObject->orderFields as $index => $orderInfo) {
-                                       // Define the table and field names
-                                       $fieldParts = explode('.', $orderInfo['field']);
-                                       if (count($fieldParts) == 1) {
-                                               $alias = $this->queryObject->mainTable;
-                                               $field = $fieldParts[0];
-                                       } else {
-                                               $alias = $fieldParts[0];
-                                               $field = $fieldParts[1];
-                                       }
-                                       // Skip all the rest of the logic for some special values
-                                       if ($field !== 'RAND()' && $field !== 'NULL') {
-                                               // If the field has an alias, change the order fields list to use it
-                                               if (isset($this->queryObject->fieldAliases[$alias][$field])) {
-                                                       $this->queryObject->orderFields[$index]['alias'] = $this->queryObject->orderFields[$index]['field'];
-                                                       $this->queryObject->orderFields[$index]['field'] = $this->queryObject->fieldAliases[$alias][$field];
-                                               }
-                                               // Get the field's true table and field name, if defined, in case an alias is used in the ORDER BY statement
-                                               if (isset($this->fieldTrueNames[$field])) {
-                                                       $alias = $this->fieldTrueNames[$field]['aliasTable'];
-                                                       $field = $this->fieldTrueNames[$field]['field'];
-                                               }
-                                               // Get the true table name and initialize new field array, if necessary
-                                               $table = $this->getTrueTableName($alias);
-                                               if (!isset($newQueryFields[$alias])) {
-                                                       $newQueryFields[$alias] = array(
-                                                               'name' => $alias,
-                                                               'table' => $table,
-                                                               'fields' => array()
-                                                       );
-                                               }
-                                               // Check if there's some explicit engine information
-                                               if (!empty($orderInfo['engine'])) {
-                                                       // If at least one field must be handled by the provider, set the flag to true
-                                                       if ($orderInfo['engine'] == 'provider') {
-                                                               $cannotUseSQLForSorting |= TRUE;
-                                                       } else {
-                                                       // Nothing to do here. If the field was forced to be applied to the source,
-                                                       // it does not need to be checked further
-                                                       }
-                                               } else {
-
-                                                       // Check the type of the field in the TCA
-                                                       // If the field is of some text type and that the table uses overlays,
-                                                       // ordering cannot happen in SQL.
-                                                       if (isset($GLOBALS['TCA'][$table])) {
-                                                               // Check if table uses overlays
-                                                               $usesOverlay = isset($GLOBALS['TCA'][$table]['ctrl']['languageField']) || isset($GLOBALS['TCA'][$table]['ctrl']['transForeignTable']);
-                                                               // Check the field type (load full TCA first)
-                                                               $isTextField = $this->isATextField($table, $field);
-                                                               $cannotUseSQLForSorting |= ($usesOverlay && $isTextField);
-                                                       }
-                                               }
-                                               // Check if the field is already part of the SELECTed fields (under its true name or an alias)
-                                               // If not, get ready to add it by defining all necessary info in temporary arrays
-                                               // (it will be added only if necessary, i.e. if at least one field needs to be ordered later)
-                                               if (!$this->isAQueryField($alias, $field) && !isset($this->queryObject->fieldAliases[$alias][$field])) {
-                                                       $fieldAlias = $alias . '$' . $field;
-                                                       $newQueryFields[$alias]['fields'][] = array('name' => $field, 'function' => FALSE);
-                                                       $newSelectFields[] = $alias . '.' . $field . ' AS ' . $fieldAlias;
-                                                       $newTrueNames[$fieldAlias] = array(
-                                                               'table' => $table,
-                                                               'aliasTable' => $alias,
-                                                               'field' => $field,
-                                                               'mapping' => array('table' => $alias, 'field' => $field)
-                                                       );
-                                                       $countNewFields++;
-                                               }
-                                       }
-                               }
-                               // If sorting cannot be left simply to SQL, prepare to return false
-                               // and add the necessary fields to the SELECT statement
-                               if ($cannotUseSQLForSorting) {
-                                       if ($countNewFields > 0) {
-                                               ArrayUtility::mergeRecursiveWithOverrule($this->queryFields, $newQueryFields);
-                                               $this->queryObject->structure['SELECT'] = array_merge($this->queryObject->structure['SELECT'], $newSelectFields);
-                                               ArrayUtility::mergeRecursiveWithOverrule($this->fieldTrueNames, $newTrueNames);
-/*
-\TYPO3\CMS\Core\Utility\DebugUtility::debug($newQueryFields, 'New query fields');
-\TYPO3\CMS\Core\Utility\DebugUtility::debug($this->queryFields, 'Updated query fields');
-\TYPO3\CMS\Core\Utility\DebugUtility::debug($newTrueNames, 'New field true names');
-\TYPO3\CMS\Core\Utility\DebugUtility::debug($this->fieldTrueNames, 'Updated field true names');
-\TYPO3\CMS\Core\Utility\DebugUtility::debug($newSelectFields, 'New select fields');
-\TYPO3\CMS\Core\Utility\DebugUtility::debug($this->queryObject->structure['SELECT'], 'Updated select structure');
- *
- */
-                                               // Free some memory
-                                               unset($newQueryFields);
-                                               unset($newSelectFields);
-                                               unset($newTrueNames);
-                                       }
-                                       $this->processOrderBy = FALSE;
-                               } else {
-                                       $this->processOrderBy = TRUE;
-                               }
-                       } else {
-                               $this->processOrderBy = TRUE;
-                       }
-               } else {
-                       $this->processOrderBy = TRUE;
-               }
-       }
-
-       /**
-        * Tries to figure out if a given field of a given table is a text field, based on its TCA definition
-        *
-        * @param string $table Name of the table
-        * @param string $field Name of the field
-        * @return bool TRUE is the field can be considered to be text
-        */
-       public function isATextField($table, $field) {
-               $isTextField = TRUE;
-               // We can guess only if there's a TCA definition
-               if (isset($GLOBALS['TCA'][$table]['columns'][$field])) {
-                       $fieldConfig = $GLOBALS['TCA'][$table]['columns'][$field]['config'];
-                       // It's text, easy :-)
-                       if ($fieldConfig['type'] == 'text') {
-                               $isTextField = TRUE;
-
-                       // It's input, further check the "eval" property
-                       } elseif ($fieldConfig['type'] == 'input') {
-                               // If the field has no eval property, assume it's just text
-                               if (empty($fieldConfig['eval'])) {
-                                       $isTextField = TRUE;
-                               } else {
-                                       $evaluations = explode(',', $fieldConfig['eval']);
-                                       // Check if some eval types are common to both array. If yes, it's not a text field.
-                                       $foundTypes = array_intersect($evaluations, self::$notTextTypes);
-                                       $isTextField = (count($foundTypes) > 0) ? FALSE : TRUE;
-                               }
-
-                       // It's another type, it's definitely not text
-                       } else {
-                               $isTextField = FALSE;
-                       }
-               }
-               return $isTextField;
-       }
-
-       /**
-        * Internal utility method that checks whether a given field
-        * can be found in the fields reference list (i.e. $this->queryFields) for
-        * a given table.
-        *
-        * @param string $table Name of the table inside which to look up
-        * @param string $field Name of the field to search for
-        * @return boolean True if the field was found, false otherwise
-        */
-       protected function isAQueryField($table, $field) {
-               $isAQueryField = FALSE;
-               if (isset($this->queryFields[$table]['fields'])) {
-                       foreach ($this->queryFields[$table]['fields'] as $fieldData) {
-                               if ($fieldData['name'] == $field) {
-                                       $isAQueryField = TRUE;
-                                       break;
-                               }
-                       }
-               }
-               return $isAQueryField;
-       }
-
-       /**
-        * This method tries to match a name to the name or alias of a table used in the query
-        * If no alias or straight table name is found, it looks for a true table name instead
-        * If nothing is found, an exception is thrown
-        *
-        * Explanations: a table name may come from an outside source, a Data Filter or another provider.
-        * In order to apply the condition from that other element to the query,
-        * the table(s) referenced in that other element must match tables used in the query.
-        * If the query uses aliases and the other element not, dataquery tries
-        * (using this method) to match the tables from the other element to aliases used
-        * in the query. This may lead to some kind of guess work in which case a warning is logged.
-        *
-        * @param string $name Name to match
-        * @param string $identifier Some key identifying the circumstances in which the call was made (used for logging)
-        * @throws InvalidQueryException
-        * @return string Alias or table name
-        */
-       protected function matchAliasOrTableName($name, $identifier) {
-               $returnedName = $name;
-
-               // If the name was already match, reuse result
-               if (isset($this->tableMatches[$name])) {
-                       $returnedName = $this->tableMatches[$name];
-
-               // If not, perform matching
-               } else {
-                       // If the name matches an existing alias, use it as is
-                       if (isset($this->queryObject->aliases[$name])) {
-                               $this->tableMatches[$name] = $name;
-
-                       // If the name is not in the list of aliases, try to match it
-                       // to a true table name
-                       } else {
-                               // Get the relation of true table names to aliases
-                               // NOTE: true table names are not necessarily unique
-                               $reversedAliasTable = array_flip($this->queryObject->aliases);
-                               if (isset($reversedAliasTable[$name])) {
-                                       $returnedName = $reversedAliasTable[$name];
-                                       $this->tableMatches[$name] = $reversedAliasTable[$name];
-                                               // Write a notice to the message queue
-                                       $message = sprintf('Potentially unreliable match of table %1$s from component %2$s', $name, $identifier);
-                                       $this->parentObject->getController()->addMessage(
-                                               self::$extKey,
-                                               $message,
-                                               'Unreliable alias match',
-                                               FlashMessage::WARNING
-                                       );
-
-                               // No match found, throw exception
-                               } else {
-                                       $message = sprintf('No match found for table %1$s from component %2$s', $name, $identifier);
-                                       throw new InvalidQueryException($message, 1291753564);
-                               }
-                       }
-               }
-               return $returnedName;
-       }
-
-       /**
-        * Adds an extra field to be SELECTed.
-        *
-        * It must be added to the SELECT list, to the list of fields being queried
-        * and to the registry of true names.
-        *
-        * @param string $field Name of the field to add
-        * @param string $tableAlias Alias of the table to add the field to
-        * @param string $table True name of the table to add the field to
-        */
-       protected function addExtraField($field, $tableAlias, $table) {
-               $newFieldName = $tableAlias . '.' . $field;
-               $newFieldAlias = $field;
-               if ($tableAlias != $this->queryObject->mainTable) {
-                       $newFieldAlias = $tableAlias . '$' . $field;
-                       $newFieldName .= ' AS ' . $newFieldAlias;
-               }
-               $this->queryObject->structure['SELECT'][] = $newFieldName;
-               $this->queryFields[$tableAlias]['fields'][] = array('name' => $field, 'function' => FALSE);
-               $this->fieldTrueNames[$newFieldAlias] = array(
-                       'table' => $table,
-                       'aliasTable' => $tableAlias,
-                       'field' => $field,
-                       'mapping' => array('table' => $tableAlias, 'field' => $field)
-               );
-       }
-
-// Setters and getters
-
-       /**
-        * Adds a condition for the WHERE clause.
-        *
-        * @param string $clause SQL WHERE clause (without WHERE)
-        * @return void
-        */
-       public function addWhereClause($clause) {
-               if (!empty($clause)) {
-                       $this->queryObject->structure['WHERE'][] = $clause;
-               }
-       }
-
-       /**
-        * Adds a condition to the ON clause of a given table.
-        *
-        * @param string $clause SQL to add to the ON clause
-        * @param string $alias Alias of the table to the statement to
-        * @return void
-        */
-       public function addOnClause($clause, $alias) {
-               if (!empty($this->queryObject->structure['JOIN'][$alias]['on'])) {
-                       $this->queryObject->structure['JOIN'][$alias]['on'] .= ' AND ';
-               }
-               $this->queryObject->structure['JOIN'][$alias]['on'] .= '(' . $clause . ')';
-       }
-
-       /**
-        * Returns the structure of the parsed query.
-        *
-        * There should be little real-life uses for this, but it is used by the
-        * test case to get the parsed structure.
-        *
-        * @return array The parsed query
-        */
-       public function getQueryStructure() {
-               return $this->queryObject->structure;
-       }
-
-       /**
-        * Returns the name (alias) of the main table of the query,
-        * which is the table name that appears in the FROM clause, or the alias, if any.
-        *
-        * @return string Main table name (alias)
-        */
-       public function getMainTableName() {
-               return $this->queryObject->mainTable;
-       }
-
-       /**
-        * Returns an array containing the list of all subtables in the query,
-        * i.e. the tables that appear in any of the JOIN statements.
-        *
-        * @return array Names of all the joined tables
-        */
-       public function getSubtablesNames() {
-               return $this->queryObject->subtables;
-       }
-
-       /**
-        * Takes an alias and returns the true table name
-        *
-        * @param string $alias Alias of a table
-        * @return string True name of the corresponding table
-        */
-       public function getTrueTableName($alias) {
-               return $this->queryObject->aliases[$alias];
-       }
-
-       /**
-        * Takes the alias and returns it's true name.
-        *
-        * The alias is the full alias as used in the query (e.g. table$field).
-        *
-        * @param string $alias Alias of a field
-        * @return array Array with the true name of the corresponding field
-        *               and the true name of the table it belongs and the alias of that table
-        */
-       public function getTrueFieldName($alias) {
-               $trueNameInformation = $this->fieldTrueNames[$alias];
-                       // Assemble field key (possibly disambiguated with function name)
-               $fieldKey = $trueNameInformation['field'];
-//             if (!empty($trueNameInformation['function'])) {
-//                     $fieldKey .= '_' . $trueNameInformation['function'];
-//             }
-                       // If the field has an explicit alias, we must also pass back that information
-               if (isset($this->queryObject->fieldAliases[$trueNameInformation['aliasTable']][$fieldKey])) {
-                       $alias = $this->queryObject->fieldAliases[$trueNameInformation['aliasTable']][$fieldKey];
-                               // Check if the alias contains a table name
-                               // If yes, strip it, as this information is already handled
-                       if (strpos($alias, '.') !== FALSE) {
-                               list(, $field) = explode('.', $alias);
-                               $alias = $field;
-                       }
-                       $trueNameInformation['mapping']['alias'] = $alias;
-               }
-               return $trueNameInformation;
-       }
-
-       /**
-        * Returns the list of fields defined for ordering the data.
-        *
-        * @return array Fields for ordering (and sort order)
-        */
-       public function getOrderByFields() {
-               return $this->queryObject->orderFields;
-       }
-
-       /**
-        * Returns the query object.
-        *
-        * @return \Tesseract\Dataquery\Utility\QueryObject
-        */
-       public function getSQLObject() {
-               return $this->queryObject;
-       }
-
-       /**
-        * Indicates whether the language overlay mechanism must/can be handled for a given table.
-        *
-        * @param string $table True name of the table to handle
-        * @return boolean True if language overlay must and can be performed, false otherwise
-        * @see \Tesseract\Dataquery\Parser\QueryParser::addTypo3Mechanisms()
-        */
-       public function mustHandleLanguageOverlay($table) {
-               return (isset($this->doOverlays[$table])) ? $this->doOverlays[$table] : FALSE;
-       }
-
-       /**
-        * Indicates whether the language overlay mechanism must/can be handled for a given table.
-        *
-        * @param string $table True name of the table to handle
-        * @return boolean True if language overlay must and can be performed, false otherwise
-        * @see \Tesseract\Dataquery\Parser\QueryParser::addTypo3Mechanisms()
-        */
-       public function mustHandleVersioningOverlay($table) {
-               return (isset($this->doVersioning[$table])) ? $this->doVersioning[$table] : FALSE;
-       }
-
-       /**
-        * Returns whether the ordering of the records was done in the SQL query or not.
-        *
-        * @return boolean True if SQL was used, false otherwise
-        */
-       public function isSqlUsedForOrdering() {
-               return $this->processOrderBy;
-       }
-
-       /**
-        * Returns true if any ordering has been defined at all. False otherwise.
-        *
-        * @return boolean True if there's at least one ordering criterion, false otherwise
-        */
-       public function hasOrdering() {
-               return count($this->queryObject->orderFields) > 0;
-       }
-
-       /**
-        * Returns the name of the first significant table to be INNER JOINed.
-        *
-        * A "significant table" is a table that has a least one field SELECTed
-        * If the first significant table is not INNER JOINed or if there are no JOINs
-        * or no INNER JOINs, an empty string is returned
-        *
-        * @return string Alias of the first significant table, if INNER JOINed, empty string otherwise
-        */
-       public function hasInnerJoinOnFirstSubtable() {
-               $returnValue = '';
-               if (count($this->queryObject->structure['JOIN']) > 0) {
-                       foreach ($this->queryObject->structure['JOIN'] as $alias => $joinInfo) {
-                               if (isset($this->queryFields[$alias])) {
-                                       if ($joinInfo['type'] == 'inner') {
-                                               $returnValue = $alias;
-                                       }
-                                       break;
-                               }
-                       }
-               }
-               return $returnValue;
-       }
-
-       /**
-        * Gets the limit that was defined for a given sub-table
-        * (i.e. a JOINed table). If no limit exists, 0 is returned.
-        *
-        * @param string $table Name of the table to find the limit for
-        * @return integer Value of the limit, or 0 if not defined
-        */
-       public function getSubTableLimit($table) {
-               return isset($this->queryObject->structure['JOIN'][$table]['limit']) ? $this->queryObject->structure['JOIN'][$table]['limit'] : 0;
-       }
-
-       /**
-        * Checks for the existence of a field (possibly with alias) called "uid"
-        * for the query's main table.
-        *
-        * @return bool TRUE if a "uid" field is present, FALSE otherwise
-        */
-       protected function checkUidForDistinctUsage() {
-               $hasUid = FALSE;
-               // There should be either an alias called "uid" or "table$uid"
-               // (where "table" is the name of the main table)
-               $possibleKeys = array('uid', $this->queryObject->mainTable . '$uid');
-               // Also add the possible aliases of the main table
-               // NOTE: this may be wrong when there are more than 1 alias for the main table,
-               // as the uid may actually belong to another table
-               $reversedAliases = array_flip($this->queryObject->aliases);
-               foreach ($reversedAliases as $table => $alias) {
-                       if ($table == $this->queryObject->mainTable) {
-                               $possibleKeys[] = $alias . '$uid';
-                       }
-               }
-               // Loop on all possible keys and exit successfully if one matches a field mapped to "uid"
-               foreach ($possibleKeys as $key) {
-                       if (isset($this->fieldTrueNames[$key]) && $this->fieldTrueNames[$key]['mapping']['field'] == 'uid') {
-                               $hasUid = TRUE;
-                               break;
-                       }
-               }
-               return $hasUid;
-       }
-
-       /**
-        * Returns information about each field that appears more than once in the current query.
-        *
-        * @return array
-        */
-       protected function checkForDuplicateFields() {
-               $duplicates = array();
-               $fieldCountPerTable = array();
-               // Loop on all included fields and make a list of aliases per table and field
-               // Note that these are the fields explicitly entered in the SELECT statement
-               // plus all base fields added for the needs of dataquery
-               foreach ($this->fieldTrueNames as $alias => $aliasInformation) {
-                       $table = $aliasInformation['aliasTable'];
-                       $field = $aliasInformation['field'];
-                       if (!isset($fieldCountPerTable[$table])) {
-                               $fieldCountPerTable[$table] = array();
-                       }
-                       if (!isset($fieldCountPerTable[$table][$field])) {
-                               $fieldCountPerTable[$table][$field] = array($alias);
-                       } else {
-                               $fieldCountPerTable[$table][$field][] = $alias;
-                       }
-               }
-               // Loop on the aliases list found in the first loop
-               // List a warning for each field (in a given table) with multiple aliases
-               foreach ($fieldCountPerTable as $table => $countPerField) {
-                       foreach ($countPerField as $field => $aliases) {
-                               if (count($aliases) > 1) {
-                                       $duplicates[] = 'In table ' . $table . ', duplicates for ' . $field . ' as: ' . implode(',', $aliases);
-                               }
-                       }
-
-               }
-               return $duplicates;
-       }
+    /**
+     * Adds where clause elements related to typical TYPO3 control parameters.
+     *
+     * The parameters are:
+     *
+     *    - the enable fields
+     *    - the language handling
+     *    - the versioning system
+     *
+     * @return    void
+     */
+    public function addTypo3Mechanisms()
+    {
+        // Add enable fields conditions
+        $this->addEnableFieldsCondition();
+        // Assemble a list of all currently selected fields for each table,
+        // skipping function calls (which can't be overlayed anyway)
+        // This is used by the next two methods, which may add some necessary fields,
+        // if not present already
+        $fieldsPerTable = array();
+        foreach ($this->queryFields as $alias => $tableData) {
+            $fieldsPerTable[$alias] = array();
+            foreach ($tableData['fields'] as $fieldData) {
+                if (!$fieldData['function']) {
+                    $fieldsPerTable[$alias][] = $fieldData['name'];
+                }
+            }
+        }
+        // Add language-related conditions
+        $this->addLanguageCondition($fieldsPerTable);
+        // Add versioning-related conditions
+        $this->addVersioningCondition($fieldsPerTable);
+    }
+
+    /**
+     * This method adds all SQL conditions needed to enforce the enable fields for
+     * all tables involved
+     *
+     * @return void
+     */
+    protected function addEnableFieldsCondition()
+    {
+        // First check if enable fields must really be added or should be ignored
+        if ($this->providerData['ignore_enable_fields'] == '0' || $this->providerData['ignore_enable_fields'] == '2') {
+
+            // Start with main table
+            // Define parameters for enable fields condition
+            $trueTableName = $this->queryObject->aliases[$this->queryObject->mainTable];
+            $showHidden = ($trueTableName === 'pages') ? $GLOBALS['TSFE']->showHiddenPage : $GLOBALS['TSFE']->showHiddenRecords;
+            $ignoreArray = $this->getIgnoreArray($this->queryObject->mainTable);
+
+            $enableClause = OverlayEngine::getEnableFieldsCondition(
+                    $trueTableName,
+                    $showHidden,
+                    $ignoreArray
+            );
+            // Replace the true table name by its alias if necessary
+            // NOTE: there's a risk that a field containing the table name might be modified abusively
+            // There's no real way around it except changing tx_overlays::getEnableFieldsCondition()
+            // to re-implement a better t3lib_page::enableFields()
+            // Adding the "." in the replacement reduces the risks
+            if ($this->queryObject->mainTable != $trueTableName) {
+                $enableClause = str_replace($trueTableName . '.', $this->queryObject->mainTable . '.', $enableClause);
+            }
+            $this->addWhereClause($enableClause);
+
+            // Add enable fields to JOINed tables
+            if (isset($this->queryObject->structure['JOIN']) && is_array($this->queryObject->structure['JOIN'])) {
+                foreach ($this->queryObject->structure['JOIN'] as $joinData) {
+
+                    // Define parameters for enable fields condition
+                    $table = $joinData['table'];
+                    $showHidden = ($table === 'pages') ? $GLOBALS['TSFE']->showHiddenPage : $GLOBALS['TSFE']->showHiddenRecords;
+                    $ignoreArray = $this->getIgnoreArray($joinData['alias']);
+
+                    $enableClause = OverlayEngine::getEnableFieldsCondition(
+                            $table,
+                            $showHidden,
+                            $ignoreArray
+                    );
+                    if (!empty($enableClause)) {
+                        if ($table != $joinData['alias']) {
+                            $enableClause = str_replace($table . '.', $joinData['alias'] . '.', $enableClause);
+                        }
+                        $this->addOnClause($enableClause, $joinData['alias']);
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * Adds SQL conditions related to language handling.
+     *
+     * Also adds the necessary fields to the list of SELECTed fields
+     *
+     * @param array $fieldsPerTable List of all fields already SELECTed, per table
+     *
+     * @return void
+     */
+    protected function addLanguageCondition($fieldsPerTable)
+    {
+        $skippedTablesForLanguageOverlays = GeneralUtility::trimExplode(',',
+                $this->providerData['skip_overlays_for_tables'], true);
+
+        // Add the language condition, if necessary
+        if (empty($this->providerData['ignore_language_handling']) && !$this->queryObject->structure['DISTINCT']) {
+
+            // Add the DB fields and the SQL conditions necessary for having everything ready to handle overlays
+            // as per the standard TYPO3 mechanism
+            // Loop on all tables involved
+            foreach ($this->queryFields as $alias => $tableData) {
+                $table = $tableData['name'];
+
+                // First entirely skip tables which are defined in the skip list
+                if (in_array($table, $skippedTablesForLanguageOverlays)) {
+                    $this->doOverlays[$table] = false;
+
+                    // Check which handling applies, based on existing TCA structure
+                    // The table must at least have a language field or point to a foreign table for translation
+                } elseif (isset($GLOBALS['TCA'][$table]['ctrl']['languageField']) || isset($GLOBALS['TCA'][$table]['ctrl']['transForeignTable'])) {
+
+                    // The table uses translations in the same table (transOrigPointerField) or in a foreign table (transForeignTable)
+                    // Prepare for overlays
+                    if (isset($GLOBALS['TCA'][$table]['ctrl']['transOrigPointerField']) || isset($GLOBALS['TCA'][$table]['ctrl']['transForeignTable'])) {
+                        // For each table, make sure that the fields necessary for handling the language overlay are included in the list of selected fields
+                        try {
+                            $fieldsForOverlayArray = OverlayEngine::selectOverlayFieldsArray($table,
+                                    implode(',', $fieldsPerTable[$alias]));
+                            // Extract which fields were added and add them to the list of fields to select
+                            $addedFields = array_diff($fieldsForOverlayArray, $fieldsPerTable[$alias]);
+                            if (count($addedFields) > 0) {
+                                foreach ($addedFields as $aField) {
+                                    $this->addExtraField($aField, $alias, $table);
+                                }
+                            }
+                            $this->doOverlays[$table] = true;
+                            // Add the language condition for the given table (only for tables containing their own translations)
+                            if (isset($GLOBALS['TCA'][$table]['ctrl']['transOrigPointerField'])) {
+                                $languageCondition = OverlayEngine::getLanguageCondition($table, $alias);
+                                if ($alias == $this->queryObject->mainTable) {
+                                    $this->addWhereClause($languageCondition);
+                                } else {
+                                    $this->addOnClause($languageCondition, $alias);
+                                }
+                            }
+                        } catch (\Exception $e) {
+                            $this->doOverlays[$table] = false;
+                        }
+                    }
+
+                    // The table simply contains a language flag.
+                    // This is just about adding the proper condition on the language field and nothing more
+                    // No overlays will be handled at a later time
+                } else {
+                    if (isset($GLOBALS['TCA'][$table]['ctrl']['languageField'])) {
+                        // Take language that corresponds to current language or [All]
+                        $languageCondition = $alias . '.' . $GLOBALS['TCA'][$table]['ctrl']['languageField'] . ' IN (' . $GLOBALS['TSFE']->sys_language_content . ', -1)';
+                        if ($alias == $this->queryObject->mainTable) {
+                            $this->addWhereClause($languageCondition);
+                        } else {
+                            $this->addOnClause($languageCondition, $alias);
+                        }
+                    }
+                }
+            }
+        }
+        //\TYPO3\CMS\Core\Utility\DebugUtility::debug($this->doOverlays);
+    }
+
+    /**
+     * Adds SQL conditions related version handling.
+     *
+     * Also add the necessary fields to the list of SELECTed fields.
+     * Contrary to the other conditions, versioning conditions are always added,
+     * if only to make sure that only LIVE records are selected.
+     *
+     * @param array $fieldsPerTable List of all fields already SELECTed, per table
+     *
+     * @return void
+     */
+    protected function addVersioningCondition($fieldsPerTable)
+    {
+        foreach ($this->queryFields as $alias => $tableData) {
+            $table = $tableData['name'];
+            $this->doVersioning[$table] = false;
+
+            // Continue if table indeed supports versioning
+            if (!empty($GLOBALS['TCA'][$table]['ctrl']['versioningWS'])) {
+                // By default make sure to take only LIVE version
+                $workspaceCondition = $alias . ".t3ver_oid = '0'";
+                // If in preview mode, assemble condition according to current workspace
+                if ($GLOBALS['TSFE']->sys_page->versioningPreview) {
+                    // For each table, make sure that the fields necessary for handling the language overlay are included in the list of selected fields
+                    try {
+                        $fieldsForOverlayArray = OverlayEngine::selectVersioningFieldsArray(
+                                $table,
+                                implode(',', $fieldsPerTable[$alias])
+                        );
+                        // Extract which fields were added and add them to the list of fields to select
+                        $addedFields = array_diff($fieldsForOverlayArray, $fieldsPerTable[$alias]);
+                        if (count($addedFields) > 0) {
+                            foreach ($addedFields as $aField) {
+                                $this->addExtraField($aField, $alias, $table);
+                            }
+                        }
+                        $this->doVersioning[$table] = true;
+                        $getVersionsDirectly = false;
+                        if ($this->providerData['get_versions_directly'] === '*' || in_array($alias, $this->providerData['get_versions_directly_exploded'])) {
+                            $getVersionsDirectly = true;
+                        }
+                        $workspaceCondition = OverlayEngine::getVersioningCondition(
+                                $table,
+                                $alias,
+                                $getVersionsDirectly
+                        );
+                    } catch (\Exception $e) {
+                        $this->doVersioning[$table] = false;
+                        $this->parentObject->getController()->addMessage(
+                                self::$extKey,
+                                'A problem happened with versioning: ' . $e->getMessage() . ' (' . $e->getCode() . ')',
+                                'Falling back to LIVE records for table ' . $table,
+                                FlashMessage::WARNING
+                        );
+                    }
+                }
+                if ($alias === $this->queryObject->mainTable) {
+                    $this->addWhereClause($workspaceCondition);
+                } else {
+                    $this->addOnClause($workspaceCondition, $alias);
+                }
+            }
+        }
+        //\TYPO3\CMS\Core\Utility\DebugUtility::debug($this->doVersioning);
+    }
+
+    /**
+     * Takes a Data Filter structure and processes its instructions.
+     *
+     * @param array $filter Data Filter structure
+     * @return void
+     */
+    public function addFilter($filter)
+    {
+        // First handle the "filter" part, which will be turned into part of a SQL WHERE clause
+        $completeFilters = array();
+        $logicalOperator = (empty($filter['logicalOperator'])) ? 'AND' : $filter['logicalOperator'];
+        if (isset($filter['filters']) && is_array($filter['filters'])) {
+            foreach ($filter['filters'] as $index => $filterData) {
+                $table = '';
+                // Check if the condition must be explicitly ignored
+                // (i.e. it is transmitted by the filter only for information)
+                // If not, resolve the table name, if possible
+                if ($filterData['void']) {
+                    $ignoreCondition = true;
+                } else {
+                    $ignoreCondition = false;
+                    $table = (empty($filterData['table'])) ? $this->queryObject->mainTable : $filterData['table'];
+                    // Check if the table is available in the query
+                    try {
+                        $table = $this->matchAliasOrTableName($table,
+                                'Filter - ' . ((empty($filterData['string'])) ? $index : $filterData['string']));
+                    } catch (InvalidQueryException $e) {
+                        $ignoreCondition = true;
+                        $this->parentObject->getController()->addMessage(
+                                self::$extKey,
+                                'The condition did not apply to a table used in the query.',
+                                'Condition ignored',
+                                FlashMessage::NOTICE,
+                                $filterData
+                        );
+                    }
+                }
+                // If the table is not in the query, ignore the condition
+                if (!$ignoreCondition) {
+                    $field = $filterData['field'];
+                    $fullField = $table . '.' . $field;
+                    // If the field is an alias, override full field definition
+                    // to whatever the alias is mapped to
+                    if (isset($this->queryObject->fieldAliasMappings[$field])) {
+                        $fullField = $this->queryObject->fieldAliasMappings[$field];
+                    }
+                    $condition = '';
+                    // Define table on which to apply the condition
+                    // Conditions will normally be applied in the WHERE clause
+                    // if the table is the main one, otherwise it is applied
+                    // in the ON clause of the relevant JOIN statement
+                    // However the application of the condition may be forced to be in the WHERE clause,
+                    // no matter which table it targets
+                    $tableForApplication = $table;
+                    if ($filterData['main']) {
+                        $tableForApplication = $this->queryObject->mainTable;
+                    }
+                    foreach ($filterData['conditions'] as $conditionData) {
+                        // If the value is special value "\all", all values must be taken,
+                        // so the condition is simply ignored
+                        if ($conditionData['value'] !== '\all') {
+                            try {
+                                $parsedCondition = SqlUtility::conditionToSql($fullField, $table, $conditionData);
+                                if (!empty($condition)) {
+                                    $condition .= ' AND ';
+                                }
+                                $condition .= '(' . $parsedCondition . ')';
+                                // If the operator was a full text search, store resulting condition which will be used
+                                // later to replace the placeholder in the SELECT part of the statement
+                                if ($conditionData['operator'] === 'fulltext' || $conditionData['operator'] === 'fulltext_natural') {
+                                    $fullFieldParts = explode('.', $fullField);
+                                    $placeholderKey = $table . '.fulltext.' . $fullFieldParts[2];
+                                    if (isset($this->queryObject->fulltextSearchPlaceholders[$placeholderKey])) {
+                                        $this->queryObject->fulltextSearchPlaceholders[$placeholderKey] = $condition;
+                                    }
+                                }
+                            } catch (InvalidQueryException $e) {
+                                $this->parentObject->getController()->addMessage(
+                                        self::$extKey,
+                                        $e->getMessage(),
+                                        'Condition ignored',
+                                        FlashMessage::WARNING,
+                                        $filterData
+                                );
+                            }
+                        }
+                    }
+                    // Add the condition only if it wasn't empty
+                    if (!empty($condition)) {
+                        if (empty($completeFilters[$tableForApplication])) {
+                            $completeFilters[$tableForApplication] = '';
+                        } else {
+                            $completeFilters[$tableForApplication] .= ' ' . $logicalOperator . ' ';
+                        }
+                        $completeFilters[$tableForApplication] .= '(' . $condition . ')';
+                    }
+                }
+            }
+            foreach ($completeFilters as $table => $whereClause) {
+                if ($table == $this->queryObject->mainTable) {
+                    $this->addWhereClause($whereClause);
+                } elseif (in_array($table, $this->queryObject->subtables)) {
+                    $this->addOnClause($whereClause, $table);
+                }
+            }
+            // Free some memory
+            unset($completeFilters);
+        }
+        // Add the eventual raw SQL in the filter
+        // Raw SQL is always added to the main where clause
+        if (!empty($filter['rawSQL'])) {
+            $this->addWhereClause($filter['rawSQL']);
+        }
+        // Handle the order by clauses
+        if (count($filter['orderby']) > 0) {
+            foreach ($filter['orderby'] as $orderData) {
+                // Special case if ordering is random
+                if ($orderData['order'] === 'RAND') {
+                    $this->queryObject->structure['ORDER BY'][] = 'RAND()';
+
+                    // Handle normal configuration
+                } else {
+                    $table = ((empty($orderData['table'])) ? $this->queryObject->mainTable : $orderData['table']);
+                    // Try applying the order clause to an existing table
+                    try {
+                        $table = $this->matchAliasOrTableName($table,
+                                'Order clause - ' . $table . ' - ' . $orderData['field'] . ' - ' . $orderData['order']);
+