Commit ed725d12 authored by Stefan Bürk's avatar Stefan Bürk Committed by Benni Mack
Browse files

[BUGFIX] Forward-compatible prepared statement support

doctrine/dbal:^3.2 changed return type of result for QueryBuilder
execute methods, no longer have used prepared statement accessible
for further query execution with other placeholder values.

To raise doctrine/dbal:^3.2 two places have been changed to reuse
the QueryBuilder instance itself instead of prepared statement with
the corresponding patch #96287, thus given up the performance gain
through reusable query execution plan in corresponding dbms systems.

This patch adds support for prepared statements to TYPO3's
QueryBuilder facade as this was not publicly available yet
for TYPO3 users to be forward-compatible with Doctrine DBAL 3.

Resolves: #96393
Related: #96287
Releases: main
Change-Id: I814670ebf9920ed3162a31f98ad9efd4031f47c9
Reviewed-on: https://review.typo3.org/c/Packages/TYPO3.CMS/+/72716

Tested-by: core-ci's avatarcore-ci <typo3@b13.com>
Tested-by: Benni Mack's avatarBenni Mack <benni@typo3.org>
Reviewed-by: Benni Mack's avatarBenni Mack <benni@typo3.org>
parent 21798082
......@@ -15,6 +15,7 @@
namespace TYPO3\CMS\Backend\Utility;
use Doctrine\DBAL\Statement;
use Psr\Http\Message\ServerRequestInterface;
use Psr\Log\LoggerInterface;
use TYPO3\CMS\Backend\Configuration\TranslationConfigurationProvider;
......@@ -431,6 +432,7 @@ class BackendUtility
if (is_array($pageForRootlineCache[$ident] ?? false)) {
$row = $pageForRootlineCache[$ident];
} else {
/** @var Statement $statement */
$statement = $runtimeCache->get('getPageForRootlineStatement-' . $statementCacheIdent);
if (!$statement) {
$queryBuilder = static::getQueryBuilderForTable('pages');
......@@ -468,18 +470,17 @@ class BackendUtility
$queryBuilder->expr()->eq('uid', $queryBuilder->createPositionalParameter($uid, \PDO::PARAM_INT)),
QueryHelper::stripLogicalOperatorPrefix($clause)
);
$statement = $queryBuilder->execute();
if (class_exists(\Doctrine\DBAL\ForwardCompatibility\Result::class) && $statement instanceof \Doctrine\DBAL\ForwardCompatibility\Result) {
$statement = $statement->getIterator();
}
$statement = $queryBuilder->prepare();
$runtimeCache->set('getPageForRootlineStatement-' . $statementCacheIdent, $statement);
} else {
$statement->bindValue(1, (int)$uid);
$statement->execute();
}
$row = $statement->fetchAssociative();
$statement->free();
$statement->bindValue(1, (int)$uid, \PDO::PARAM_INT);
$result = $statement->executeQuery();
$row = $result->fetchAssociative();
$result->free();
if ($row) {
if ($workspaceOL) {
self::workspaceOL('pages', $row);
......
......@@ -2377,19 +2377,19 @@ class DataHandler implements LoggerAwareInterface
$newValue = $originalValue = $value;
$queryBuilder = $this->getUniqueCountStatement($newValue, $table, $field, (int)$id, (int)$newPid);
// For as long as records with the test-value existing, try again (with incremented numbers appended)
$statement = $queryBuilder->execute();
if ($statement->fetchOne()) {
$statement = $queryBuilder->prepare();
$result = $statement->executeQuery();
if ($result->fetchOne()) {
for ($counter = 0; $counter <= 100; $counter++) {
$result->free();
$newValue = $value . $counter;
if (class_exists(\Doctrine\DBAL\ForwardCompatibility\Result::class) && $statement instanceof \Doctrine\DBAL\ForwardCompatibility\Result) {
$statement = $statement->getIterator();
}
$statement->bindValue(1, $newValue);
$statement->execute();
if (!$statement->fetchOne()) {
$result = $statement->executeQuery();
if (!$result->fetchOne()) {
break;
}
}
$result->free();
}
if ($originalValue !== $newValue) {
......
......@@ -25,6 +25,7 @@ use Doctrine\DBAL\Platforms\PostgreSQL94Platform as PostgreSqlPlatform;
use Doctrine\DBAL\Platforms\SQLServer2012Platform;
use Doctrine\DBAL\Result;
use Doctrine\DBAL\Schema\AbstractSchemaManager;
use Doctrine\DBAL\Types\Type;
use Doctrine\DBAL\VersionAwarePlatformDriver;
use Psr\Log\LoggerAwareInterface;
use Psr\Log\LoggerAwareTrait;
......
<?php
declare(strict_types=1);
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
namespace TYPO3\CMS\Core\Database\Query;
class NamedParameterNotSupportedForPreparedStatementException extends \InvalidArgumentException
{
public static function new(string $placeholderName): NamedParameterNotSupportedForPreparedStatementException
{
return new self(
sprintf(
"Cannot prepare statement for QueryBuilder because unsupported named placeholder '%s'",
$placeholderName
),
1639249867
);
}
}
......@@ -18,6 +18,8 @@ declare(strict_types=1);
namespace TYPO3\CMS\Core\Database\Query;
use Doctrine\DBAL\DBALException;
use Doctrine\DBAL\Driver\Statement as DriverStatement;
use Doctrine\DBAL\ParameterType;
use Doctrine\DBAL\Platforms\MySqlPlatform;
use Doctrine\DBAL\Platforms\OraclePlatform;
use Doctrine\DBAL\Platforms\PostgreSQL94Platform as PostgreSqlPlatform;
......@@ -25,6 +27,8 @@ use Doctrine\DBAL\Platforms\SqlitePlatform;
use Doctrine\DBAL\Platforms\SQLServer2012Platform as SQLServerPlatform;
use Doctrine\DBAL\Query\Expression\CompositeExpression;
use Doctrine\DBAL\Result;
use Doctrine\DBAL\Statement;
use Doctrine\DBAL\Types\Type;
use TYPO3\CMS\Core\Database\Connection;
use TYPO3\CMS\Core\Database\Query\Expression\ExpressionBuilder;
use TYPO3\CMS\Core\Database\Query\Restriction\DefaultRestrictionContainer;
......@@ -32,6 +36,7 @@ use TYPO3\CMS\Core\Database\Query\Restriction\LimitToTablesRestrictionContainer;
use TYPO3\CMS\Core\Database\Query\Restriction\QueryRestrictionContainerInterface;
use TYPO3\CMS\Core\Database\Query\Restriction\QueryRestrictionInterface;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Core\Utility\MathUtility;
/**
* Object oriented approach to building SQL queries.
......@@ -202,6 +207,49 @@ class QueryBuilder
return $this->concreteQueryBuilder;
}
/**
* Create prepared statement out of QueryBuilder instance.
*
* doctrine/dbal does not provide support for prepared statement
* in QueryBuilder, but as TYPO3 uses the API throughout the code
* via QueryBuilder, so the functionality of
* prepared statements for multiple executions is added.
*
* You should be aware that this method will throw a named
* 'UnsupportedPreparedStatementParameterTypeException()'
* exception, if 'PARAM_INT_ARRAY' or 'PARAM_STR_ARRAY' is set,
* as this is not supported for prepared statements directly.
*
* NamedPlaceholder are not supported, and if one or
* more are set a 'NamedParameterNotSupportedForPreparedStatementException'
* will be thrown.
*
* @return Statement
*/
public function prepare(): Statement
{
$connection = $this->getConnection();
$originalWhereConditions = null;
if ($this->getType() === \Doctrine\DBAL\Query\QueryBuilder::SELECT) {
$originalWhereConditions = $this->addAdditionalWhereConditions();
}
$sql = $this->concreteQueryBuilder->getSQL();
$params = $this->concreteQueryBuilder->getParameters();
$types = $this->concreteQueryBuilder->getParameterTypes();
$this->throwExceptionOnInvalidPreparedStatementParamArrayType($types);
$this->throwExceptionOnNamedParameterForPreparedStatement($params);
$statement = $connection->prepare($sql);
$this->bindTypedValues($statement, $params, $types);
if ($originalWhereConditions !== null) {
$this->concreteQueryBuilder->add('where', $originalWhereConditions, false);
}
return $statement;
}
/**
* Executes this query using the bound parameters and their types.
*
......@@ -211,7 +259,7 @@ class QueryBuilder
* decorator class also as preparation for extension authors, that
* they are able to write code which is compatible across two core
* versions and avoid deprecation warning. Additional this will ease
* backport without the need to switch if execute() is not used anymore.
* backports without the need to switch between execute() and executeQuery().
*
* It is recommended to use directly executeQuery() for 'SELECT' and
* executeStatement() for 'INSERT', 'UPDATE' and 'DELETE' queries.
......@@ -1306,4 +1354,101 @@ class QueryBuilder
$this->concreteQueryBuilder = clone $this->concreteQueryBuilder;
$this->restrictionContainer = clone $this->restrictionContainer;
}
private function throwExceptionOnInvalidPreparedStatementParamArrayType(array $types): void
{
$invalidTypeMap = [
Connection::PARAM_INT_ARRAY => 'PARAM_INT_ARRAY',
Connection::PARAM_STR_ARRAY => 'PARAM_STR_ARRAY',
];
foreach ($types as $type) {
if ($invalidTypeMap[$type] ?? false) {
throw UnsupportedPreparedStatementParameterTypeException::new($invalidTypeMap[$type]);
}
}
}
private function throwExceptionOnNamedParameterForPreparedStatement(array $params): void
{
foreach ($params as $key => $value) {
if (is_string($key) && !MathUtility::canBeInterpretedAsInteger($key)) {
throw NamedParameterNotSupportedForPreparedStatementException::new($key);
}
}
}
/**
* Binds a set of parameters, some or all of which are typed with a PDO binding type
* or DBAL mapping type, to a given statement.
*
* Cloned from doctrine/dbal connection, as we need to call from external
* to support and work with prepared statement from QueryBuilder instance
* directly.
*
* This needs to be checked with each doctrine/dbal release raise.
*
* @param DriverStatement $stmt Prepared statement
* @param list<mixed>|array<string, mixed> $params Statement parameters
* @param array<int, int|string|Type|null>|array<string, int|string|Type|null> $types Parameter types
*/
private function bindTypedValues(DriverStatement $stmt, array $params, array $types): void
{
// Check whether parameters are positional or named. Mixing is not allowed.
if (is_int(key($params))) {
$bindIndex = 1;
foreach ($params as $key => $value) {
if (isset($types[$key])) {
$type = $types[$key];
[$value, $bindingType] = $this->getBindingInfo($value, $type);
$stmt->bindValue($bindIndex, $value, $bindingType);
} else {
$stmt->bindValue($bindIndex, $value);
}
++$bindIndex;
}
} else {
// Named parameters
foreach ($params as $name => $value) {
if (isset($types[$name])) {
$type = $types[$name];
[$value, $bindingType] = $this->getBindingInfo($value, $type);
$stmt->bindValue($name, $value, $bindingType);
} else {
$stmt->bindValue($name, $value);
}
}
}
}
/**
* Gets the binding type of a given type.
*
* Cloned from doctrine/dbal connection, as we need to call from external
* to support and work with prepared statement from QueryBuilder instance
* directly.
*
* This needs to be checked with each doctrine/dbal release raise.
*
* @param mixed $value The value to bind.
* @param int|string|Type|null $type The type to bind (PDO or DBAL).
*
* @return array{mixed, int} [0] => the (escaped) value, [1] => the binding type.
*/
private function getBindingInfo($value, $type): array
{
if (is_string($type)) {
$type = Type::getType($type);
}
if ($type instanceof Type) {
$value = $type->convertToDatabaseValue($value, $this->getConnection()->getDatabasePlatform());
$bindingType = $type->getBindingType();
} else {
$bindingType = $type ?? ParameterType::STRING;
}
return [$value, $bindingType];
}
}
<?php
declare(strict_types=1);
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
namespace TYPO3\CMS\Core\Database\Query;
class UnsupportedPreparedStatementParameterTypeException extends \InvalidArgumentException
{
public static function new(string $parameterType): UnsupportedPreparedStatementParameterTypeException
{
return new self(
sprintf(
"Parameter type '%s' is not allowed for prepared statement retrieved from QueryBuilder. Use executeQuery() or executeStatement() directly.",
$parameterType
),
1639245170
);
}
}
"pages",,,,,,,,,,,,,,
,"uid","pid","sorting","deleted","sys_language_uid","l10n_parent","t3_origuid","t3ver_wsid","t3ver_state","t3ver_stage","t3ver_oid","title","slug",
,1,0,256,0,0,0,0,0,0,0,0,"First Root Page","/",
,10,1,128,0,0,0,0,0,0,0,0,"Page 1","/page-1",
,11,10,128,0,0,0,0,0,0,0,0,"SubPage 1 of Page 1","/page-1/sub-page-1",
,12,10,256,0,0,0,0,0,0,0,0,"SubPage 2 of Page 1","/page-1/sub-page-2",
,20,1,256,0,0,0,0,0,0,0,0,"Page 2","/page-2",
,21,20,128,0,0,0,0,0,0,0,0,"SubPage 1 of Page 2","/page-2/sub-page-2",
,22,20,256,0,0,0,0,0,0,0,0,"SubPage 2 of Page 1","/page-1/sub-page-2",
<?php
declare(strict_types=1);
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
namespace TYPO3\CMS\Core\Tests\Functional\Database\Query\QueryBuilder;
use Doctrine\DBAL\ParameterType;
use TYPO3\CMS\Core\Database\ConnectionPool;
use TYPO3\CMS\Core\Database\Query\NamedParameterNotSupportedForPreparedStatementException;
use TYPO3\CMS\Core\Database\Query\Restriction\DeletedRestriction;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\TestingFramework\Core\Functional\FunctionalTestCase;
class NamedPlaceholderPreparedStatementTest extends FunctionalTestCase
{
protected function setUp(): void
{
parent::setUp();
$this->withDatabaseSnapshot(function () {
$this->setUpDatabase();
});
}
protected function setUpDatabase(): void
{
$this->importCSVDataSet(__DIR__ . '/Fixtures/DataSet/queryBuilder_preparedStatement.csv');
}
/**
* @test
*/
public function throwsNamedParameterNotSupportedForPreparedStatementExceptionIfNamedPlacholderAreSetOnPrepare(): void
{
$this->expectExceptionObject(
NamedParameterNotSupportedForPreparedStatementException::new('dcValue1')
);
$queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
->getQueryBuilderForTable('pages');
$queryBuilder->getRestrictions()
->removeAll()
->add(GeneralUtility::makeInstance(DeletedRestriction::class));
$queryBuilder
->select(...['*'])
->from('pages')
->where(
$queryBuilder->expr()->eq(
'pid',
$queryBuilder->createNamedParameter(10, ParameterType::INTEGER)
)
)
->orderBy('sorting', 'ASC')
// add deterministic sort order as last sorting information, which many dbms
// and version does it by itself, but not all.
->addOrderBy('uid', 'ASC')
->prepare();
}
}
<?php
declare(strict_types=1);
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
namespace TYPO3\CMS\Core\Tests\Functional\Database\Query\QueryBuilder;
use Doctrine\DBAL\ParameterType;
use Doctrine\DBAL\Statement;
use TYPO3\CMS\Core\Cache\CacheManager;
use TYPO3\CMS\Core\Database\Connection;
use TYPO3\CMS\Core\Database\ConnectionPool;
use TYPO3\CMS\Core\Database\Query\QueryBuilder;
use TYPO3\CMS\Core\Database\Query\Restriction\DeletedRestriction;
use TYPO3\CMS\Core\Database\Query\UnsupportedPreparedStatementParameterTypeException;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\TestingFramework\Core\Functional\FunctionalTestCase;
class PositionPlaceholderPreparedStatementTest extends FunctionalTestCase
{
protected function setUp(): void
{
parent::setUp();
$this->withDatabaseSnapshot(function () {
$this->setUpDatabase();
});
}
protected function setUpDatabase(): void
{
$this->importCSVDataSet(__DIR__ . '/Fixtures/DataSet/queryBuilder_preparedStatement.csv');
}
/**
* @test
*/
public function canBeInstantiated(): void
{
$queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
->getQueryBuilderForTable('pages');
self::assertIsObject($queryBuilder);
self::assertInstanceOf(QueryBuilder::class, $queryBuilder);
}
/**
* @test
*/
public function preparedStatementWithPositionPlaceholderAndBindValueWorks(): void
{
$queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
->getQueryBuilderForTable('pages');
$queryBuilder->getRestrictions()
->removeAll()
->add(GeneralUtility::makeInstance(DeletedRestriction::class));
$statement = $queryBuilder
->select(...['*'])
->from('pages')
->where(
$queryBuilder->expr()->eq(
'pid',
$queryBuilder->createPositionalParameter(10, ParameterType::INTEGER)
)
)
->orderBy('sorting', 'ASC')
// add deterministic sort order as last sorting information, which many dbms
// and version does it by itself, but not all.
->addOrderBy('uid', 'ASC')
->prepare();
// first execution of prepared statement
$result1 = $statement->executeQuery();
$rows1 = $result1->fetchAllAssociative();
self::assertSame(2, count($rows1));
self::assertSame(11, (int)($rows1[0]['uid'] ?? 0));
self::assertSame(12, (int)($rows1[1]['uid'] ?? 0));
// second execution of prepared statement with changed placeholder value
$statement->bindValue(1, 20, ParameterType::INTEGER);
$result2 = $statement->executeQuery();
$rows2 = $result2->fetchAllAssociative();
self::assertSame(2, count($rows2));
self::assertSame(21, (int)($rows2[0]['uid'] ?? 0));
self::assertSame(22, (int)($rows2[1]['uid'] ?? 0));
}
/**
* @test
*/
public function preparedStatementWithPositionPlaceholderAndBindValueWithWileLoopWorks(): void
{
$queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
->getQueryBuilderForTable('pages');
$queryBuilder->getRestrictions()
->removeAll()
->add(GeneralUtility::makeInstance(DeletedRestriction::class));
$statement = $queryBuilder
->select(...['*'])
->from('pages')
->where(
$queryBuilder->expr()->eq(
'pid',
$queryBuilder->createPositionalParameter(10, ParameterType::INTEGER)
)
)
->orderBy('sorting', 'ASC')
// add deterministic sort order as last sorting information, which many dbms
// and version does it by itself, but not all.
->addOrderBy('uid', 'ASC')
->prepare();
// first execution of prepared statement
$result1 = $statement->executeQuery();
$rows1 = [];
while ($row = $result1->fetchAssociative()) {
$rows1[] = $row;
}
self::assertSame(2, count($rows1));
self::assertSame(11, (int)($rows1[0]['uid'] ?? 0));
self::assertSame(12, (int)($rows1[1]['uid'] ?? 0));
// second execution of prepared statement with changed placeholder value
$statement->bindValue(1, 20, ParameterType::INTEGER);
$result2 = $statement->executeQuery();
$rows2 = [];
while ($row = $result2->fetchAssociative()) {
$rows2[] = $row;
}
self::assertSame(2, count($rows2));
self::assertSame(21, (int)($rows2[0]['uid'] ?? 0));
self::assertSame(22, (int)($rows2[1]['uid'] ?? 0));
}
/**
* @test
*/
public function preparedStatementWithoutRetrievingFullResultSetAndWithoutFreeingPriorResultSetWorks(): void
{
$queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
->getQueryBuilderForTable('pages');
$queryBuilder->getRestrictions()
->removeAll()
->add(GeneralUtility::makeInstance(DeletedRestriction::class));
$statement = $queryBuilder
->select(...['*'])
->from('pages')