Commit 8290da21 authored by Marc Bastian Heinrichs's avatar Marc Bastian Heinrichs Committed by Marc Bastian Heinrichs
Browse files

[TASK] Optimize persisting a dirty objectStorage

This change optimizes persisting a dirty objectStorage
by not removing and inserting/updating all relations,
but insert/update/remove only new/dirty relations.
In the issue a wiki page is linked with some additional infos.

Resolves: #28091
Releases: 6.1
Change-Id: I1e861b62df0379eb84126c7f70f82287e23f0bdd
Reviewed-on: https://review.typo3.org/3390
Reviewed-by: Christian Kuhn
Tested-by: Christian Kuhn
Reviewed-by: Tymoteusz Motylewski
Tested-by: Tymoteusz Motylewski
Reviewed-by: Marc Bastian Heinrichs
Tested-by: Marc Bastian Heinrichs
parent c4df59f7
<?php
namespace TYPO3\CMS\Extbase\Persistence\Exception;
/*************************************************************
* Copyright notice
*
* All rights reserved.
*
* This script is part of the TYPO3 project. The TYPO3 project is
* free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* The GNU General Public License can be found at
* http://www.gnu.org/copyleft/gpl.html.
* A copy is found in the textfile GPL.txt and important notices to the license
* from the author is found in LICENSE.txt distributed with these scripts.
*
*
* This script is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details
*
* This copyright notice MUST APPEAR in all copies of the script!
***************************************************************/
/**
* An "Illegal Relation Type" exception
*/
class IllegalRelationTypeException extends \TYPO3\CMS\Extbase\Persistence\Exception {
}
?>
\ No newline at end of file
......@@ -457,25 +457,42 @@ class Backend implements \TYPO3\CMS\Extbase\Persistence\Generic\BackendInterface
$columnMap = $this->dataMapper->getDataMap($className)->getColumnMap($propertyName);
$propertyMetaData = $this->reflectionService->getClassSchema($className)->getProperty($propertyName);
foreach ($this->getRemovedChildObjects($parentObject, $propertyName) as $removedObject) {
$this->detachObjectFromParentObject($removedObject, $parentObject, $propertyName);
if ($columnMap->getTypeOfRelation() === \TYPO3\CMS\Extbase\Persistence\Generic\Mapper\ColumnMap::RELATION_HAS_MANY && $propertyMetaData['cascade'] === 'remove') {
$this->removeObject($removedObject);
} else {
$this->detachObjectFromParentObject($removedObject, $parentObject, $propertyName);
}
}
if ($columnMap->getTypeOfRelation() === \TYPO3\CMS\Extbase\Persistence\Generic\Mapper\ColumnMap::RELATION_HAS_AND_BELONGS_TO_MANY) {
$this->deleteAllRelationsFromRelationtable($parentObject, $propertyName);
}
$currentUids = array();
$sortingPosition = 1;
$updateSortingOfFollowing = FALSE;
foreach ($objectStorage as $object) {
if (empty($currentUids)) {
$sortingPosition = 1;
} else {
$sortingPosition++;
}
$cleanProperty = $parentObject->_getCleanProperty($propertyName);
if ($object->_isNew()) {
$this->insertObject($object);
$this->attachObjectToParentObject($object, $parentObject, $propertyName, $sortingPosition);
// if a new object is inserted, all objects after this need to have their sorting updated
$updateSortingOfFollowing = TRUE;
} elseif ($objectStorage->isRelationDirty($object) || $cleanProperty->getPosition($object) !== $objectStorage->getPosition($object)) {
$this->updateRelationOfObjectToParentObject($object, $parentObject, $propertyName, $sortingPosition);
// if a relation is dirty (speaking the same object is removed an added again at a different position), all objects after this needs to be updated the sorting
$updateSortingOfFollowing = TRUE;
} elseif ($updateSortingOfFollowing) {
if ($sortingPosition > $objectStorage->getPosition($object)) {
$this->updateRelationOfObjectToParentObject($object, $parentObject, $propertyName, $sortingPosition);
} else {
$sortingPosition = $objectStorage->getPosition($object);
}
}
$currentUids[] = $object->getUid();
$this->attachObjectToParentObject($object, $parentObject, $propertyName, $sortingPosition);
$sortingPosition++;
}
if ($columnMap->getParentKeyFieldName() === NULL) {
$row[$columnMap->getColumnName()] = implode(',', $currentUids);
} else {
......@@ -518,27 +535,69 @@ class Backend implements \TYPO3\CMS\Extbase\Persistence\Generic\BackendInterface
$parentDataMap = $this->dataMapper->getDataMap(get_class($parentObject));
$parentColumnMap = $parentDataMap->getColumnMap($parentPropertyName);
if ($parentColumnMap->getTypeOfRelation() === \TYPO3\CMS\Extbase\Persistence\Generic\Mapper\ColumnMap::RELATION_HAS_MANY) {
$row = array();
$parentKeyFieldName = $parentColumnMap->getParentKeyFieldName();
if ($parentKeyFieldName !== NULL) {
$row[$parentKeyFieldName] = $parentObject->getUid();
$parentTableFieldName = $parentColumnMap->getParentTableFieldName();
if ($parentTableFieldName !== NULL) {
$row[$parentTableFieldName] = $parentDataMap->getTableName();
}
}
$childSortByFieldName = $parentColumnMap->getChildSortByFieldName();
if (!empty($childSortByFieldName)) {
$row[$childSortByFieldName] = $sortingPosition;
}
if (count($row) > 0) {
$this->updateObject($object, $row);
}
$this->attachObjectToParentObjectRelationHasMany($object, $parentObject, $parentPropertyName, $sortingPosition);
} elseif ($parentColumnMap->getTypeOfRelation() === \TYPO3\CMS\Extbase\Persistence\Generic\Mapper\ColumnMap::RELATION_HAS_AND_BELONGS_TO_MANY) {
$this->insertRelationInRelationtable($object, $parentObject, $parentPropertyName, $sortingPosition);
}
}
/**
* Updates the fields defining the relation between the object and the parent object.
*
* @param \TYPO3\CMS\Extbase\DomainObject\DomainObjectInterface $object
* @param \TYPO3\CMS\Extbase\DomainObject\DomainObjectInterface $parentObject
* @param string $parentPropertyName
* @param integer $sortingPosition
* @return void
*/
protected function updateRelationOfObjectToParentObject(\TYPO3\CMS\Extbase\DomainObject\DomainObjectInterface $object, \TYPO3\CMS\Extbase\DomainObject\AbstractEntity $parentObject, $parentPropertyName, $sortingPosition = 0) {
$parentDataMap = $this->dataMapper->getDataMap(get_class($parentObject));
$parentColumnMap = $parentDataMap->getColumnMap($parentPropertyName);
if ($parentColumnMap->getTypeOfRelation() === \TYPO3\CMS\Extbase\Persistence\Generic\Mapper\ColumnMap::RELATION_HAS_MANY) {
$this->attachObjectToParentObjectRelationHasMany($object, $parentObject, $parentPropertyName, $sortingPosition);
} elseif ($parentColumnMap->getTypeOfRelation() === \TYPO3\CMS\Extbase\Persistence\Generic\Mapper\ColumnMap::RELATION_HAS_AND_BELONGS_TO_MANY) {
$this->updateRelationInRelationTable($object, $parentObject, $parentPropertyName, $sortingPosition);
}
}
/**
* Updates fields defining the relation between the object and the parent object in relation has-many.
*
* @param \TYPO3\CMS\Extbase\DomainObject\DomainObjectInterface $object
* @param \TYPO3\CMS\Extbase\DomainObject\AbstractEntity $parentObject
* @param string $parentPropertyName
* @param integer $sortingPosition
* @throws \TYPO3\CMS\Extbase\Persistence\Exception\IllegalRelationTypeException
* @return void
*/
protected function attachObjectToParentObjectRelationHasMany(\TYPO3\CMS\Extbase\DomainObject\DomainObjectInterface $object, \TYPO3\CMS\Extbase\DomainObject\AbstractEntity $parentObject, $parentPropertyName, $sortingPosition = 0) {
$parentDataMap = $this->dataMapper->getDataMap(get_class($parentObject));
$parentColumnMap = $parentDataMap->getColumnMap($parentPropertyName);
if ($parentColumnMap->getTypeOfRelation() !== \TYPO3\CMS\Extbase\Persistence\Generic\Mapper\ColumnMap::RELATION_HAS_MANY) {
throw new \TYPO3\CMS\Extbase\Persistence\Exception\IllegalRelationTypeException(
'Parent column relation type is ' . $parentColumnMap->getTypeOfRelation()
. ' but should be ' . \TYPO3\CMS\Extbase\Persistence\Generic\Mapper\ColumnMap::RELATION_HAS_MANY,
1345368105
);
}
$row = array();
$parentKeyFieldName = $parentColumnMap->getParentKeyFieldName();
if ($parentKeyFieldName !== NULL) {
$row[$parentKeyFieldName] = $parentObject->getUid();
$parentTableFieldName = $parentColumnMap->getParentTableFieldName();
if ($parentTableFieldName !== NULL) {
$row[$parentTableFieldName] = $parentDataMap->getTableName();
}
}
$childSortByFieldName = $parentColumnMap->getChildSortByFieldName();
if (!empty($childSortByFieldName)) {
$row[$childSortByFieldName] = $sortingPosition;
}
if (!empty($row)) {
$this->updateObject($object, $row);
}
}
/**
* Updates the fields defining the relation between the object and the parent object.
*
......@@ -656,6 +715,40 @@ class Backend implements \TYPO3\CMS\Extbase\Persistence\Generic\BackendInterface
return $res;
}
/**
* Inserts mm-relation into a relation table
*
* @param \TYPO3\CMS\Extbase\DomainObject\DomainObjectInterface $object The related object
* @param \TYPO3\CMS\Extbase\DomainObject\DomainObjectInterface $parentObject The parent object
* @param string $propertyName The name of the parent object's property where the related objects are stored in
* @param integer $sortingPosition Defaults to NULL
* @return integer The uid of the inserted row
*/
protected function updateRelationInRelationTable(\TYPO3\CMS\Extbase\DomainObject\DomainObjectInterface $object, \TYPO3\CMS\Extbase\DomainObject\DomainObjectInterface $parentObject, $propertyName, $sortingPosition = 0) {
$dataMap = $this->dataMapper->getDataMap(get_class($parentObject));
$columnMap = $dataMap->getColumnMap($propertyName);
$row = array(
$columnMap->getParentKeyFieldName() => (int)$parentObject->getUid(),
$columnMap->getChildKeyFieldName() => (int)$object->getUid(),
$columnMap->getChildSortByFieldName() => (int)$sortingPosition
);
$relationTableName = $columnMap->getRelationTableName();
// FIXME Reenable support for tablenames
// $childTableName = $columnMap->getChildTableName();
// if (isset($childTableName)) {
// $row['tablenames'] = $childTableName;
// }
$relationTableMatchFields = $columnMap->getRelationTableMatchFields();
if (is_array($relationTableMatchFields) && count($relationTableMatchFields) > 0) {
$row = array_merge($relationTableMatchFields, $row);
}
$res = $this->storageBackend->updateRelationTableRow(
$relationTableName,
$row);
return $res;
}
/**
* Delete all mm-relations of a parent from a relation table
*
......@@ -697,6 +790,60 @@ class Backend implements \TYPO3\CMS\Extbase\Persistence\Generic\BackendInterface
return $res;
}
/**
* Fetches maximal value currently used for sorting field in parent table
*
* @param \TYPO3\CMS\Extbase\DomainObject\DomainObjectInterface $parentObject The parent object
* @param string $parentPropertyName The name of the parent object's property where the related objects are stored in
* @throws \TYPO3\CMS\Extbase\Persistence\Exception\IllegalRelationTypeException
* @return mixed the max value
*/
protected function fetchMaxSortingFromParentTable(\TYPO3\CMS\Extbase\DomainObject\DomainObjectInterface $parentObject, $parentPropertyName) {
$parentDataMap = $this->dataMapper->getDataMap(get_class($parentObject));
$parentColumnMap = $parentDataMap->getColumnMap($parentPropertyName);
if ($parentColumnMap->getTypeOfRelation() === \TYPO3\CMS\Extbase\Persistence\Generic\Mapper\ColumnMap::RELATION_HAS_MANY) {
$tableName = $parentColumnMap->getChildTableName();
$sortByFieldName = $parentColumnMap->getChildSortByFieldName();
if (empty($sortByFieldName)) {
return FALSE;
}
$matchFields = array();
$parentKeyFieldName = $parentColumnMap->getParentKeyFieldName();
if ($parentKeyFieldName !== NULL) {
$matchFields[$parentKeyFieldName] = $parentObject->getUid();
$parentTableFieldName = $parentColumnMap->getParentTableFieldName();
if ($parentTableFieldName !== NULL) {
$matchFields[$parentTableFieldName] = $parentDataMap->getTableName();
}
}
if (empty($matchFields)) {
return FALSE;
}
} elseif ($parentColumnMap->getTypeOfRelation() === \TYPO3\CMS\Extbase\Persistence\Generic\Mapper\ColumnMap::RELATION_HAS_AND_BELONGS_TO_MANY) {
$tableName = $parentColumnMap->getRelationTableName();
$sortByFieldName = $parentColumnMap->getChildSortByFieldName();
$matchFields = array(
$parentColumnMap->getParentKeyFieldName() => (int)$parentObject->getUid()
);
$relationTableMatchFields = $parentColumnMap->getRelationTableMatchFields();
if (is_array($relationTableMatchFields) && count($relationTableMatchFields) > 0) {
$matchFields = array_merge($relationTableMatchFields, $matchFields);
}
} else {
throw new \TYPO3\CMS\Extbase\Persistence\Exception\IllegalRelationTypeException('Unexpected parent column relation type:' . $parentColumnMap->getTypeOfRelation(), 1345368106);
}
$result = $this->storageBackend->getMaxValueFromTable(
$tableName,
$matchFields,
$sortByFieldName);
return $result;
}
/**
* Updates a given object in the storage
*
......
......@@ -73,7 +73,7 @@ class ColumnMap {
/**
* The type of relation
*
* @var int
* @var string
*/
protected $typeOfRelation;
......
......@@ -38,7 +38,7 @@ interface BackendInterface {
* @param string $tableName The database table name
* @param array $row The row to insert
* @param boolean $isRelation TRUE if we are currently inserting into a relation table, FALSE by default
* @return mixed|void
* @return integer the UID of the inserted row
*/
public function addRow($tableName, array $row, $isRelation = FALSE);
......@@ -52,6 +52,15 @@ interface BackendInterface {
*/
public function updateRow($tableName, array $row, $isRelation = FALSE);
/**
* Updates a relation row in the storage
*
* @param string $tableName The database relation table name
* @param array $row The row to be updated
* @return boolean
*/
public function updateRelationTableRow($tableName, array $row);
/**
* Deletes a row in the storage
*
......@@ -62,6 +71,16 @@ interface BackendInterface {
*/
public function removeRow($tableName, array $identifier, $isRelation = FALSE);
/**
* Fetches maximal value for given table column
*
* @param string $tableName The database table name
* @param array $identifier An array of identifier array('fieldname' => value). This array will be transformed to a WHERE clause
* @param string $columnName column name to get the max value from
* @return mixed the max value
*/
public function getMaxValueFromTable($tableName, $identifier, $columnName);
/**
* Returns the number of items matching the query.
*
......
......@@ -208,6 +208,42 @@ class Typo3DbBackend implements \TYPO3\CMS\Extbase\Persistence\Generic\Storage\B
return $returnValue;
}
/**
* Updates a relation row in the storage.
*
* @param string $tableName The database relation table name
* @param array $row The row to be updated
* @throws \InvalidArgumentException
* @return boolean
*/
public function updateRelationTableRow($tableName, array $row) {
if (!isset($row['uid_local']) && !isset($row['uid_foreign'])) {
throw new \InvalidArgumentException(
'The given row must contain a value for "uid_local" and "uid_foreign".', 1360500126
);
}
$uidLocal = (int) $row['uid_local'];
$uidForeign = (int) $row['uid_foreign'];
unset($row['uid_local']);
unset($row['uid_foreign']);
$fields = array();
$parameters = array();
foreach ($row as $columnName => $value) {
$fields[] = $columnName . '=?';
$parameters[] = $value;
}
$parameters[] = $uidLocal;
$parameters[] = $uidForeign;
$sqlString = 'UPDATE ' . $tableName . ' SET ' . implode(', ', $fields) . ' WHERE uid_local=? AND uid_foreign=?';
$this->replacePlaceholders($sqlString, $parameters);
$returnValue = $this->databaseHandle->sql_query($sqlString);
$this->checkSqlErrors($sqlString);
return $returnValue;
}
/**
* Deletes a row in the storage
*
......@@ -228,6 +264,24 @@ class Typo3DbBackend implements \TYPO3\CMS\Extbase\Persistence\Generic\Storage\B
return $returnValue;
}
/**
* Fetches maximal value for given table column from database.
*
* @param string $tableName The database table name
* @param array $identifier An array of identifier array('fieldname' => value). This array will be transformed to a WHERE clause
* @param string $columnName column name to get the max value from
* @return mixed the max value
*/
public function getMaxValueFromTable($tableName, $identifier, $columnName) {
$sqlString = 'SELECT ' . $columnName . ' FROM ' . $tableName . ' WHERE ' . $this->parseIdentifier($identifier) . ' ORDER BY ' . $columnName . ' DESC LIMIT 1';
$this->replacePlaceholders($sqlString, $identifier);
$result = $this->databaseHandle->sql_query($sqlString);
$row = $this->databaseHandle->sql_fetch_assoc($result);
$this->checkSqlErrors($sqlString);
return $row[$columnName];
}
/**
* Fetches row data from the database
*
......
......@@ -68,6 +68,30 @@ class ObjectStorage implements \Countable, \Iterator, \ArrayAccess, \TYPO3\CMS\E
*/
protected $isModified = FALSE;
/**
* An array holding the internal position the object was added.
* The object entry is unsetted when the object gets removed from the objectstorage
*
* @var array
*/
protected $addedObjectsPositions = array();
/**
* An array holding the internal position the object was added before, when it would
* be removed from the objectstorage
*
* @var array
*/
protected $removedObjectsPositions = array();
/**
* An internal var holding the count of added objects to be stored as position.
* It would be resetted, when all objects will be removed from the objectstorage
*
* @var integer
*/
protected $positionCounter = 0;
/**
* Rewinds the iterator to the first storage element.
*
......@@ -135,6 +159,9 @@ class ObjectStorage implements \Countable, \Iterator, \ArrayAccess, \TYPO3\CMS\E
public function offsetSet($object, $information) {
$this->isModified = TRUE;
$this->storage[spl_object_hash($object)] = array('obj' => $object, 'inf' => $information);
$this->positionCounter++;
$this->addedObjectsPositions[spl_object_hash($object)] = $this->positionCounter;
}
/**
......@@ -156,6 +183,13 @@ class ObjectStorage implements \Countable, \Iterator, \ArrayAccess, \TYPO3\CMS\E
public function offsetUnset($object) {
$this->isModified = TRUE;
unset($this->storage[spl_object_hash($object)]);
if (empty($this->storage)) {
$this->positionCounter = 0;
}
$this->removedObjectsPositions[spl_object_hash($object)] = $this->addedObjectsPositions[spl_object_hash($object)];
unset($this->addedObjectsPositions[spl_object_hash($object)]);
}
/**
......@@ -297,6 +331,29 @@ class ObjectStorage implements \Countable, \Iterator, \ArrayAccess, \TYPO3\CMS\E
public function _isDirty() {
return $this->isModified;
}
}
/**
* Returns TRUE if an object is added, then removed and added at a different position
*
* @param mixed $object
* @return boolean
*/
public function isRelationDirty($object) {
return (isset($this->addedObjectsPositions[spl_object_hash($object)])
&& isset($this->removedObjectsPositions[spl_object_hash($object)])
&& ($this->addedObjectsPositions[spl_object_hash($object)] !== $this->removedObjectsPositions[spl_object_hash($object)]));
}
/**
* @param mixed $object
* @return integer|NULL
*/
public function getPosition($object) {
if (!isset($this->addedObjectsPositions[spl_object_hash($object)])) {
return NULL;
}
return $this->addedObjectsPositions[spl_object_hash($object)];
}
}
?>
\ No newline at end of file
......@@ -192,6 +192,71 @@ class ObjectStorageTest extends \TYPO3\CMS\Extbase\Tests\Unit\BaseTestCase {
$objectStorage->attach($object2, 'bar');
$this->assertEquals($objectStorage->toArray(), array($object1, $object2));
}
}
/**
* @test
*/
public function allRelationsAreNotDirtyOnAttaching() {
$objectStorage = new \TYPO3\CMS\Extbase\Persistence\ObjectStorage();
$object1 = new \StdClass();
$object2 = new \StdClass();
$object3 = new \StdClass();
$objectStorage->attach($object1);
$objectStorage->attach($object2);
$objectStorage->attach($object3);
$this->assertFalse($objectStorage->isRelationDirty($object1));
$this->assertFalse($objectStorage->isRelationDirty($object2));
$this->assertFalse($objectStorage->isRelationDirty($object3));
}
/**
* @test
*/
public function allRelationsAreNotDirtyOnAttachingAndRemoving() {
$objectStorage = new \TYPO3\CMS\Extbase\Persistence\ObjectStorage();
$object1 = new \StdClass;
$object2 = new \StdClass;
$object3 = new \StdClass;
$objectStorage->attach($object1);
$objectStorage->attach($object2);
$objectStorage->detach($object2);
$objectStorage->attach($object3);
$this->assertFalse($objectStorage->isRelationDirty($object1));
$this->assertFalse($objectStorage->isRelationDirty($object3));
}
/**
* @test
*/
public function theRelationsAreNotDirtyOnReAddingAtSamePosition() {
$objectStorage = new \TYPO3\CMS\Extbase\Persistence\ObjectStorage();
$object1 = new \StdClass;
$object2 = new \StdClass;
$objectStorage->attach($object1);
$objectStorage->attach($object2);
$clonedStorage = clone $objectStorage;
$objectStorage->removeAll($clonedStorage);
$objectStorage->attach($object1);
$objectStorage->attach($object2);
$this->assertFalse($objectStorage->isRelationDirty($object1));
$this->assertFalse($objectStorage->isRelationDirty($object2));
}
/**
* @test
*/
public function theRelationsAreDirtyOnReAddingAtOtherPosition() {
$objectStorage = new \TYPO3\CMS\Extbase\Persistence\ObjectStorage();
$object1 = new \StdClass;
$object2 = new \StdClass;
$objectStorage->attach($object1);
$objectStorage->attach($object2);
$clonedStorage = clone $objectStorage;
$objectStorage->removeAll($clonedStorage);
$objectStorage->attach($object2);
$objectStorage->attach($object1);
$this->assertTrue($objectStorage->isRelationDirty($object1));
$this->assertTrue($objectStorage->isRelationDirty($object2));
}
}
?>
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment