Commit c034d904 authored by Andreas Fernandez's avatar Andreas Fernandez Committed by Anja Leichsenring
Browse files

[BUGFIX] Chunk requests in documentation viewer to improve performance

Improve performance of „View Upgrade Documentation“ module by reading
changelogs in chunks.

At first, all available versions are loaded. After that, all files are
loaded chunked by their respective version.

Resolves: #86281
Releases: master
Change-Id: I4c07842d9389028c1899022721f66b866fb81919
Reviewed-on: https://review.typo3.org/58299


Reviewed-by: Christian Kuhn's avatarChristian Kuhn <lolli@schwarzbu.ch>
Tested-by: Christian Kuhn's avatarChristian Kuhn <lolli@schwarzbu.ch>
Tested-by: default avatarTYPO3com <no-reply@typo3.com>
Reviewed-by: Anja Leichsenring's avatarAnja Leichsenring <aleichsenring@ab-softlab.de>
Tested-by: Anja Leichsenring's avatarAnja Leichsenring <aleichsenring@ab-softlab.de>
parent ec30f217
......@@ -751,7 +751,7 @@ class UpgradeController extends AbstractController
}
/**
* Render list of .rst files
* Render list of versions
*
* @param ServerRequestInterface $request
* @return ResponseInterface
......@@ -759,11 +759,33 @@ class UpgradeController extends AbstractController
public function upgradeDocsGetContentAction(ServerRequestInterface $request): ResponseInterface
{
$formProtection = FormProtectionFactory::get(InstallToolFormProtection::class);
$documentationFiles = $this->getDocumentationFiles();
$documentationDirectories = $this->getDocumentationDirectories();
$view = $this->initializeStandaloneView($request, 'Upgrade/UpgradeDocsGetContent.html');
$view->assignMultiple([
'upgradeDocsMarkReadToken' => $formProtection->generateToken('installTool', 'upgradeDocsMarkRead'),
'upgradeDocsUnmarkReadToken' => $formProtection->generateToken('installTool', 'upgradeDocsUnmarkRead'),
'upgradeDocsVersions' => $documentationDirectories,
]);
return new JsonResponse([
'success' => true,
'html' => $view->render(),
]);
}
/**
* Render list of .rst files
*
* @param ServerRequestInterface $request
* @return ResponseInterface
*/
public function upgradeDocsGetChangelogForVersionAction(ServerRequestInterface $request): ResponseInterface
{
$version = $request->getQueryParams()['install']['version'] ?? '';
$this->assertValidVersion($version);
$documentationFiles = $this->getDocumentationFiles($version);
$view = $this->initializeStandaloneView($request, 'Upgrade/UpgradeDocsGetChangelogForVersion.html');
$view->assignMultiple([
'upgradeDocsFiles' => $documentationFiles['normalFiles'],
'upgradeDocsReadFiles' => $documentationFiles['readFiles'],
'upgradeDocsNotAffectedFiles' => $documentationFiles['notAffectedFiles'],
......@@ -1137,16 +1159,29 @@ class UpgradeController extends AbstractController
}
}
/**
* @return string[]
*/
protected function getDocumentationDirectories(): array
{
$documentationFileService = new DocumentationFile();
$documentationDirectories = $documentationFileService->findDocumentationDirectories(
str_replace('\\', '/', realpath(ExtensionManagementUtility::extPath('core') . 'Documentation/Changelog'))
);
return array_reverse($documentationDirectories);
}
/**
* Get a list of '.rst' files and their details for "Upgrade documentation" view.
*
* @param string $version
* @return array
*/
protected function getDocumentationFiles(): array
protected function getDocumentationFiles(string $version): array
{
$documentationFileService = new DocumentationFile();
$documentationFiles = $documentationFileService->findDocumentationFiles(
str_replace('\\', '/', realpath(ExtensionManagementUtility::extPath('core') . 'Documentation/Changelog'))
str_replace('\\', '/', realpath(ExtensionManagementUtility::extPath('core') . 'Documentation/Changelog/' . $version))
);
$documentationFiles = array_reverse($documentationFiles);
......@@ -1184,24 +1219,14 @@ class UpgradeController extends AbstractController
}
$readFiles = [];
foreach ($documentationFiles as $section => &$files) {
foreach ($files as $fileId => $fileData) {
if (in_array($fileData['file_hash'], $hashesMarkedAsRead, true)) {
$fileData['section'] = $section;
$readFiles[$fileId] = $fileData;
unset($files[$fileId]);
}
}
}
$notAffectedFiles = [];
foreach ($documentationFiles as $section => &$files) {
foreach ($files as $fileId => $fileData) {
if (in_array($fileData['file_hash'], $hashesMarkedAsNotAffected, true)) {
$fileData['section'] = $section;
$notAffectedFiles[$fileId] = $fileData;
unset($files[$fileId]);
}
foreach ($documentationFiles as $fileId => $fileData) {
if (in_array($fileData['file_hash'], $hashesMarkedAsRead, true)) {
$readFiles[$fileId] = $fileData;
unset($documentationFiles[$fileId]);
} elseif (in_array($fileData['file_hash'], $hashesMarkedAsNotAffected, true)) {
$notAffectedFiles[$fileId] = $fileData;
unset($documentationFiles[$fileId]);
}
}
......@@ -1228,4 +1253,17 @@ class UpgradeController extends AbstractController
}
return $line;
}
/**
* Asserts that the given version is valid
*
* @param string $version
* @throws \InvalidArgumentException
*/
protected function assertValidVersion(string $version): void
{
if ($version !== 'master' && !preg_match('/^\d+.\d+(?:.(?:\d+|x))?$/', $version)) {
throw new \InvalidArgumentException('Given version "' . $version . '" is invalid', 1537209128);
}
}
}
......@@ -16,6 +16,8 @@ namespace TYPO3\CMS\Install\UpgradeAnalysis;
* The TYPO3 project - inspiring people to share!
*/
use Symfony\Component\Finder\Finder;
use Symfony\Component\Finder\SplFileInfo;
use TYPO3\CMS\Core\Registry;
use TYPO3\CMS\Core\Utility\ExtensionManagementUtility;
use TYPO3\CMS\Core\Utility\GeneralUtility;
......@@ -59,6 +61,34 @@ class DocumentationFile
$this->changelogPath = str_replace('\\', '/', $this->changelogPath);
}
/**
* Traverse given directory, select directories
*
* @param string $path
* @return string[] Version directories
* @throws \InvalidArgumentException
*/
public function findDocumentationDirectories(string $path): array
{
if (strcasecmp($path, $this->changelogPath) < 0 || strpos($path, $this->changelogPath) === false) {
throw new \InvalidArgumentException('the given path does not belong to the changelog dir. Aborting', 1537158043);
}
$finder = new Finder();
$finder
->depth(0)
->sortByName()
->in($path);
$directories = [];
foreach ($finder->directories() as $directory) {
/** @var $directory SplFileInfo */
$directories[] = $directory->getBasename();
}
return $directories;
}
/**
* Traverse given directory, select files
*
......@@ -72,15 +102,7 @@ class DocumentationFile
throw new \InvalidArgumentException('the given path does not belong to the changelog dir. Aborting', 1485425530);
}
$documentationFiles = [];
$versionDirectories = scandir($path);
$fileInfo = pathinfo($path);
$absolutePath = str_replace('\\', '/', $fileInfo['dirname']) . '/' . $fileInfo['basename'];
foreach ($versionDirectories as $version) {
$directory = $absolutePath . '/' . $version;
$documentationFiles += $this->getDocumentationFilesForVersion($directory, $version);
}
$documentationFiles = $this->getDocumentationFilesForVersion($path);
$this->tagsTotal = $this->collectTagTotal($documentationFiles);
return $documentationFiles;
......@@ -91,6 +113,7 @@ class DocumentationFile
*
* @param string $file Absolute path to documentation file
* @return array
* @throws \InvalidArgumentException
*/
public function getListEntry(string $file): array
{
......@@ -99,6 +122,7 @@ class DocumentationFile
}
$lines = file($file, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
$headline = $this->extractHeadline($lines);
$entry['version'] = PathUtility::basename(PathUtility::dirname($file));
$entry['headline'] = $headline;
$entry['filepath'] = $file;
$entry['tags'] = $this->extractTags($lines);
......@@ -202,43 +226,36 @@ class DocumentationFile
}
/**
* True for real directories and a valid version
*
* @param string $versionDirectory
* @param string $docDirectory
* @param string $version
* @return bool
*/
protected function isRelevantDirectory(string $versionDirectory, string $version): bool
protected function versionHasDocumentationFiles(string $docDirectory, string $version): bool
{
return is_dir($versionDirectory) && $version !== '.' && $version !== '..';
$absolutePath = str_replace('\\', '/', $docDirectory) . '/' . $version;
$finder = $this->getDocumentFinder()->in($absolutePath);
return $finder->files()->count() > 0;
}
/**
* Handle a single directory
*
* @param string $docDirectory
* @param string $version
* @return array
*/
protected function getDocumentationFilesForVersion(
string $docDirectory,
string $version
): array {
$documentationFiles = [];
if ($this->isRelevantDirectory($docDirectory, $version)) {
$documentationFiles[$version] = [];
$absolutePath = str_replace('\\', '/', PathUtility::dirname($docDirectory)) . '/' . $version;
$rstFiles = scandir($docDirectory);
foreach ($rstFiles as $file) {
$fileInfo = pathinfo($file);
if ($this->isRelevantFile($fileInfo)) {
$filePath = $absolutePath . '/' . $fileInfo['basename'];
$documentationFiles[$version] += $this->getListEntry($filePath);
}
}
protected function getDocumentationFilesForVersion(string $docDirectory): array
{
$documentationFiles = [[]];
$absolutePath = str_replace('\\', '/', $docDirectory);
$finder = $this->getDocumentFinder()->in($absolutePath);
foreach ($finder->files() as $file) {
/** @var $file SplFileInfo */
$documentationFiles[] = $this->getListEntry($file->getPathname());
}
return $documentationFiles;
return array_merge(...$documentationFiles);
}
/**
......@@ -249,14 +266,12 @@ class DocumentationFile
*/
protected function collectTagTotal($documentationFiles): array
{
$tags = [];
foreach ($documentationFiles as $versionArray) {
foreach ($versionArray as $fileArray) {
$tags = array_merge(array_unique($tags), $fileArray['tags']);
}
$tags = [[]];
foreach ($documentationFiles as $fileArray) {
$tags[] = $fileArray['tags'];
}
return array_unique($tags);
return array_unique(array_merge(...$tags));
}
/**
......@@ -297,7 +312,6 @@ class DocumentationFile
* @param string $rstContent
*
* @return string
* @throws \InvalidArgumentException
*/
protected function parseContent(string $rstContent): string
{
......@@ -309,4 +323,17 @@ class DocumentationFile
$content = preg_replace('/.. include::(.*)/', '', $content);
return trim($content);
}
/**
* @return Finder
*/
protected function getDocumentFinder(): Finder
{
$finder = new Finder();
$finder
->depth(0)
->name('/^(Feature|Breaking|Deprecation|Important)\-\d+.+\.rst$/i');
return $finder;
}
}
<html xmlns:f="http://typo3.org/ns/TYPO3/CMS/Fluid/ViewHelpers" data-namespace-typo3-fluid="true">
<div class="panel panel-rst panel-{fileArray.class} risk-medium upgrade_analysis_item_to_filter item" data-item-tags="{fileArray.tagList}" data-item-version="{version}" id="heading{id}">
<div class="panel panel-rst panel-{fileArray.class} risk-medium upgrade_analysis_item_to_filter item" data-item-tags="{fileArray.tagList}" data-item-version="{fileArray.version}" data-item-state="{state}" id="heading{id}">
<div class="panel-heading" role="tab">
<h3 class="panel-title">
<f:if condition="{read}">
......
<html xmlns:f="http://typo3.org/ns/TYPO3/CMS/Fluid/ViewHelpers" data-namespace-typo3-fluid="true">
<f:for each="{upgradeDocsFiles}" key="fileId" as="fileArray">
<f:render partial="Upgrade/UpgradeDocs/PanelItem" arguments="{id: 'file-{fileId}', fileArray: fileArray, state: 'regular'}"/>
</f:for>
<f:for each="{upgradeDocsReadFiles}" key="fileId" as="fileArray">
<f:render partial="Upgrade/UpgradeDocs/PanelItem" arguments="{id: 'file-{fileId}', fileArray: fileArray, state: 'read', read: 'true'}"/>
</f:for>
<f:for each="{upgradeDocsNotAffectedFiles}" key="fileId" as="fileArray">
<f:render partial="Upgrade/UpgradeDocs/PanelItem" arguments="{id: 'file-{fileId}', fileArray: fileArray, state: 'notAffected', read: 'true'}"/>
</f:for>
</html>
<html xmlns:f="http://typo3.org/ns/TYPO3/CMS/Fluid/ViewHelpers" data-namespace-typo3-fluid="true">
<html xmlns:core="http://typo3.org/ns/TYPO3/CMS/Core/ViewHelpers" xmlns:f="http://typo3.org/ns/TYPO3/CMS/Fluid/ViewHelpers" data-namespace-typo3-fluid="true">
<div class="t3js-module-content" data-upgrade-cocs-mark-read-token="{upgradeDocsMarkReadToken}" data-upgrade-docs-unmark-read-token="{upgradeDocsUnmarkReadToken}">
<div class="t3js-module-content" data-upgrade-docs-mark-read-token="{upgradeDocsMarkReadToken}" data-upgrade-docs-unmark-read-token="{upgradeDocsUnmarkReadToken}">
<div class="row">
<div class="col-md-12">
<div class="form-group">
......@@ -10,6 +10,7 @@
type="text"
class="form-control t3js-upgradeDocs-fulltext-search"
placeholder="search setting"
disabled
>
</div>
</div>
......@@ -26,6 +27,7 @@
style="width:100%;"
multiple
tabindex=""
disabled
>
</select>
</div>
......@@ -34,31 +36,26 @@
</div>
<div class="panel-group panel-group-rst" role="tablist" aria-multiselectable="true">
<f:for each="{upgradeDocsFiles}" as="versionArray" key="version" iteration="iterator">
<f:if condition="{versionArray -> f:count()} > 0">
<div class="panel panel-default panel-version">
<div class="panel-heading" role="tab" id="heading-{iterator.index}">
<h2 class="panel-title">
<a href="#version-{iterator.index}"
class="collapsed" data-toggle="collapse"
aria-expanded="false"
aria-controls="#version-{iterator.index}"
>
<span class="caret"></span>
Version: <strong>{version}</strong>
</a>
</h2>
</div>
<div class="panel-collapse collapse"
id="version-{iterator.index}" role="tabpanel" data-group-version="{version}">
<div class="panel-body" role="tablist" aria-multiselectable="false">
<f:for each="{versionArray}" as="fileArray" iteration="fileIterator">
<f:render partial="Upgrade/UpgradeDocs/PanelItem" arguments="{id:'file-{iterator.index}-{fileIterator.index}', fileArray:fileArray, version:version}"/>
</f:for>
</div>
</div>
<f:for each="{upgradeDocsVersions}" as="version" iteration="iterator">
<div class="panel panel-default panel-version t3js-version-changes" data-version="{version}">
<div class="panel-heading" role="tab" id="heading-{iterator.index}">
<h2 class="panel-title">
<a href="#version-{iterator.index}"
class="collapsed" data-toggle="collapse"
aria-expanded="false"
aria-controls="#version-{iterator.index}"
>
<span class="caret"></span>
Version: <strong>{version}</strong>
<span class="pull-right t3js-panel-loading"><core:icon identifier="spinner-circle" size="small" /></span>
</a>
</h2>
</div>
</f:if>
<div class="panel-collapse collapse"
id="version-{iterator.index}" role="tabpanel" data-group-version="{version}">
<div class="panel-body t3js-changelog-list" role="tablist" aria-multiselectable="false"></div>
</div>
</div>
</f:for>
<div class="panel panel-default panel-version">
......@@ -71,12 +68,8 @@
</a>
</h2>
</div>
<div class="collapse" id="collapseRead" role="tabpanel">
<div class="panel-body panel-body-read" role="tablist" aria-multiselectable="false">
<f:for each="{upgradeDocsReadFiles}" as="fileArray" iteration="fileIterator">
<f:render partial="Upgrade/UpgradeDocs/PanelItem" arguments="{id:'read-{fileIterator.index}', fileArray:fileArray, version:fileArray.section, read:'true'}"/>
</f:for>
</div>
<div class="collapse" id="collapseRead" role="tabpanel" data-group-version="read">
<div class="panel-body panel-body-read t3js-changelog-list" role="tablist" aria-multiselectable="false"></div>
</div>
</div>
......@@ -89,13 +82,8 @@
</a>
</h2>
</div>
<div class="collapse" id="collapseNotAffected" role="tabpanel">
<div class="panel-body panel-body-not-affected" role="tablist" aria-multiselectable="false">
<f:for each="{upgradeDocsNotAffectedFiles}" as="fileArray" iteration="fileIterator">
<f:render partial="Upgrade/UpgradeDocs/PanelItem" arguments="{id:'scanner-{fileIterator.index}', fileArray:fileArray, version:fileArray.section, read:'true'}"
/>
</f:for>
</div>
<div class="collapse" id="collapseNotAffected" role="tabpanel" data-group-version="notAffected">
<div class="panel-body panel-body-not-affected t3js-changelog-list" role="tablist" aria-multiselectable="false"></div>
</div>
</div>
</div>
......
......@@ -33,6 +33,8 @@ define([
selectorRestFileItem: '.upgrade_analysis_item_to_filter',
selectorFulltextSearch: '.t3js-upgradeDocs-fulltext-search',
selectorChosenField: '.t3js-upgradeDocs-chosen-select',
selectorChangeLogsForVersionContainer: '.t3js-version-changes',
selectorChangeLogsForVersion: '.t3js-changelog-list',
chosenField: null,
fulltextSearchField: null,
......@@ -40,7 +42,7 @@ define([
initialize: function(currentModal) {
var self = this;
this.currentModal = currentModal;
var isInIframe = (window.location != window.parent.location) ? true : false;
var isInIframe = (window.location !== window.parent.location);
if (isInIframe) {
top.require(['TYPO3/CMS/Install/chosen.jquery.min'], function () {
self.getContent();
......@@ -75,41 +77,69 @@ define([
success: function(data) {
if (data.success === true && data.html !== 'undefined' && data.html.length > 0) {
modalContent.empty().append(data.html);
modalContent.find('[data-toggle="tooltip"]').tooltip({container: 'body'});
self.chosenField = modalContent.find(self.selectorChosenField);
self.fulltextSearchField = modalContent.find(self.selectorFulltextSearch);
self.fulltextSearchField.clearable().focus();
self.initializeFullTextSearch();
self.initializeChosenSelector();
self.chosenField.on('change', function() {
self.combinedFilterSearch();
});
self.fulltextSearchField.on('keyup', function() {
self.combinedFilterSearch();
});
self.renderTags();
} else {
Notification.error('Something went wrong');
self.loadChangelogs();
}
},
error: function(xhr) {
Router.handleAjaxError(xhr);
}
});
},
initializeChosenSelector: function() {
loadChangelogs: function() {
var self = this;
var tagString = '';
$(self.currentModal.find(this.selectorRestFileItem)).each(function() {
tagString += $(this).data('item-tags') + ',';
var promises = [];
this.currentModal.find(this.selectorChangeLogsForVersionContainer).each(function(index, el) {
var $request = $.ajax({
url: Router.getUrl('upgradeDocsGetChangelogForVersion'),
cache: false,
data: {
install: {
version: el.dataset.version
}
},
success: function(data) {
if (data.success === true) {
var $panelGroup = $(el);
var $container = $panelGroup.find(self.selectorChangeLogsForVersion);
$container.html(data.html);
self.renderTags($container);
self.moveNotRelevantDocuments($container);
// Remove loading spinner form panel
$panelGroup.find('.t3js-panel-loading').remove();
} else {
Notification.error('Something went wrong');
}
},
error: function(xhr) {
Router.handleAjaxError(xhr);
}
});
promises.push($request);
});
var tagArray = this.trimExplodeAndUnique(',', tagString).sort(function(a, b) {
// Sort case-insensitive by name
return a.toLowerCase().localeCompare(b.toLowerCase());
$.when.apply($, promises).done(function () {
self.fulltextSearchField.prop('disabled', false);
self.appendItemsToChosenSelector();
});
$.each(tagArray, function(i, tag) {
self.chosenField.append('<option>' + tag + '</option>');
},
initializeFullTextSearch: function() {
var self = this;
this.fulltextSearchField = this.currentModal.find(this.selectorFulltextSearch);
this.fulltextSearchField.clearable().focus();
this.initializeChosenSelector();
this.fulltextSearchField.on('keyup', function() {
self.combinedFilterSearch();
});
},
initializeChosenSelector: function() {
var self = this;
this.chosenField = this.currentModal.find(this.selectorModalBody).find(this.selectorChosenField);
var config = {
'.chosen-select': {width: "100%", placeholder_text_multiple: "tags"},
'.chosen-select-deselect': {allow_single_deselect: true},
......@@ -118,8 +148,30 @@ define([
'.chosen-select-width': {width: "100%"}
};
for (var selector in config) {
self.currentModal.find(selector).chosen(config[selector]);
this.currentModal.find(selector).chosen(config[selector]);
}
this.chosenField.on('change', function() {
self.combinedFilterSearch();
});
},
/**
* Appends tags to the chosen selector
*/
appendItemsToChosenSelector: function() {
var self = this;
var tagString = '';
$(this.currentModal.find(this.selectorRestFileItem)).each(function() {
tagString += $(this).data('item-tags') + ',';
});
var tagArray = this.trimExplodeAndUnique(',', tagString).sort(function(a, b) {
// Sort case-insensitive by name
return a.toLowerCase().localeCompare(b.toLowerCase());
});
this.chosenField.prop('disabled', false);
$.each(tagArray, function(i, tag) {
self.chosenField.append('<option>' + tag + '</option>');