Commit ce34dbd7 authored by Jochen Roth's avatar Jochen Roth Committed by Oliver Bartsch
Browse files

[TASK] Introduce composer manifest checks

The extension manager now provides a new module,
which allows an integrator to display all available
extensions with composer deficits, like missing
composer.json or missing extension-key.

The new module informs about the deficit and
automatically generates a valid composer.json.
proposal. In case no composer.json exists, the
corresponding ext_emconf is sent to a new TER
endpoint (https://extensions.typo3.org/composerize).
This endpoint then generates a new composer.json
proposal by resolving all dependencies.

Furthermore, a new report is added to EXT:reports
which also informs about such extensions by directly
linking to the new EM module.

This helps especially in non-composer-mode installations
to ease the upgrade path for future TYPO3 versions which
(hopefully) will rely on composer.json only for e.g.
PackageStates.php.

Resolves: #93931
Releases: master, 10.4
Change-Id: I1230363d5d03e03bff39e7070faf4e331532a292
Reviewed-on: https://review.typo3.org/c/Packages/TYPO3.CMS/+/68778


Tested-by: core-ci's avatarcore-ci <typo3@b13.com>
Tested-by: Benni Mack's avatarBenni Mack <benni@typo3.org>
Tested-by: Oliver Bartsch's avatarOliver Bartsch <bo@cedev.de>
Tested-by: Jochen's avatarJochen <rothjochen@gmail.com>
Reviewed-by: Benni Mack's avatarBenni Mack <benni@typo3.org>
Reviewed-by: Helmut Hummel's avatarHelmut Hummel <typo3@helhum.io>
Reviewed-by: Oliver Bartsch's avatarOliver Bartsch <bo@cedev.de>
Reviewed-by: Jochen's avatarJochen <rothjochen@gmail.com>
parent c0e99719
...@@ -116,7 +116,10 @@ table { ...@@ -116,7 +116,10 @@ table {
.btn-default { .btn-default {
@include button-variant($gray-200, $gray-500, $gray-800, $gray-400, $gray-600); @include button-variant($gray-200, $gray-500, $gray-800, $gray-400, $gray-600);
}
.btn-default,
.btn-warning {
padding: 0.375rem; padding: 0.375rem;
} }
......
...@@ -15,6 +15,7 @@ import InfoWindow = require('TYPO3/CMS/Backend/InfoWindow'); ...@@ -15,6 +15,7 @@ import InfoWindow = require('TYPO3/CMS/Backend/InfoWindow');
import RegularEvent = require('TYPO3/CMS/Core/Event/RegularEvent'); import RegularEvent = require('TYPO3/CMS/Core/Event/RegularEvent');
import shortcutMenu = require('TYPO3/CMS/Backend/Toolbar/ShortcutMenu'); import shortcutMenu = require('TYPO3/CMS/Backend/Toolbar/ShortcutMenu');
import windowManager = require('TYPO3/CMS/Backend/WindowManager'); import windowManager = require('TYPO3/CMS/Backend/WindowManager');
import moduleMenuApp = require('TYPO3/CMS/Backend/ModuleMenu');
import documentService = require('TYPO3/CMS/Core/DocumentService'); import documentService = require('TYPO3/CMS/Core/DocumentService');
import Utility = require('TYPO3/CMS/Backend/Utility'); import Utility = require('TYPO3/CMS/Backend/Utility');
...@@ -69,6 +70,7 @@ class ActionDispatcher { ...@@ -69,6 +70,7 @@ class ActionDispatcher {
'TYPO3.InfoWindow.showItem': InfoWindow.showItem.bind(null), 'TYPO3.InfoWindow.showItem': InfoWindow.showItem.bind(null),
'TYPO3.ShortcutMenu.createShortcut': shortcutMenu.createShortcut.bind(shortcutMenu), 'TYPO3.ShortcutMenu.createShortcut': shortcutMenu.createShortcut.bind(shortcutMenu),
'TYPO3.WindowManager.localOpen': windowManager.localOpen.bind(windowManager), 'TYPO3.WindowManager.localOpen': windowManager.localOpen.bind(windowManager),
'TYPO3.ModuleMenu.showModule': moduleMenuApp.App.showModule.bind(moduleMenuApp.App),
}; };
} }
......
...@@ -10,4 +10,4 @@ ...@@ -10,4 +10,4 @@
* *
* The TYPO3 project - inspiring people to share! * The TYPO3 project - inspiring people to share!
*/ */
define(["require","exports","TYPO3/CMS/Backend/InfoWindow","TYPO3/CMS/Core/Event/RegularEvent","TYPO3/CMS/Backend/Toolbar/ShortcutMenu","TYPO3/CMS/Backend/WindowManager","TYPO3/CMS/Core/DocumentService","TYPO3/CMS/Backend/Utility"],(function(e,t,n,s,a,r,i,c){"use strict";class o{constructor(){this.delegates={},this.createDelegates(),i.ready().then(()=>this.registerEvents())}static resolveArguments(e){if(e.dataset.dispatchArgs){const t=e.dataset.dispatchArgs.replace(/&quot;/g,'"'),n=JSON.parse(t);return n instanceof Array?c.trimItems(n):null}if(e.dataset.dispatchArgsList){const t=e.dataset.dispatchArgsList.split(",");return c.trimItems(t)}return null}static enrichItems(e,t,n){return e.map(e=>e instanceof Object&&e.$event?e.$target?n:e.$event?t:void 0:e)}createDelegates(){this.delegates={"TYPO3.InfoWindow.showItem":n.showItem.bind(null),"TYPO3.ShortcutMenu.createShortcut":a.createShortcut.bind(a),"TYPO3.WindowManager.localOpen":r.localOpen.bind(r)}}registerEvents(){new s("click",this.handleClickEvent.bind(this)).delegateTo(document,"[data-dispatch-action]")}handleClickEvent(e,t){e.preventDefault(),this.delegateTo(t)}delegateTo(e){const t=e.dataset.dispatchAction,n=o.resolveArguments(e);this.delegates[t]&&this.delegates[t].apply(null,n||[])}}return new o})); define(["require","exports","TYPO3/CMS/Backend/InfoWindow","TYPO3/CMS/Core/Event/RegularEvent","TYPO3/CMS/Backend/Toolbar/ShortcutMenu","TYPO3/CMS/Backend/WindowManager","TYPO3/CMS/Backend/ModuleMenu","TYPO3/CMS/Core/DocumentService","TYPO3/CMS/Backend/Utility"],(function(e,t,n,s,a,r,i,c,o){"use strict";class d{constructor(){this.delegates={},this.createDelegates(),c.ready().then(()=>this.registerEvents())}static resolveArguments(e){if(e.dataset.dispatchArgs){const t=e.dataset.dispatchArgs.replace(/&quot;/g,'"'),n=JSON.parse(t);return n instanceof Array?o.trimItems(n):null}if(e.dataset.dispatchArgsList){const t=e.dataset.dispatchArgsList.split(",");return o.trimItems(t)}return null}static enrichItems(e,t,n){return e.map(e=>e instanceof Object&&e.$event?e.$target?n:e.$event?t:void 0:e)}createDelegates(){this.delegates={"TYPO3.InfoWindow.showItem":n.showItem.bind(null),"TYPO3.ShortcutMenu.createShortcut":a.createShortcut.bind(a),"TYPO3.WindowManager.localOpen":r.localOpen.bind(r),"TYPO3.ModuleMenu.showModule":i.App.showModule.bind(i.App)}}registerEvents(){new s("click",this.handleClickEvent.bind(this)).delegateTo(document,"[data-dispatch-action]")}handleClickEvent(e,t){e.preventDefault(),this.delegateTo(t)}delegateTo(e){const t=e.dataset.dispatchAction,n=d.resolveArguments(e);this.delegates[t]&&this.delegates[t].apply(null,n||[])}}return new d}));
\ No newline at end of file \ No newline at end of file
<?php
declare(strict_types=1);
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
namespace TYPO3\CMS\Core\Package;
use Symfony\Component\Finder\Finder;
use TYPO3\CMS\Core\Core\Environment;
use TYPO3\CMS\Core\Utility\GeneralUtility;
/**
* Detects extensions with composer deficits, e.g. missing
* composer.json file or missing extension-key property.
*/
class ComposerDeficitDetector
{
public const EXTENSION_COMPOSER_MANIFEST_VALID = 0;
public const EXTENSION_COMPOSER_MANIFEST_MISSING = 1;
public const EXTENSION_KEY_MISSING = 2;
/**
* Get all extensions with composer deficit
*/
public function getExtensionsWithComposerDeficit(): array
{
$finder = Finder::create()->directories()->depth(0)->in(Environment::getExtensionsPath());
$extensionsWithDeficit = [];
if ($finder->hasResults()) {
foreach ($finder as $extensionFolder) {
$extensionKey = $extensionFolder->getFilename();
try {
$extensionComposerDeficit = $this->checkExtensionComposerDeficit($extensionKey);
} catch (\InvalidArgumentException $e) {
// Skip invalid extensions
continue;
}
if ($extensionComposerDeficit !== self::EXTENSION_COMPOSER_MANIFEST_VALID) {
$extensionsWithDeficit[$extensionKey] = $extensionComposerDeficit;
}
}
}
return $extensionsWithDeficit;
}
/**
* Check an extension key for composer deficits like invalid or missing composer.json
*/
public function checkExtensionComposerDeficit(string $extensionKey): int
{
if (!$this->isValidExtensionKey($extensionKey)) {
throw new \InvalidArgumentException('Extension key ' . $extensionKey . ' is not valid.', 1619446378);
}
$composerManifestPath = Environment::getExtensionsPath() . '/' . $extensionKey . '/composer.json';
if (!file_exists($composerManifestPath) || !($composerManifest = file_get_contents($composerManifestPath))) {
return self::EXTENSION_COMPOSER_MANIFEST_MISSING;
}
$composerManifest = json_decode($composerManifest, true) ?? [];
if (!is_array($composerManifest) || $composerManifest === []) {
// Treat empty or invalid composer.json as missing
return self::EXTENSION_COMPOSER_MANIFEST_MISSING;
}
return empty($composerManifest['extra']['typo3/cms']['extension-key'])
? self::EXTENSION_KEY_MISSING
: self::EXTENSION_COMPOSER_MANIFEST_VALID;
}
protected function isValidExtensionKey(string $extensionKey): bool
{
return preg_match('/^[0-9a-z._\-]+$/i', $extensionKey)
&& GeneralUtility::isAllowedAbsPath(Environment::getExtensionsPath() . '/' . $extensionKey);
}
}
.. include:: ../../Includes.txt
================================================================
Important: #93931 - Validation of Exensions' composer.json files
================================================================
See :issue:`93931`
Description
===========
Future TYPO3 versions will require extensions to have a valid
`composer.json` file as a replacement for `ext_emconf.php`.
This description file is used to define dependencies and the
loading order of extensions within TYPO3.
In order to support site administrators by creating valid
composer.json files for their extensions, the Extension manager
now lists all affected extensions with details about the necessary
adaptations. Site administrators can also use the new proposal
functionality, which suggests a possible and valid composer.json
file for those extensions by accessing TYPO3.org (TER). TYPO3.org
is used to resolve dependencies to extensions, available in the TER.
You can also check your current installation for such extensions
in the reports module.
Further information on the transition phase and examples
of valid composer.json files for TYPO3 Extensions can be found on
https://extensions.typo3.org/help/composer-support
.. index:: Backend, ext:extensionmanager
...@@ -65,6 +65,11 @@ class AbstractModuleController extends AbstractController ...@@ -65,6 +65,11 @@ class AbstractModuleController extends AbstractController
'controller' => 'List', 'controller' => 'List',
'action' => 'index', 'action' => 'index',
'label' => $this->translate('installedExtensions') 'label' => $this->translate('installedExtensions')
],
'extensionComposerStatus' => [
'controller' => 'ExtensionComposerStatus',
'action' => 'list',
'label' => $this->translate('extensionComposerStatus')
] ]
]; ];
......
<?php
declare(strict_types=1);
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
namespace TYPO3\CMS\Extensionmanager\Controller;
use TYPO3\CMS\Backend\Form\FormResultCompiler;
use TYPO3\CMS\Backend\Form\NodeFactory;
use TYPO3\CMS\Core\Core\Environment;
use TYPO3\CMS\Core\Imaging\Icon;
use TYPO3\CMS\Core\Package\ComposerDeficitDetector;
use TYPO3\CMS\Core\Utility\ExtensionManagementUtility;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Core\Utility\MathUtility;
use TYPO3\CMS\Core\Utility\PathUtility;
use TYPO3\CMS\Extbase\Mvc\View\ViewInterface;
use TYPO3\CMS\Extensionmanager\Service\ComposerManifestProposalGenerator;
use TYPO3\CMS\Extensionmanager\Utility\ListUtility;
/**
* Provide information about extensions' composer status
*
* @internal This class is a specific controller implementation and is not considered part of the Public TYPO3 API.
*/
class ExtensionComposerStatusController extends AbstractModuleController
{
/**
* @var ComposerDeficitDetector
*/
protected $composerDeficitDetector;
/**
* @var ComposerDeficitDetector
*/
protected $composerManifestProposalGenerator;
/**
* @var NodeFactory
*/
protected $nodeFactory;
/**
* @var ListUtility
*/
protected $listUtility;
/**
* @var string
*/
protected $returnUrl = '';
public function __construct(
ComposerDeficitDetector $composerDeficitDetector,
ComposerManifestProposalGenerator $composerManifestProposalGenerator,
NodeFactory $nodeFactory,
ListUtility $listUtility
) {
$this->composerDeficitDetector = $composerDeficitDetector;
$this->composerManifestProposalGenerator = $composerManifestProposalGenerator;
$this->nodeFactory = $nodeFactory;
$this->listUtility = $listUtility;
}
protected function initializeAction(): void
{
parent::initializeAction();
if ($this->request->hasArgument('returnUrl')) {
$this->returnUrl = GeneralUtility::sanitizeLocalUrl(
(string)$this->request->getArgument('returnUrl')
);
}
}
protected function initializeView(ViewInterface $view): void
{
parent::initializeView($view);
$this->registerDocHeaderButtons();
}
public function listAction(): void
{
$extensions = [];
$basePackagePath = Environment::getExtensionsPath() . '/';
$detailLinkReturnUrl = $this->uriBuilder->reset()->uriFor('list', array_filter(['returnUrl' => $this->returnUrl]));
foreach ($this->composerDeficitDetector->getExtensionsWithComposerDeficit() as $extensionKey => $deficit) {
$extensionPath = $basePackagePath . $extensionKey . '/';
$extensions[$extensionKey] = [
'deficit' => $deficit,
'packagePath' => $extensionPath,
'icon' => $this->getExtensionIcon($extensionPath),
'detailLink' => $this->uriBuilder->reset()->uriFor('detail', [
'extensionKey' => $extensionKey,
'returnUrl' => $detailLinkReturnUrl
])
];
}
ksort($extensions);
$this->view->assign('extensions', $this->listUtility->enrichExtensionsWithEmConfInformation($extensions));
$this->generateMenu();
}
public function detailAction(string $extensionKey): void
{
if ($extensionKey === '') {
$this->redirect('list');
}
$deficit = $this->composerDeficitDetector->checkExtensionComposerDeficit($extensionKey);
$this->view->assignMultiple([
'extensionKey' => $extensionKey,
'deficit' => $deficit
]);
if ($deficit !== ComposerDeficitDetector::EXTENSION_COMPOSER_MANIFEST_VALID) {
$this->view->assign('composerManifestMarkup', $this->getComposerManifestMarkup($extensionKey));
}
}
protected function getComposerManifestMarkup(string $extensionKey): string
{
$formResultCompiler = GeneralUtility::makeInstance(FormResultCompiler::class);
$composerManifest = $this->composerManifestProposalGenerator->getComposerManifestProposal($extensionKey);
if ($composerManifest === '') {
return '';
}
$rows = MathUtility::forceIntegerInRange(count(explode(LF, $composerManifest)), 1, PHP_INT_MAX);
$fakeFieldTca = [
'renderType' => 't3editor',
'tableName' => $extensionKey,
'fieldName' => 'composer.json',
'effectivePid' => 0,
'parameterArray' => [
'itemFormElName' => 'composerManifest-' . $extensionKey,
'itemFormElValue' => $composerManifest,
'fieldConf' => [
'config' => [
'readOnly' => true,
'rows' => ++$rows,
'codeMirrorFirstLineNumber' => 1,
]
]
]
];
$resultArray = $this->nodeFactory->create($fakeFieldTca)->render();
$formResultCompiler->mergeResult($resultArray);
$formResultCompiler->addCssFiles();
$formResultCompiler->printNeededJSFunctions();
return $resultArray['html'];
}
protected function registerDocHeaderButtons(): void
{
$buttonBar = $this->view->getModuleTemplate()->getDocHeaderComponent()->getButtonBar();
if ($this->returnUrl !== '') {
$buttonBar->addButton(
$buttonBar
->makeLinkButton()
->setHref($this->returnUrl)
->setClasses('typo3-goBack')
->setTitle($this->getLanguageService()->sL('LLL:EXT:core/Resources/Private/Language/locallang_core.xlf:labels.goBack'))
->setIcon($this->view->getModuleTemplate()->getIconFactory()->getIcon('actions-view-go-back', Icon::SIZE_SMALL))
);
}
}
protected function getExtensionIcon(string $extensionPath): string
{
$icon = ExtensionManagementUtility::getExtensionIcon($extensionPath);
return $icon ? PathUtility::getAbsoluteWebPath($extensionPath . $icon) : '';
}
protected function getLanguageService()
{
return $GLOBALS['LANG'];
}
}
<?php
declare(strict_types=1);
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
namespace TYPO3\CMS\Extensionmanager\Report;
use Psr\Http\Message\ServerRequestInterface;
use TYPO3\CMS\Backend\Routing\UriBuilder;
use TYPO3\CMS\Core\Localization\LanguageService;
use TYPO3\CMS\Core\Package\ComposerDeficitDetector;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Reports\RequestAwareStatusProviderInterface;
use TYPO3\CMS\Reports\Status as ReportStatus;
/**
* Extension composer status reports
*
* @internal This class is a specific EXT:reports implementation and is not part of the Public TYPO3 API.
*/
class ExtensionComposerStatus implements RequestAwareStatusProviderInterface
{
/**
* @var ComposerDeficitDetector
*/
protected $composerDeficitDetector;
/**
* @var UriBuilder
*/
protected $uriBuilder;
public function __construct(ComposerDeficitDetector $composerDeficitDetector = null, UriBuilder $uriBuilder = null)
{
$this->composerDeficitDetector = $composerDeficitDetector ?? GeneralUtility::makeInstance(ComposerDeficitDetector::class);
$this->uriBuilder = $uriBuilder ?? GeneralUtility::makeInstance(UriBuilder::class);
}
public function getStatus(ServerRequestInterface $request = null): array
{
$status = [];
$request = $request ?? $this->getRequest();
$extensionsWithComposerDeficit = $this->composerDeficitDetector->getExtensionsWithComposerDeficit();
$languageService = $this->getLanguageService();
$languageService->includeLLFile('EXT:extensionmanager/Resources/Private/Language/locallang.xlf');
$labelPrefix = 'report.status.composerManifest.';
$deficits = [
ComposerDeficitDetector::EXTENSION_COMPOSER_MANIFEST_MISSING => 'composerJsonMissing',
ComposerDeficitDetector::EXTENSION_KEY_MISSING => 'extensionKeyMissing'
];
$dispatchAction = 'TYPO3.ModuleMenu.showModule';
$dispatchArgs = [
'tools_ExtensionmanagerExtensionmanager',
'&' . http_build_query([
'tx_extensionmanager_tools_extensionmanagerextensionmanager' => [
'action' => 'list',
'controller' => 'ExtensionComposerStatus',
'returnUrl' => $request->getAttribute('normalizedParams')->getRequestUri()
]
])
];
foreach ($deficits as $key => $deficit) {
$message = '';
$extensionsToReport = count(array_filter($extensionsWithComposerDeficit, static function ($extensionDeficit) use ($key) {
return $extensionDeficit === $key;
}));
if ($extensionsToReport) {
$linkTitle = $languageService->getLL($labelPrefix . 'update');
$linkAttributes = GeneralUtility::implodeAttributes([
'href' => '#',
'title' => $linkTitle,
'class' => 'text-decoration-underline',
// relies on module 'TYPO3/CMS/Backend/ActionDispatcher'
'data-dispatch-action' => $dispatchAction,
'data-dispatch-args' => GeneralUtility::jsonEncodeForHtmlAttribute($dispatchArgs),
], true);
$message = sprintf($languageService->getLL($labelPrefix . $deficit . '.message'), $extensionsToReport)
. '<br /><a ' . $linkAttributes . '>' . htmlspecialchars($linkTitle) . '</a>';
}
$status[] = GeneralUtility::makeInstance(
ReportStatus::class,
$this->getLanguageService()->getLL($labelPrefix . $deficit),
$this->getLanguageService()->getLL($extensionsToReport ? 'status_checkFailed' : 'status_none'),
$message,
$extensionsToReport ? ReportStatus::WARNING : ReportStatus::OK
);
}
return $status;
}
protected function getLanguageService(): LanguageService
{
return $GLOBALS['LANG'];
}
protected function getRequest(): ServerRequestInterface
{
return $GLOBALS['TYPO3_REQUEST'];
}
}
<?php
declare(strict_types=1);
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
namespace TYPO3\CMS\Extensionmanager\Service;
use GuzzleHttp\Exception\TransferException;
use TYPO3\CMS\Core\Core\Environment;
use TYPO3\CMS\Core\Http\RequestFactory;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Extensionmanager\Utility\EmConfUtility;
/**
* Service for generating composer manifest proposals
*/
class ComposerManifestProposalGenerator
{
private const TER_COMPOSER_ENDPOINT = 'https://extensions.typo3.org/composerize';
/**
* @var RequestFactory
*/
protected $requestFactory;
/**
* @var EmConfUtility
*/
protected $emConfUtility;
public function __construct(RequestFactory $requestFactory, EmConfUtility $emConfUtility)
{
$this->requestFactory = $requestFactory;
$this->emConfUtility = $emConfUtility;
}
/**
* Return the generated composer manifest content
*
* @param string $extensionKey
* @return string
*/
public function getComposerManifestProposal(string $extensionKey): string
{
if (!$this->isValidExtensionKey($extensionKey)) {
throw new \InvalidArgumentException('Extension key ' . $extensionKey . ' is not valid.', 1619446379);
}