Commit f66149f7 authored by Silvia Bigler's avatar Silvia Bigler Committed by Benni Mack
Browse files

[FEATURE] Add placeholder processor in Yaml import

Rework placeholder processing to allow custom processors

Resolves: #90267
Releases: master
Change-Id: If884062c09a770d5eabbc9436e1a23360290f7e2
Reviewed-on: https://review.typo3.org/c/Packages/TYPO3.CMS/+/63079


Tested-by: default avatarTYPO3com <noreply@typo3.com>
Tested-by: Susanne Moog's avatarSusanne Moog <look@susi.dev>
Tested-by: Benni Mack's avatarBenni Mack <benni@typo3.org>
Reviewed-by: Susanne Moog's avatarSusanne Moog <look@susi.dev>
Reviewed-by: Daniel Goerz's avatarDaniel Goerz <daniel.goerz@posteo.de>
Reviewed-by: Benni Mack's avatarBenni Mack <benni@typo3.org>
parent 6e4bc758
......@@ -15,6 +15,7 @@ namespace TYPO3\CMS\Core\Configuration\Loader;
*/
use Symfony\Component\Yaml\Yaml;
use TYPO3\CMS\Core\Configuration\Processor\PlaceholderProcessorList;
use TYPO3\CMS\Core\Utility\ArrayUtility;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Core\Utility\PathUtility;
......@@ -135,27 +136,6 @@ class YamlFileLoader
return $streamlinedFileName;
}
/**
* Return value from environment variable
*
* Environment variables may only contain word characters and underscores (a-zA-Z0-9_)
* to be compatible to shell environments.
*
* @param string $value
* @return string
*/
protected function getValueFromEnv(string $value): string
{
$matches = [];
preg_match_all('/%env\([\'"]?(\w+)[\'"]?\)%/', $value, $matches);
$envVars = array_combine($matches[0], $matches[1]);
foreach ($envVars as $substring => $envVarName) {
$envVar = getenv($envVarName);
$value = $envVar ? str_replace($substring, $envVar, $value) : $value;
}
return $value;
}
/**
* Checks for the special "imports" key on the main level of a file,
* which calls "load" recursively.
......@@ -189,64 +169,101 @@ class YamlFileLoader
protected function processPlaceholders(array $content, array $referenceArray): array
{
foreach ($content as $k => $v) {
if ($this->isEnvPlaceholder($v)) {
$content[$k] = $this->getValueFromEnv($v);
} elseif ($this->isPlaceholder($v)) {
$content[$k] = $this->getValueFromReferenceArray($v, $referenceArray);
} elseif (is_array($v)) {
if (is_array($v)) {
$content[$k] = $this->processPlaceholders($v, $referenceArray);
} elseif ($this->containsPlaceholder($v)) {
$content[$k] = $this->processPlaceholderLine($v, $referenceArray);
}
}
return $content;
}
/**
* Returns the value for a placeholder as fetched from the referenceArray
*
* @param string $placeholder the string to search for
* @param array $referenceArray the main configuration array where to look up the data
*
* @return array|mixed|string
* @param string $line
* @param array $referenceArray
* @return mixed
*/
protected function getValueFromReferenceArray(string $placeholder, array $referenceArray)
protected function processPlaceholderLine(string $line, array $referenceArray)
{
$pointer = trim($placeholder, '%');
$parts = explode('.', $pointer);
$referenceData = $referenceArray;
foreach ($parts as $part) {
if (isset($referenceData[$part])) {
$referenceData = $referenceData[$part];
$parts = $this->getParts($line);
foreach ($parts as $partKey => $part) {
$result = $this->processSinglePlaceholder($partKey, $part, $referenceArray);
// Replace whole content if placeholder is the only thing in this line
if ($line === $partKey) {
$line = $result;
} elseif (is_string($result) || is_numeric($result)) {
$line = str_replace($partKey, $result, $line);
} else {
// return unsubstituted placeholder
return $placeholder;
throw new \UnexpectedValueException(
'Placeholder can not be substituted if result is not string or numeric',
1581502783
);
}
if ($result !== $partKey && $this->containsPlaceholder($line)) {
$line = $this->processPlaceholderLine($line, $referenceArray);
}
}
if ($this->isPlaceholder($referenceData)) {
$referenceData = $this->getValueFromReferenceArray($referenceData, $referenceArray);
return $line;
}
/**
* @param string $placeholder
* @param string $value
* @param array $referenceArray
* @return mixed
*/
protected function processSinglePlaceholder(string $placeholder, string $value, array $referenceArray)
{
$processorList = GeneralUtility::makeInstance(
PlaceholderProcessorList::class,
$GLOBALS['TYPO3_CONF_VARS']['SYS']['yamlLoader']['placeholderProcessors']
);
foreach ($processorList->compile() as $processor) {
if ($processor->canProcess($placeholder, $referenceArray)) {
try {
$result = $processor->process($value, $referenceArray);
} catch (\UnexpectedValueException $e) {
$result = $placeholder;
}
if (is_array($result)) {
$result = $this->processPlaceholders($result, $referenceArray);
}
break;
}
}
return $referenceData;
return $result ?? $placeholder;
}
/**
* Checks if a value is a string and begins and ends with %...%
*
* @param mixed $value the probe to check for
* @return bool
* @param string $placeholders
* @return array
*/
protected function isPlaceholder($value): bool
protected function getParts(string $placeholders): array
{
return is_string($value) && strpos($value, '%') === 0 && substr($value, -1) === '%';
// find occurences of placeholders like %some()% and %array.access%.
// Only find the innermost ones, so we can nest them.
preg_match_all(
'/%[^(%]+?\([\'"]?([^(]*?)[\'"]?\)%|%([^%()]*?)%/',
$placeholders,
$parts,
PREG_UNMATCHED_AS_NULL
);
$matches = array_filter(
array_merge($parts[1], $parts[2])
);
return array_combine($parts[0], $matches);
}
/**
* Checks if a value is a string and contains an env placeholder
* Finds possible placeholders.
* May find false positives for complexer structures, but they will be sorted later on.
*
* @param mixed $value the probe to check for
* @param $value
* @return bool
*/
protected function isEnvPlaceholder($value): bool
protected function containsPlaceholder($value): bool
{
return is_string($value) && (strpos($value, '%env(') !== false);
return is_string($value) && substr_count($value, '%') >= 2;
}
protected function hasFlag(int $flag): bool
......
<?php
namespace TYPO3\CMS\Core\Configuration\Processor\Placeholder;
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
/**
* Return value from environment variable
*
* Environment variables may only contain word characters and underscores (a-zA-Z0-9_)
* to be compatible to shell environments.
*/
class EnvVariableProcessor implements PlaceholderProcessorInterface
{
public function canProcess(string $placeholder, array $referenceArray): bool
{
return is_string($placeholder) && (strpos($placeholder, '%env(') !== false);
}
/**
* @param string $value
* @param array|null $referenceArray
* @return mixed|string
*/
public function process(string $value, array $referenceArray)
{
$envVar = getenv($value);
if (!$envVar) {
throw new \UnexpectedValueException('Value not found', 1581501124);
}
return $envVar;
}
}
<?php
namespace TYPO3\CMS\Core\Configuration\Processor\Placeholder;
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
interface PlaceholderProcessorInterface
{
/**
* @param string $placeholder
* @param array $referenceArray
* @return bool
*/
public function canProcess(string $placeholder, array $referenceArray): bool;
/**
* @param string $value
* @param array $referenceArray
* @return mixed
*/
public function process(string $value, array $referenceArray);
}
<?php
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
namespace TYPO3\CMS\Core\Configuration\Processor\Placeholder;
/**
* Returns the value for a placeholder as fetched from the referenceArray
*
* Class ValueFromReferenceArrayProcessor
*/
class ValueFromReferenceArrayProcessor implements PlaceholderProcessorInterface
{
/**
* @param string $placeholder
* @param array $referenceArray
* @return bool
*/
public function canProcess(string $placeholder, array $referenceArray): bool
{
return strpos($placeholder, '(') === false;
}
/**
* Returns the value for a placeholder as fetched from the referenceArray
*
* @param string $value the string to search for
* @param array $referenceArray the main configuration array where to look up the data
*
* @return array|mixed|string
*/
public function process(string $value, array $referenceArray)
{
$parts = explode('.', $value);
$referenceData = $referenceArray;
foreach ($parts as $part) {
if (isset($referenceData[$part])) {
$referenceData = $referenceData[$part];
} else {
// return unsubstituted placeholder
throw new \UnexpectedValueException('Value not found', 1581501216);
}
}
return $referenceData;
}
}
<?php
namespace TYPO3\CMS\Core\Configuration\Processor;
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
use TYPO3\CMS\Core\Configuration\Processor\Placeholder\PlaceholderProcessorInterface;
use TYPO3\CMS\Core\Service\DependencyOrderingService;
use TYPO3\CMS\Core\Utility\GeneralUtility;
/**
* Orders and returns given PlaceholderProcessors
*/
class PlaceholderProcessorList
{
/**
* @var PlaceholderProcessorInterface[]
*/
protected $processors;
public function __construct($processorList = [])
{
$this->processors = $processorList;
}
/**
* @return PlaceholderProcessorInterface[]
*/
public function compile(): array
{
$processors = [];
$orderingService = GeneralUtility::makeInstance(DependencyOrderingService::class);
$orderedProcessors = $orderingService->orderByDependencies($this->processors, 'before', 'after');
foreach ($orderedProcessors as $processorClassName => $providerConfig) {
if (isset($providerConfig['disabled']) && $providerConfig['disabled'] === true) {
continue;
}
$processor = GeneralUtility::makeInstance($processorClassName);
if (!$processor instanceof PlaceholderProcessorInterface) {
throw new \UnexpectedValueException(
'Placeholder processor ' . $processorClassName . ' must implement PlaceholderProcessorInterface',
1581343410
);
}
$processors[] = $processor;
}
return $processors;
}
}
......@@ -1055,6 +1055,16 @@ return [
],
],
],
'yamlLoader' => [
'placeholderProcessors' => [
\TYPO3\CMS\Core\Configuration\Processor\Placeholder\EnvVariableProcessor::class => [],
\TYPO3\CMS\Core\Configuration\Processor\Placeholder\ValueFromReferenceArrayProcessor::class => [
'after' => [
\TYPO3\CMS\Core\Configuration\Processor\Placeholder\EnvVariableProcessor::class
]
]
]
]
],
'EXT' => [ // Options related to the Extension Management
'allowGlobalInstall' => false,
......
.. include:: ../../Includes.txt
==============================================================
Feature: #90267 - Custom placeholder processing in site config
==============================================================
See :issue:`90267`
Description
===========
The Yaml import for site configuration was changed to allow custom placeholder processors.
Impact
======
It is now possible to register a new placeholder processor:
.. code-block:: php
$GLOBALS['TYPO3_CONF_VARS']['SYS']['yamlLoader']['placeholderProcessors'][\Vendor\MyExtension\PlaceholderProcessor\CustomPlaceholderProcessor::class] = [];
There are some options available to sort or disable placeholder processors if necessary.
.. code-block:: php
$GLOBALS['TYPO3_CONF_VARS']['SYS']['yamlLoader']['placeholderProcessors'][\Vendor\MyExtension\PlaceholderProcessor\CustomPlaceholderProcessor::class] = [
'before' => [
\TYPO3\CMS\Core\Configuration\Processor\Placeholder\ValueFromReferenceArrayProcessor::class
],
'after' => [
\TYPO3\CMS\Core\Configuration\Processor\Placeholder\EnvVariableProcessor::class
],
'disabled' => false
];
New placeholder processors must implement the `\TYPO3\CMS\Core\Configuration\Processor\Placeholder\PlaceholderProcessorInterface`
Placeholders look mostly like functions.
So an implementation may look like the following
.. code-block:: php
class ExamplePlaceholderProcessor implements PlaceholderProcessorInterface
{
public function canProcess(string $placeholder, array $referenceArray): bool
{
return strpos($placeholder, '%example(') !== false;
}
public function process(string $value, array $referenceArray)
{
// do some processing
$result = $this->getValue($value);
// Throw this exception if the placeholder can't be substituted
if (!$envVar) {
throw new \UnexpectedValueException('Value not found', 1581596096);
}
return $result;
}
}
This may be used like the following in the site config
.. code-block:: yaml
someVariable: '%example(somevalue)%'
anotherVariable: 'inline::%example(anotherValue)%::placeholder'
If a new processor returns a string or number, it may also be used inline as above.
If it returns an array, it can not be used inline since the whole content will be replaced with the new value.
.. index:: Backend, PHP-API, ext:core
......@@ -129,6 +129,44 @@ options:
- option1
- option2
betterthanbefore: \'%firstset.myinitialversion%\'
muchbetterthanbefore: \'some::%options.0%::option\'
';
$expected = [
'firstset' => [
'myinitialversion' => 13
],
'options' => [
'option1',
'option2'
],
'betterthanbefore' => 13,
'muchbetterthanbefore' => 'some::option1::option'
];
// Accessible mock to $subject since getFileContents calls GeneralUtility methods
$subject = $this->getAccessibleMock(YamlFileLoader::class, ['getFileContents', 'getStreamlinedFileName']);
$subject->expects(self::once())->method('getStreamlinedFileName')->with($fileName)->willReturn($fileName);
$subject->expects(self::once())->method('getFileContents')->with($fileName)->willReturn($fileContents);
$output = $subject->load($fileName);
self::assertSame($expected, $output);
}
/**
* Method checking for nested placeholders
* @test
*/
public function loadWithNestedPlaceholders(): void
{
$fileName = 'Berta.yml';
$fileContents = '
firstset:
myinitialversion: 13
options:
- option1
- option2
betterthanbefore: \'%env(foo)%\'
';
$expected = [
......@@ -146,7 +184,10 @@ betterthanbefore: \'%firstset.myinitialversion%\'
$subject = $this->getAccessibleMock(YamlFileLoader::class, ['getFileContents', 'getStreamlinedFileName']);
$subject->expects(self::once())->method('getStreamlinedFileName')->with($fileName)->willReturn($fileName);
$subject->expects(self::once())->method('getFileContents')->with($fileName)->willReturn($fileContents);
putenv('foo=%firstset.myinitialversion%');
$output = $subject->load($fileName);
putenv('foo=');
self::assertSame($expected, $output);
}
......@@ -227,6 +268,11 @@ betterthanbefore: \'%firstset.myinitialversion%\'
'carl: \'%env(foo)%::%env(bar)%::%env(baz)%\'',
['carl' => '%env(foo)%::%env(bar)%::%env(baz)%']
],
'nested env variables' => [
['foo=bar', 'bar=heinz'],
'carl: \'%env(%env(foo)%)%\'',
['carl' => 'heinz']
],
];
}
......@@ -328,11 +374,11 @@ betterthanbefore: \'%env(mynonexistingenv)%\'
],
'invalid placeholder with two % but not at the end' => [
'%cool%again',
false
true
],
'invalid placeholder with two % but not at the beginning nor end' => [
'did%you%know',
false
true
],
'valid placeholder with just numbers' => [
'%13%',
......@@ -352,10 +398,10 @@ betterthanbefore: \'%env(mynonexistingenv)%\'
* @param bool $expected
* @skip
*/
public function isPlaceholderTest($placeholderValue, bool $expected)
public function containsPlaceholderTest($placeholderValue, bool $expected)
{
$subject = $this->getAccessibleMock(YamlFileLoader::class, ['dummy']);
$output = $subject->_call('isPlaceholder', $placeholderValue);
$output = $subject->_call('containsPlaceholder', $placeholderValue);
self::assertSame($expected, $output);
}
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment