[FEATURE] Add placeholder processor in Yaml import 79/63079/6
authorSilvia Bigler <sbigler@snowflake.ch>
Thu, 13 Feb 2020 12:19:09 +0000 (13:19 +0100)
committerBenni Mack <benni@typo3.org>
Fri, 21 Feb 2020 18:11:51 +0000 (19:11 +0100)
Rework placeholder processing to allow custom processors

Resolves: #90267
Releases: master
Change-Id: If884062c09a770d5eabbc9436e1a23360290f7e2
Reviewed-on: https://review.typo3.org/c/Packages/TYPO3.CMS/+/63079
Tested-by: TYPO3com <noreply@typo3.com>
Tested-by: Susanne Moog <look@susi.dev>
Tested-by: Benni Mack <benni@typo3.org>
Reviewed-by: Susanne Moog <look@susi.dev>
Reviewed-by: Daniel Goerz <daniel.goerz@posteo.de>
Reviewed-by: Benni Mack <benni@typo3.org>
typo3/sysext/core/Classes/Configuration/Loader/YamlFileLoader.php
typo3/sysext/core/Classes/Configuration/Processor/Placeholder/EnvVariableProcessor.php [new file with mode: 0644]
typo3/sysext/core/Classes/Configuration/Processor/Placeholder/PlaceholderProcessorInterface.php [new file with mode: 0644]
typo3/sysext/core/Classes/Configuration/Processor/Placeholder/ValueFromReferenceArrayProcessor.php [new file with mode: 0644]
typo3/sysext/core/Classes/Configuration/Processor/PlaceholderProcessorList.php [new file with mode: 0644]
typo3/sysext/core/Configuration/DefaultConfiguration.php
typo3/sysext/core/Documentation/Changelog/master/Feature-90267-CustomPlaceholderProcessingInSiteConfig.rst [new file with mode: 0644]
typo3/sysext/core/Tests/Unit/Configuration/Loader/YamlFileLoaderTest.php

index cd8770f..c2b65b3 100644 (file)
@@ -15,6 +15,7 @@ namespace TYPO3\CMS\Core\Configuration\Loader;
  */
 
 use Symfony\Component\Yaml\Yaml;
+use TYPO3\CMS\Core\Configuration\Processor\PlaceholderProcessorList;
 use TYPO3\CMS\Core\Utility\ArrayUtility;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 use TYPO3\CMS\Core\Utility\PathUtility;
@@ -136,27 +137,6 @@ class YamlFileLoader
     }
 
     /**
-     * Return value from environment variable
-     *
-     * Environment variables may only contain word characters and underscores (a-zA-Z0-9_)
-     * to be compatible to shell environments.
-     *
-     * @param string $value
-     * @return string
-     */
-    protected function getValueFromEnv(string $value): string
-    {
-        $matches = [];
-        preg_match_all('/%env\([\'"]?(\w+)[\'"]?\)%/', $value, $matches);
-        $envVars = array_combine($matches[0], $matches[1]);
-        foreach ($envVars as $substring => $envVarName) {
-            $envVar = getenv($envVarName);
-            $value = $envVar ? str_replace($substring, $envVar, $value) : $value;
-        }
-        return $value;
-    }
-
-    /**
      * Checks for the special "imports" key on the main level of a file,
      * which calls "load" recursively.
      * @param array $content
@@ -189,64 +169,101 @@ class YamlFileLoader
     protected function processPlaceholders(array $content, array $referenceArray): array
     {
         foreach ($content as $k => $v) {
-            if ($this->isEnvPlaceholder($v)) {
-                $content[$k] = $this->getValueFromEnv($v);
-            } elseif ($this->isPlaceholder($v)) {
-                $content[$k] = $this->getValueFromReferenceArray($v, $referenceArray);
-            } elseif (is_array($v)) {
+            if (is_array($v)) {
                 $content[$k] = $this->processPlaceholders($v, $referenceArray);
+            } elseif ($this->containsPlaceholder($v)) {
+                $content[$k] = $this->processPlaceholderLine($v, $referenceArray);
             }
         }
         return $content;
     }
 
     /**
-     * Returns the value for a placeholder as fetched from the referenceArray
-     *
-     * @param string $placeholder the string to search for
-     * @param array $referenceArray the main configuration array where to look up the data
-     *
-     * @return array|mixed|string
+     * @param string $line
+     * @param array $referenceArray
+     * @return mixed
      */
-    protected function getValueFromReferenceArray(string $placeholder, array $referenceArray)
+    protected function processPlaceholderLine(string $line, array $referenceArray)
     {
-        $pointer = trim($placeholder, '%');
-        $parts = explode('.', $pointer);
-        $referenceData = $referenceArray;
-        foreach ($parts as $part) {
-            if (isset($referenceData[$part])) {
-                $referenceData = $referenceData[$part];
+        $parts = $this->getParts($line);
+        foreach ($parts as $partKey => $part) {
+            $result = $this->processSinglePlaceholder($partKey, $part, $referenceArray);
+            // Replace whole content if placeholder is the only thing in this line
+            if ($line === $partKey) {
+                $line = $result;
+            } elseif (is_string($result) || is_numeric($result)) {
+                $line = str_replace($partKey, $result, $line);
             } else {
-                // return unsubstituted placeholder
-                return $placeholder;
+                throw new \UnexpectedValueException(
+                    'Placeholder can not be substituted if result is not string or numeric',
+                    1581502783
+                );
+            }
+            if ($result !== $partKey && $this->containsPlaceholder($line)) {
+                $line = $this->processPlaceholderLine($line, $referenceArray);
             }
         }
-        if ($this->isPlaceholder($referenceData)) {
-            $referenceData = $this->getValueFromReferenceArray($referenceData, $referenceArray);
+        return $line;
+    }
+
+    /**
+     * @param string $placeholder
+     * @param string $value
+     * @param array $referenceArray
+     * @return mixed
+     */
+    protected function processSinglePlaceholder(string $placeholder, string $value, array $referenceArray)
+    {
+        $processorList = GeneralUtility::makeInstance(
+            PlaceholderProcessorList::class,
+            $GLOBALS['TYPO3_CONF_VARS']['SYS']['yamlLoader']['placeholderProcessors']
+        );
+        foreach ($processorList->compile() as $processor) {
+            if ($processor->canProcess($placeholder, $referenceArray)) {
+                try {
+                    $result = $processor->process($value, $referenceArray);
+                } catch (\UnexpectedValueException $e) {
+                    $result = $placeholder;
+                }
+                if (is_array($result)) {
+                    $result = $this->processPlaceholders($result, $referenceArray);
+                }
+                break;
+            }
         }
-        return $referenceData;
+        return $result ?? $placeholder;
     }
 
     /**
-     * Checks if a value is a string and begins and ends with %...%
-     *
-     * @param mixed $value the probe to check for
-     * @return bool
+     * @param string $placeholders
+     * @return array
      */
-    protected function isPlaceholder($value): bool
+    protected function getParts(string $placeholders): array
     {
-        return is_string($value) && strpos($value, '%') === 0 && substr($value, -1) === '%';
+        // find occurences of placeholders like %some()% and %array.access%.
+        // Only find the innermost ones, so we can nest them.
+        preg_match_all(
+            '/%[^(%]+?\([\'"]?([^(]*?)[\'"]?\)%|%([^%()]*?)%/',
+            $placeholders,
+            $parts,
+            PREG_UNMATCHED_AS_NULL
+        );
+        $matches = array_filter(
+            array_merge($parts[1], $parts[2])
+        );
+        return array_combine($parts[0], $matches);
     }
 
     /**
-     * Checks if a value is a string and contains an env placeholder
+     * Finds possible placeholders.
+     * May find false positives for complexer structures, but they will be sorted later on.
      *
-     * @param mixed $value the probe to check for
+     * @param $value
      * @return bool
      */
-    protected function isEnvPlaceholder($value): bool
+    protected function containsPlaceholder($value): bool
     {
-        return is_string($value) && (strpos($value, '%env(') !== false);
+        return is_string($value) && substr_count($value, '%') >= 2;
     }
 
     protected function hasFlag(int $flag): bool
diff --git a/typo3/sysext/core/Classes/Configuration/Processor/Placeholder/EnvVariableProcessor.php b/typo3/sysext/core/Classes/Configuration/Processor/Placeholder/EnvVariableProcessor.php
new file mode 100644 (file)
index 0000000..f95f131
--- /dev/null
@@ -0,0 +1,44 @@
+<?php
+
+namespace TYPO3\CMS\Core\Configuration\Processor\Placeholder;
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+/**
+ * Return value from environment variable
+ *
+ * Environment variables may only contain word characters and underscores (a-zA-Z0-9_)
+ * to be compatible to shell environments.
+ */
+class EnvVariableProcessor implements PlaceholderProcessorInterface
+{
+    public function canProcess(string $placeholder, array $referenceArray): bool
+    {
+        return is_string($placeholder) && (strpos($placeholder, '%env(') !== false);
+    }
+
+    /**
+     * @param string $value
+     * @param array|null $referenceArray
+     * @return mixed|string
+     */
+    public function process(string $value, array $referenceArray)
+    {
+        $envVar = getenv($value);
+        if (!$envVar) {
+            throw new \UnexpectedValueException('Value not found', 1581501124);
+        }
+        return $envVar;
+    }
+}
diff --git a/typo3/sysext/core/Classes/Configuration/Processor/Placeholder/PlaceholderProcessorInterface.php b/typo3/sysext/core/Classes/Configuration/Processor/Placeholder/PlaceholderProcessorInterface.php
new file mode 100644 (file)
index 0000000..ba0e53b
--- /dev/null
@@ -0,0 +1,33 @@
+<?php
+
+namespace TYPO3\CMS\Core\Configuration\Processor\Placeholder;
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+interface PlaceholderProcessorInterface
+{
+    /**
+     * @param string $placeholder
+     * @param array $referenceArray
+     * @return bool
+     */
+    public function canProcess(string $placeholder, array $referenceArray): bool;
+
+    /**
+     * @param string $value
+     * @param array $referenceArray
+     * @return mixed
+     */
+    public function process(string $value, array $referenceArray);
+}
diff --git a/typo3/sysext/core/Classes/Configuration/Processor/Placeholder/ValueFromReferenceArrayProcessor.php b/typo3/sysext/core/Classes/Configuration/Processor/Placeholder/ValueFromReferenceArrayProcessor.php
new file mode 100644 (file)
index 0000000..0e95e4f
--- /dev/null
@@ -0,0 +1,56 @@
+<?php
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\Configuration\Processor\Placeholder;
+
+/**
+ * Returns the value for a placeholder as fetched from the referenceArray
+ *
+ * Class ValueFromReferenceArrayProcessor
+ */
+class ValueFromReferenceArrayProcessor implements PlaceholderProcessorInterface
+{
+    /**
+     * @param string $placeholder
+     * @param array $referenceArray
+     * @return bool
+     */
+    public function canProcess(string $placeholder, array $referenceArray): bool
+    {
+        return strpos($placeholder, '(') === false;
+    }
+
+    /**
+     * Returns the value for a placeholder as fetched from the referenceArray
+     *
+     * @param string $value the string to search for
+     * @param array $referenceArray the main configuration array where to look up the data
+     *
+     * @return array|mixed|string
+     */
+    public function process(string $value, array $referenceArray)
+    {
+        $parts = explode('.', $value);
+        $referenceData = $referenceArray;
+        foreach ($parts as $part) {
+            if (isset($referenceData[$part])) {
+                $referenceData = $referenceData[$part];
+            } else {
+                // return unsubstituted placeholder
+                throw new \UnexpectedValueException('Value not found', 1581501216);
+            }
+        }
+        return $referenceData;
+    }
+}
diff --git a/typo3/sysext/core/Classes/Configuration/Processor/PlaceholderProcessorList.php b/typo3/sysext/core/Classes/Configuration/Processor/PlaceholderProcessorList.php
new file mode 100644 (file)
index 0000000..474c7ee
--- /dev/null
@@ -0,0 +1,62 @@
+<?php
+
+namespace TYPO3\CMS\Core\Configuration\Processor;
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+use TYPO3\CMS\Core\Configuration\Processor\Placeholder\PlaceholderProcessorInterface;
+use TYPO3\CMS\Core\Service\DependencyOrderingService;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
+/**
+ * Orders and returns given PlaceholderProcessors
+ */
+class PlaceholderProcessorList
+{
+    /**
+     * @var PlaceholderProcessorInterface[]
+     */
+    protected $processors;
+
+    public function __construct($processorList = [])
+    {
+        $this->processors = $processorList;
+    }
+
+    /**
+     * @return PlaceholderProcessorInterface[]
+     */
+    public function compile(): array
+    {
+        $processors = [];
+        $orderingService = GeneralUtility::makeInstance(DependencyOrderingService::class);
+        $orderedProcessors = $orderingService->orderByDependencies($this->processors, 'before', 'after');
+
+        foreach ($orderedProcessors as $processorClassName => $providerConfig) {
+            if (isset($providerConfig['disabled']) && $providerConfig['disabled'] === true) {
+                continue;
+            }
+
+            $processor = GeneralUtility::makeInstance($processorClassName);
+            if (!$processor instanceof PlaceholderProcessorInterface) {
+                throw new \UnexpectedValueException(
+                    'Placeholder processor ' . $processorClassName . ' must implement PlaceholderProcessorInterface',
+                    1581343410
+                );
+            }
+            $processors[] = $processor;
+        }
+        return $processors;
+    }
+}
index 5366f2b..5bc9ffd 100644 (file)
@@ -1055,6 +1055,16 @@ return [
                 ],
             ],
         ],
+        'yamlLoader' => [
+            'placeholderProcessors' => [
+                \TYPO3\CMS\Core\Configuration\Processor\Placeholder\EnvVariableProcessor::class => [],
+                \TYPO3\CMS\Core\Configuration\Processor\Placeholder\ValueFromReferenceArrayProcessor::class => [
+                    'after' => [
+                        \TYPO3\CMS\Core\Configuration\Processor\Placeholder\EnvVariableProcessor::class
+                    ]
+                ]
+            ]
+        ]
     ],
     'EXT' => [ // Options related to the Extension Management
         'allowGlobalInstall' => false,
diff --git a/typo3/sysext/core/Documentation/Changelog/master/Feature-90267-CustomPlaceholderProcessingInSiteConfig.rst b/typo3/sysext/core/Documentation/Changelog/master/Feature-90267-CustomPlaceholderProcessingInSiteConfig.rst
new file mode 100644 (file)
index 0000000..bde8a0e
--- /dev/null
@@ -0,0 +1,77 @@
+.. include:: ../../Includes.txt
+
+==============================================================
+Feature: #90267 - Custom placeholder processing in site config
+==============================================================
+
+See :issue:`90267`
+
+Description
+===========
+
+The Yaml import for site configuration was changed to allow custom placeholder processors.
+
+
+Impact
+======
+
+It is now possible to register a new placeholder processor:
+
+.. code-block:: php
+
+   $GLOBALS['TYPO3_CONF_VARS']['SYS']['yamlLoader']['placeholderProcessors'][\Vendor\MyExtension\PlaceholderProcessor\CustomPlaceholderProcessor::class] = [];
+
+There are some options available to sort or disable placeholder processors if necessary.
+
+.. code-block:: php
+
+   $GLOBALS['TYPO3_CONF_VARS']['SYS']['yamlLoader']['placeholderProcessors'][\Vendor\MyExtension\PlaceholderProcessor\CustomPlaceholderProcessor::class] = [
+      'before' => [
+         \TYPO3\CMS\Core\Configuration\Processor\Placeholder\ValueFromReferenceArrayProcessor::class
+      ],
+      'after' => [
+         \TYPO3\CMS\Core\Configuration\Processor\Placeholder\EnvVariableProcessor::class
+      ],
+      'disabled' => false
+   ];
+
+New placeholder processors must implement the `\TYPO3\CMS\Core\Configuration\Processor\Placeholder\PlaceholderProcessorInterface`
+
+Placeholders look mostly like functions.
+So an implementation may look like the following
+
+.. code-block:: php
+
+   class ExamplePlaceholderProcessor implements PlaceholderProcessorInterface
+   {
+      public function canProcess(string $placeholder, array $referenceArray): bool
+      {
+         return strpos($placeholder, '%example(') !== false;
+      }
+
+      public function process(string $value, array $referenceArray)
+      {
+         // do some processing
+         $result = $this->getValue($value);
+
+         // Throw this exception if the placeholder can't be substituted
+         if (!$envVar) {
+            throw new \UnexpectedValueException('Value not found', 1581596096);
+         }
+         return $result;
+      }
+   }
+
+
+This may be used like the following in the site config
+
+.. code-block:: yaml
+
+   someVariable: '%example(somevalue)%'
+   anotherVariable: 'inline::%example(anotherValue)%::placeholder'
+
+If a new processor returns a string or number, it may also be used inline as above.
+If it returns an array, it can not be used inline since the whole content will be replaced with the new value.
+
+
+.. index:: Backend, PHP-API, ext:core
index 45439fb..ddabb51 100644 (file)
@@ -129,6 +129,44 @@ options:
     - option1
     - option2
 betterthanbefore: \'%firstset.myinitialversion%\'
+muchbetterthanbefore: \'some::%options.0%::option\'
+';
+
+        $expected = [
+            'firstset' => [
+                'myinitialversion' => 13
+            ],
+            'options' => [
+                'option1',
+                'option2'
+            ],
+            'betterthanbefore' => 13,
+            'muchbetterthanbefore' => 'some::option1::option'
+        ];
+
+        // Accessible mock to $subject since getFileContents calls GeneralUtility methods
+        $subject = $this->getAccessibleMock(YamlFileLoader::class, ['getFileContents', 'getStreamlinedFileName']);
+        $subject->expects(self::once())->method('getStreamlinedFileName')->with($fileName)->willReturn($fileName);
+        $subject->expects(self::once())->method('getFileContents')->with($fileName)->willReturn($fileContents);
+        $output = $subject->load($fileName);
+        self::assertSame($expected, $output);
+    }
+
+    /**
+     * Method checking for nested placeholders
+     * @test
+     */
+    public function loadWithNestedPlaceholders(): void
+    {
+        $fileName = 'Berta.yml';
+        $fileContents = '
+
+firstset:
+  myinitialversion: 13
+options:
+    - option1
+    - option2
+betterthanbefore: \'%env(foo)%\'
 ';
 
         $expected = [
@@ -146,7 +184,10 @@ betterthanbefore: \'%firstset.myinitialversion%\'
         $subject = $this->getAccessibleMock(YamlFileLoader::class, ['getFileContents', 'getStreamlinedFileName']);
         $subject->expects(self::once())->method('getStreamlinedFileName')->with($fileName)->willReturn($fileName);
         $subject->expects(self::once())->method('getFileContents')->with($fileName)->willReturn($fileContents);
+
+        putenv('foo=%firstset.myinitialversion%');
         $output = $subject->load($fileName);
+        putenv('foo=');
         self::assertSame($expected, $output);
     }
 
@@ -227,6 +268,11 @@ betterthanbefore: \'%firstset.myinitialversion%\'
                 'carl: \'%env(foo)%::%env(bar)%::%env(baz)%\'',
                 ['carl' => '%env(foo)%::%env(bar)%::%env(baz)%']
             ],
+            'nested env variables' => [
+                ['foo=bar', 'bar=heinz'],
+                'carl: \'%env(%env(foo)%)%\'',
+                ['carl' => 'heinz']
+            ],
         ];
     }
 
@@ -328,11 +374,11 @@ betterthanbefore: \'%env(mynonexistingenv)%\'
             ],
             'invalid placeholder with two % but not at the end' => [
                 '%cool%again',
-                false
+                true
             ],
             'invalid placeholder with two % but not at the beginning nor end' => [
                 'did%you%know',
-                false
+                true
             ],
             'valid placeholder with just numbers' => [
                 '%13%',
@@ -352,10 +398,10 @@ betterthanbefore: \'%env(mynonexistingenv)%\'
      * @param bool $expected
      * @skip
      */
-    public function isPlaceholderTest($placeholderValue, bool $expected)
+    public function containsPlaceholderTest($placeholderValue, bool $expected)
     {
         $subject = $this->getAccessibleMock(YamlFileLoader::class, ['dummy']);
-        $output = $subject->_call('isPlaceholder', $placeholderValue);
+        $output = $subject->_call('containsPlaceholder', $placeholderValue);
         self::assertSame($expected, $output);
     }
 }