[FEATURE] Allow environment variables in site config 58/58358/13
authorSusanne Moog <susanne.moog@typo3.org>
Thu, 20 Sep 2018 19:19:59 +0000 (21:19 +0200)
committerFrank Naegler <frank.naegler@typo3.org>
Fri, 28 Sep 2018 08:24:11 +0000 (10:24 +0200)
To enable easy configuration of environment specific
site setting, usage of environment variables in site
configuration yaml has been introduced.

The TYPO3 Core Yaml loader is now able to resolve
variables from environment variables.

Resolves: #86409
Releases: master
Change-Id: Ic1e32d231aa7e92b3feb4ed4c31bed72520d71fb
Reviewed-on: https://review.typo3.org/58358
Tested-by: TYPO3com <no-reply@typo3.com>
Reviewed-by: Jan Helke <typo3@helke.de>
Tested-by: Jan Helke <typo3@helke.de>
Reviewed-by: Frank Naegler <frank.naegler@typo3.org>
Tested-by: Frank Naegler <frank.naegler@typo3.org>
typo3/sysext/backend/Classes/Form/FormDataProvider/SiteDatabaseEditRow.php
typo3/sysext/backend/Tests/Unit/Form/FormDataProvider/SiteDatabaseEditRowTest.php
typo3/sysext/core/Classes/Configuration/Loader/YamlFileLoader.php
typo3/sysext/core/Classes/Configuration/SiteConfiguration.php
typo3/sysext/core/Documentation/Changelog/master/Feature-86409-AllowUsageOfEnvironmentVariablesInSiteConfiguration.rst [new file with mode: 0644]
typo3/sysext/core/Tests/Unit/Configuration/Loader/YamlFileLoaderTest.php

index 8b01683..6f12603 100644 (file)
@@ -16,6 +16,8 @@ namespace TYPO3\CMS\Backend\Form\FormDataProvider;
  */
 
 use TYPO3\CMS\Backend\Form\FormDataProviderInterface;
+use TYPO3\CMS\Core\Configuration\SiteConfiguration;
+use TYPO3\CMS\Core\Core\Environment;
 use TYPO3\CMS\Core\Site\SiteFinder;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 
@@ -42,13 +44,13 @@ class SiteDatabaseEditRow implements FormDataProviderInterface
         $tableName = $result['tableName'];
         $siteFinder = GeneralUtility::makeInstance(SiteFinder::class);
         if ($tableName === 'site') {
-            $siteConfigurationForPageUid = (int)$result['vanillaUid'];
-            $rowData = $siteFinder->getSiteByRootPageId($siteConfigurationForPageUid)->getConfiguration();
+            $rootPageId = (int)$result['vanillaUid'];
+            $rowData = $this->getRawConfigurationForSiteWithRootPageId($siteFinder, $rootPageId);
             $result['databaseRow']['uid'] = $rowData['rootPageId'];
             $result['databaseRow']['identifier'] = $result['customData']['siteIdentifier'];
         } elseif (in_array($tableName, ['site_errorhandling', 'site_language', 'site_route', 'site_base_variant'], true)) {
-            $siteConfigurationForPageUid = (int)($result['inlineTopMostParentUid'] ?? $result['inlineParentUid']);
-            $rowData = $siteFinder->getSiteByRootPageId($siteConfigurationForPageUid)->getConfiguration();
+            $rootPageId = (int)($result['inlineTopMostParentUid'] ?? $result['inlineParentUid']);
+            $rowData = $this->getRawConfigurationForSiteWithRootPageId($siteFinder, $rootPageId);
             $parentFieldName = $result['inlineParentFieldName'];
             if (!isset($rowData[$parentFieldName])) {
                 throw new \RuntimeException('Field "' . $parentFieldName . '" not found', 1520886092);
@@ -69,4 +71,20 @@ class SiteDatabaseEditRow implements FormDataProviderInterface
         $result['databaseRow']['pid'] = 0;
         return $result;
     }
+
+    /**
+     * @param SiteFinder $siteFinder
+     * @param int $rootPageId
+     * @return array
+     */
+    protected function getRawConfigurationForSiteWithRootPageId(SiteFinder $siteFinder, int $rootPageId): array
+    {
+        $site = $siteFinder->getSiteByRootPageId($rootPageId);
+        $siteConfiguration = GeneralUtility::makeInstance(
+            SiteConfiguration::class,
+            Environment::getConfigPath() . '/sites'
+        );
+        // load config as it is stored on disk (without replacements)
+        return $siteConfiguration->load($site->getIdentifier());
+    }
 }
index 2cc100c..5b3d62b 100644 (file)
@@ -17,6 +17,9 @@ namespace TYPO3\CMS\Backend\Tests\Unit\Form\FormDataProvider;
  */
 
 use TYPO3\CMS\Backend\Form\FormDataProvider\SiteDatabaseEditRow;
+use TYPO3\CMS\Core\Configuration\SiteConfiguration;
+use TYPO3\CMS\Core\Core\ApplicationContext;
+use TYPO3\CMS\Core\Core\Environment;
 use TYPO3\CMS\Core\Site\Entity\Site;
 use TYPO3\CMS\Core\Site\SiteFinder;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
@@ -27,6 +30,22 @@ use TYPO3\TestingFramework\Core\Unit\UnitTestCase;
  */
 class SiteDatabaseEditRowTest extends UnitTestCase
 {
+    public function setUp()
+    {
+        $this->backupEnvironment = true;
+        parent::setUp();
+        Environment::initialize(
+            $this->prophesize(ApplicationContext::class)->reveal(),
+            true,
+            false,
+            '',
+            '',
+            '',
+            '',
+            '',
+            ''
+        );
+    }
     /**
      * @test
      */
@@ -93,7 +112,10 @@ class SiteDatabaseEditRowTest extends UnitTestCase
         GeneralUtility::addInstance(SiteFinder::class, $siteFinderProphecy->reveal());
         $siteProphecy = $this->prophesize(Site::class);
         $siteFinderProphecy->getSiteByRootPageId(23)->willReturn($siteProphecy->reveal());
-        $siteProphecy->getConfiguration()->willReturn($rowData);
+        $siteProphecy->getIdentifier()->willReturn('testident');
+        $siteConfiguration = $this->prophesize(SiteConfiguration::class);
+        $siteConfiguration->load('testident')->willReturn($rowData);
+        GeneralUtility::addInstance(SiteConfiguration::class, $siteConfiguration->reveal());
 
         $expected = $input;
         $expected['databaseRow'] = [
@@ -126,7 +148,10 @@ class SiteDatabaseEditRowTest extends UnitTestCase
         GeneralUtility::addInstance(SiteFinder::class, $siteFinderProphecy->reveal());
         $siteProphecy = $this->prophesize(Site::class);
         $siteFinderProphecy->getSiteByRootPageId(5)->willReturn($siteProphecy->reveal());
-        $siteProphecy->getConfiguration()->willReturn($rowData);
+        $siteProphecy->getIdentifier()->willReturn('testident');
+        $siteConfiguration = $this->prophesize(SiteConfiguration::class);
+        $siteConfiguration->load('testident')->willReturn($rowData);
+        GeneralUtility::addInstance(SiteConfiguration::class, $siteConfiguration->reveal());
 
         $this->expectException(\RuntimeException::class);
         $this->expectExceptionCode(1520886092);
@@ -152,7 +177,10 @@ class SiteDatabaseEditRowTest extends UnitTestCase
         GeneralUtility::addInstance(SiteFinder::class, $siteFinderProphecy->reveal());
         $siteProphecy = $this->prophesize(Site::class);
         $siteFinderProphecy->getSiteByRootPageId(5)->willReturn($siteProphecy->reveal());
-        $siteProphecy->getConfiguration()->willReturn($rowData);
+        $siteProphecy->getIdentifier()->willReturn('testident');
+        $siteConfiguration = $this->prophesize(SiteConfiguration::class);
+        $siteConfiguration->load('testident')->willReturn($rowData);
+        GeneralUtility::addInstance(SiteConfiguration::class, $siteConfiguration->reveal());
 
         $this->expectException(\RuntimeException::class);
         $this->expectExceptionCode(1520886092);
@@ -182,7 +210,10 @@ class SiteDatabaseEditRowTest extends UnitTestCase
         GeneralUtility::addInstance(SiteFinder::class, $siteFinderProphecy->reveal());
         $siteProphecy = $this->prophesize(Site::class);
         $siteFinderProphecy->getSiteByRootPageId(5)->willReturn($siteProphecy->reveal());
-        $siteProphecy->getConfiguration()->willReturn($rowData);
+        $siteProphecy->getIdentifier()->willReturn('testident');
+        $siteConfiguration = $this->prophesize(SiteConfiguration::class);
+        $siteConfiguration->load('testident')->willReturn($rowData);
+        GeneralUtility::addInstance(SiteConfiguration::class, $siteConfiguration->reveal());
 
         $expected = $input;
         $expected['databaseRow'] = [
index 5491d5c..95a1b5f 100644 (file)
@@ -30,18 +30,22 @@ use TYPO3\CMS\Core\Utility\GeneralUtility;
  *
  * - Special placeholder values set via %optionA.suboptionB% replace the value with the named path of the configuration
  *   The placeholders will act as a full replacement of this value.
+ *
+ * - Environment placeholder values set via %env(option)% will be replaced by env variables of the same name
  */
 class YamlFileLoader
 {
+    public const PROCESS_PLACEHOLDERS = 1;
+    public const PROCESS_IMPORTS = 2;
 
     /**
      * Loads and parses a YAML file, and returns an array with the found data
      *
      * @param string $fileName either relative to TYPO3's base project folder or prefixed with EXT:...
+     * @param int $flags Flags to configure behaviour of the loader: see public PROCESS_ constants above
      * @return array the configuration as array
-     * @throws \RuntimeException when the file is empty or is of invalid format
      */
-    public function load(string $fileName): array
+    public function load(string $fileName, int $flags = self::PROCESS_PLACEHOLDERS | self::PROCESS_IMPORTS): array
     {
         $content = $this->getFileContents($fileName);
         $content = Yaml::parse($content);
@@ -50,10 +54,13 @@ class YamlFileLoader
             throw new \RuntimeException('YAML file "' . $fileName . '" could not be parsed into valid syntax, probably empty?', 1497332874);
         }
 
-        $content = $this->processImports($content);
-
-        // Check for "%" placeholders
-        $content = $this->processPlaceholders($content, $content);
+        if (($flags & self::PROCESS_IMPORTS) === self::PROCESS_IMPORTS) {
+            $content = $this->processImports($content);
+        }
+        if (($flags & self::PROCESS_PLACEHOLDERS) === self::PROCESS_PLACEHOLDERS) {
+            // Check for "%" placeholders
+            $content = $this->processPlaceholders($content, $content);
+        }
 
         return $content;
     }
@@ -76,6 +83,25 @@ class YamlFileLoader
     }
 
     /**
+     * Return value from environment variable
+     *
+     * Environment variables may only contain word characters and underscores (a-zA-Z0-9_)
+     * to be compatible to shell environments.
+     *
+     * @param string $value
+     * @return string
+     */
+    protected function getValueFromEnv(string $value): string
+    {
+        $matched = preg_match('/%env\([\'"]?(\w+)[\'"]?\)%/', $value, $matches);
+        if ($matched === 1) {
+            $envVar = getenv($matches[1]);
+            $value = $envVar ? str_replace($matches[0], $envVar, $value) : $value;
+        }
+        return $value;
+    }
+
+    /**
      * Checks for the special "imports" key on the main level of a file,
      * which calls "load" recursively.
      * @param array $content
@@ -107,7 +133,9 @@ class YamlFileLoader
     protected function processPlaceholders(array $content, array $referenceArray): array
     {
         foreach ($content as $k => $v) {
-            if ($this->isPlaceholder($v)) {
+            if ($this->isEnvPlaceholder($v)) {
+                $content[$k] = $this->getValueFromEnv($v);
+            } elseif ($this->isPlaceholder($v)) {
                 $content[$k] = $this->getValueFromReferenceArray($v, $referenceArray);
             } elseif (is_array($v)) {
                 $content[$k] = $this->processPlaceholders($v, $referenceArray);
@@ -155,6 +183,17 @@ class YamlFileLoader
     }
 
     /**
+     * Checks if a value is a string and contains an env placeholder
+     *
+     * @param mixed $value the probe to check for
+     * @return bool
+     */
+    protected function isEnvPlaceholder($value): bool
+    {
+        return is_string($value) && (strpos($value, '%env(') !== false);
+    }
+
+    /**
      * Same as array_replace_recursive except that when in simple arrays (= YAML lists), the entries are
      * appended (array_merge)
      *
index ea77cde..61f5b65 100644 (file)
@@ -125,6 +125,23 @@ class SiteConfiguration
     }
 
     /**
+     * Load plain configuration
+     * This method should only be used in case the original configuration as it exists in the file should be loaded,
+     * for example for writing / editing configuration.
+     *
+     * All read related actions should be performed on the site entity.
+     *
+     * @param string $siteIdentifier
+     * @return array
+     */
+    public function load(string $siteIdentifier): array
+    {
+        $fileName = $this->configPath . '/' . $siteIdentifier . '/' . $this->configFileName;
+        $loader = GeneralUtility::makeInstance(YamlFileLoader::class);
+        return $loader->load(GeneralUtility::fixWindowsFilePath($fileName), YamlFileLoader::PROCESS_IMPORTS);
+    }
+
+    /**
      * Add or update a site configuration
      *
      * @param string $siteIdentifier
diff --git a/typo3/sysext/core/Documentation/Changelog/master/Feature-86409-AllowUsageOfEnvironmentVariablesInSiteConfiguration.rst b/typo3/sysext/core/Documentation/Changelog/master/Feature-86409-AllowUsageOfEnvironmentVariablesInSiteConfiguration.rst
new file mode 100644 (file)
index 0000000..6365951
--- /dev/null
@@ -0,0 +1,38 @@
+.. include:: ../../Includes.txt
+
+============================================================================
+Feature: #86409 - Allow usage of environment variables in site configuration
+============================================================================
+
+See :issue:`86409`
+
+Description
+===========
+
+To enable environment variable based configuration the TYPO3 Core Yaml loader has been adjusted to
+be able to resolve environment variables. Resolving of variables in the loader can be enabled or
+disabled via flags. When editing the site configuration through the backend interface the resolving
+of environment variables needs to be disabled to be able to add environment configuration through
+the interface.
+
+The format for environment variables is %env(ENV_NAME)%. Environment variables may be used to replace
+complete values or parts of a value.
+
+
+Impact
+======
+
+In site configuration environment variables can be used. One common example would be the base url
+that can now be configured via an environment variable.
+
+Additionally, the Yaml Loader class has two new flags: PROCESS_PLACEHOLDERS and PROCESS_IMPORTS
+PROCESS_PLACEHOLDERS decides whether or not placeholders (`%abc%`) will be resolved.
+PROCESS_IMPORTS decides whether or not imports (`imports` key) will be resolved.
+
+Example usage in site configuration:
+
+.. code-block:: yaml
+
+       base: 'https://%env(BASE_DOMAIN)%/'
+
+.. index:: Backend, ext:core
index b5f508d..6b7d14c 100644 (file)
@@ -96,7 +96,7 @@ betterthanbefore: 2
      * Method checking for placeholders
      * @test
      */
-    public function loadWithPlacholders()
+    public function loadWithPlaceholders(): void
     {
         $fileName = 'Berta.yml';
         $fileContents = '
@@ -127,6 +127,101 @@ betterthanbefore: \'%firstset.myinitialversion%\'
         $this->assertSame($expected, $output);
     }
 
+    public function loadWithEnvVarDataProvider(): array
+    {
+        return [
+            'plain' => [
+                'foo=heinz',
+                'carl: \'%env(foo)%\'',
+                ['carl' => 'heinz']
+            ],
+            'quoted var' => [
+                'foo=heinz',
+                "carl: '%env(''foo'')%'",
+                ['carl' => 'heinz']
+            ],
+            'double quoted var' => [
+                'foo=heinz',
+                "carl: '%env(\"foo\")%'",
+                ['carl' => 'heinz']
+            ],
+            'var in the middle' => [
+                'foo=heinz',
+                "carl: 'https://%env(foo)%/foo'",
+                ['carl' => 'https://heinz/foo']
+            ],
+            'quoted var in the middle' => [
+                'foo=heinz',
+                "carl: 'https://%env(''foo'')%/foo'",
+                ['carl' => 'https://heinz/foo']
+            ],
+            'double quoted var in the middle' => [
+                'foo=heinz',
+                "carl: 'https://%env(\"foo\")%/foo'",
+                ['carl' => 'https://heinz/foo']
+            ],
+        ];
+    }
+
+    /**
+     * Method checking for env placeholders
+     *
+     * @dataProvider loadWithEnvVarDataProvider
+     * @test
+     * @param string $env
+     * @param string $yamlContent
+     * @param array $expected
+     */
+    public function loadWithEnvVarPlaceholders(string $env, string $yamlContent, array $expected): void
+    {
+        putenv($env);
+        $fileName = 'Berta.yml';
+        $fileContents = $yamlContent;
+
+        // Accessible mock to $subject since getFileContents calls GeneralUtility methods
+        $subject = $this->getAccessibleMock(YamlFileLoader::class, ['getFileContents']);
+        $subject->expects($this->once())->method('getFileContents')->with($fileName)->willReturn($fileContents);
+        $output = $subject->load($fileName);
+        $this->assertSame($expected, $output);
+        putenv('foo=');
+    }
+
+    /**
+     * Method checking for env placeholders
+     *
+     * @test
+     */
+    public function loadWithEnvVarPlaceholdersDoesNotReplaceWithNonExistingValues(): void
+    {
+        $fileName = 'Berta.yml';
+        $fileContents = '
+
+firstset:
+  myinitialversion: 13
+options:
+    - option1
+    - option2
+betterthanbefore: \'%env(mynonexistingenv)%\'
+';
+
+        $expected = [
+            'firstset' => [
+                'myinitialversion' => 13
+            ],
+            'options' => [
+                'option1',
+                'option2'
+            ],
+            'betterthanbefore' => '%env(mynonexistingenv)%'
+        ];
+
+        // Accessible mock to $subject since getFileContents calls GeneralUtility methods
+        $subject = $this->getAccessibleMock(YamlFileLoader::class, ['getFileContents']);
+        $subject->expects($this->once())->method('getFileContents')->with($fileName)->willReturn($fileContents);
+        $output = $subject->load($fileName);
+        $this->assertSame($expected, $output);
+    }
+
     /**
      * dataprovider for tests isPlaceholderTest
      * @return array