[FEATURE] Introduce new @import syntax for TS includes 46/54446/6
authorBenni Mack <benni@typo3.org>
Thu, 19 Oct 2017 14:27:16 +0000 (16:27 +0200)
committerBenjamin Kott <benjamin.kott@outlook.com>
Fri, 20 Oct 2017 13:11:21 +0000 (15:11 +0200)
The original '<INCLUDE_TYPOSCRIPT...>' syntax is hard to understand,
error-prone and overloaded with features.

This patch introduces a new way to include files based on SymfonyFinder
and simple logic to allow to include files or folders:

The following syntax (leaned towards SASS imports) is added:
Imports one file:

- @import 'EXT:myext/Configuration/TypoScript/myfile.typoscript'

Imports all files in a folder (always sorted by name):
- @import 'EXT:myext/Configuration/TypoScript/*'
- @import 'EXT:myext/Configuration/TypoScript/'

Imports all files ending with ".typoscript":
- @import 'EXT:myext/Configuration/TypoScript/*.typoscript'

Automatically adds '.typoscript' file ending and includes setup.typoscript
- @import 'EXT:myext/Configuration/TypoScript/setup'

This is all done with Symfony Finder to find the files.

Resolves: #82812
Releases: master
Change-Id: I4b64a087ef8c6aa85063c19c1882c9ed3448d9b5
Reviewed-on: https://review.typo3.org/54446
Tested-by: TYPO3com <no-reply@typo3.com>
Tested-by: Markus Sommer <markus.sommer@typo3.org>
Reviewed-by: Markus Sommer <markus.sommer@typo3.org>
Reviewed-by: Benjamin Kott <benjamin.kott@outlook.com>
Tested-by: Benjamin Kott <benjamin.kott@outlook.com>
typo3/sysext/core/Classes/TypoScript/Parser/TypoScriptParser.php
typo3/sysext/core/Documentation/Changelog/master/Feature-82812-NewSyntaxForImportingTypoScriptFiles.rst [new file with mode: 0644]
typo3/sysext/core/Tests/Unit/FileStreamWrapperTest.php
typo3/sysext/core/Tests/Unit/TypoScript/Fixtures/badfilename.php [new file with mode: 0644]
typo3/sysext/core/Tests/Unit/TypoScript/Fixtures/recursive_includes_setup.typoscript [new file with mode: 0644]
typo3/sysext/core/Tests/Unit/TypoScript/Fixtures/setup.typoscript [new file with mode: 0644]
typo3/sysext/core/Tests/Unit/TypoScript/Parser/TypoScriptParserTest.php

index 0c443b6..afcf120 100644 (file)
@@ -15,6 +15,7 @@ namespace TYPO3\CMS\Core\TypoScript\Parser;
  */
 
 use Psr\Log\LoggerInterface;
+use Symfony\Component\Finder\Finder;
 use TYPO3\CMS\Backend\Configuration\TypoScript\ConditionMatching\ConditionMatcher as BackendConditionMatcher;
 use TYPO3\CMS\Core\Configuration\TypoScript\ConditionMatching\AbstractConditionMatcher;
 use TYPO3\CMS\Core\Log\LogManager;
@@ -802,6 +803,9 @@ class TypoScriptParser
 ';
         }
 
+        // Checking for @import syntax imported files
+        $string = self::addImportsFromExternalFiles($string, $cycle_counter, $returnFiles, $includedFiles, $parentFilenameOrPath);
+
         // If no tags found, no need to do slower preg_split
         if (strpos($string, '<INCLUDE_TYPOSCRIPT:') !== false) {
             $splitRegEx = '/\r?\n\s*<INCLUDE_TYPOSCRIPT:\s*(?i)source\s*=\s*"((?i)file|dir):\s*([^"]*)"(.*)>[\ \t]*/';
@@ -908,6 +912,138 @@ class TypoScriptParser
     }
 
     /**
+     * Splits the unparsed TypoScript content into @import statements
+     *
+     * @param string $typoScript unparsed TypoScript
+     * @param int $cycleCounter counter to stop recursion
+     * @param bool $returnFiles whether to populate the included Files or not
+     * @param array $includedFiles - by reference - if any included files are added, they are added here
+     * @param string $parentFilenameOrPath the current imported file to resolve relative paths - handled by reference
+     * @return string the unparsed TypoScript with included external files
+     */
+    protected static function addImportsFromExternalFiles($typoScript, $cycleCounter, $returnFiles, &$includedFiles, &$parentFilenameOrPath)
+    {
+        // Check for new syntax "@import 'EXT:bennilove/Configuration/TypoScript/*'"
+        if (strpos($typoScript, '@import \'') !== false || strpos($typoScript, '@import "') !== false) {
+            $splitRegEx = '/\r?\n\s*@import\s[\'"]([^\'"]*)[\'"][\ \t]?/';
+            $parts = preg_split($splitRegEx, LF . $typoScript . LF, -1, PREG_SPLIT_DELIM_CAPTURE);
+            // First text part goes through
+            $newString = $parts[0] . LF;
+            $partCount = count($parts);
+            for ($i = 1; $i + 2 <= $partCount; $i += 2) {
+                $filename = $parts[$i];
+                $tsContentsTillNextInclude = $parts[$i + 1];
+                // Resolve a possible relative paths if a parent file is given
+                if ($parentFilenameOrPath !== '' && $filename[0] === '.') {
+                    $filename = PathUtility::getAbsolutePathOfRelativeReferencedFileOrPath($parentFilenameOrPath, $filename);
+                }
+                $newString .= self::importExternalTypoScriptFile($filename, $cycleCounter, $returnFiles, $includedFiles);
+                // Prepend next normal (not file) part to output string
+                $newString .= $tsContentsTillNextInclude;
+            }
+            // Add a line break before and after the included code in order to make sure that the parser always has a LF.
+            $typoScript = LF . trim($newString) . LF;
+        }
+        return $typoScript;
+    }
+
+    /**
+     * Include file $filename. Contents of the file will be returned, filename is added to &$includedFiles.
+     * Further include/import statements in the contents are processed recursively.
+     *
+     * @param string $filename Full absolute path+filename to the typoscript file to be included
+     * @param int $cycleCounter Counter for detecting endless loops
+     * @param bool $returnFiles When set, filenames of included files will be prepended to the array &$includedFiles
+     * @param array &$includedFiles Array to which the filenames of included files will be prepended (referenced)
+     * @return string the unparsed TypoScript content from external files
+     */
+    protected static function importExternalTypoScriptFile($filename, $cycleCounter, $returnFiles, array &$includedFiles)
+    {
+        if (strpos('..', $filename) !== false) {
+            return self::typoscriptIncludeError('Invalid filepath "' . $filename . '" (containing "..").');
+        }
+
+        $content = '';
+        $absoluteFileName = GeneralUtility::getFileAbsFileName($filename);
+        if ((string)$absoluteFileName === '') {
+            return self::typoscriptIncludeError('Illegal filepath "' . $filename . '".');
+        }
+
+        $finder = new Finder();
+        $finder
+            // no recursive mode on purpose
+            ->depth(0)
+            // no directories should be fetched
+            ->files()
+            ->sortByName();
+
+        // Search all files in the folder
+        if (is_dir($absoluteFileName)) {
+            $finder->in($absoluteFileName);
+            // Used for the TypoScript comments
+            $readableFilePrefix = $filename;
+        } else {
+            // Apparently this is not a folder, so the restriction
+            // is the folder so we restrict into this folder
+            $finder->in(dirname($absoluteFileName));
+            if (!is_file($absoluteFileName)
+                && strpos(basename($absoluteFileName), '*') === false
+                && substr(basename($absoluteFileName), -11) !== '.typoscript') {
+                $absoluteFileName .= '*.typoscript';
+            }
+            $finder->name(basename($absoluteFileName));
+            $readableFilePrefix = dirname($filename);
+        }
+
+        foreach ($finder as $fileObject) {
+            // Clean filename output for comments
+            $readableFileName = rtrim($readableFilePrefix, '/') . '/' . $fileObject->getFilename();
+            $content .= '### @import \'' . $readableFileName . '\' begin ###' . LF;
+            // Check for allowed files
+            if (!GeneralUtility::verifyFilenameAgainstDenyPattern($fileObject->getFilename())) {
+                $content .= self::typoscriptIncludeError('File "' . $readableFileName . '" was not included since it is not allowed due to fileDenyPattern.');
+            } else {
+                $includedFiles[] = $fileObject->getPathname();
+                // check for includes in included text
+                $included_text = self::checkIncludeLines($fileObject->getContents(), $cycleCounter++, $returnFiles, $absoluteFileName);
+                // If the method also has to return all included files, merge currently included
+                // files with files included by recursively calling itself
+                if ($returnFiles && is_array($included_text)) {
+                    $includedFiles = array_merge($includedFiles, $included_text['files']);
+                    $included_text = $included_text['typoscript'];
+                }
+                $content .= $included_text . LF;
+            }
+            $content .= '### @import \'' . $readableFileName . '\' end ###' . LF;
+
+            // load default TypoScript for content rendering templates like
+            // fluid_styled_content if those have been included through e.g.
+            // @import "fluid_styled_content/Configuration/TypoScript/setup.typoscript"
+            if (strpos(strtoupper($filename), 'EXT:') === 0) {
+                $filePointerPathParts = explode('/', substr($filename, 4));
+                // remove file part, determine whether to load setup or constants
+                list($includeType) = explode('.', array_pop($filePointerPathParts));
+
+                if (in_array($includeType, ['setup', 'constants'], true)) {
+                    // adapt extension key to required format (no underscores)
+                    $filePointerPathParts[0] = str_replace('_', '', $filePointerPathParts[0]);
+
+                    // load default TypoScript
+                    $defaultTypoScriptKey = implode('/', $filePointerPathParts) . '/';
+                    if (in_array($defaultTypoScriptKey, $GLOBALS['TYPO3_CONF_VARS']['FE']['contentRenderingTemplates'], true)) {
+                        $content .= $GLOBALS['TYPO3_CONF_VARS']['FE']['defaultTypoScript_' . $includeType . '.']['defaultContentRendering'];
+                    }
+                }
+            }
+        }
+
+        if (empty($content)) {
+            return self::typoscriptIncludeError('No file or folder found for importing TypoScript on "' . $filename . '".');
+        }
+        return $content;
+    }
+
+    /**
      * Include file $filename. Contents of the file will be prepended to &$newstring, filename to &$includedFiles
      * Further include_typoscript tags in the contents are processed recursively
      *
diff --git a/typo3/sysext/core/Documentation/Changelog/master/Feature-82812-NewSyntaxForImportingTypoScriptFiles.rst b/typo3/sysext/core/Documentation/Changelog/master/Feature-82812-NewSyntaxForImportingTypoScriptFiles.rst
new file mode 100644 (file)
index 0000000..0044daa
--- /dev/null
@@ -0,0 +1,54 @@
+.. include:: ../../Includes.txt
+
+===========================================================
+Feature: #82812 - New syntax for importing TypoScript files
+===========================================================
+
+See :issue:`82812`
+
+Description
+===========
+
+A new syntax for importing external TypoScript files has been introduced, which acts as a preprocessor
+before the actual parsing (Condition evaluation) is made.
+
+It's main purpose is ease the use of TypoScript includes and make it easier for integrators and
+frontend developers to work with distributed TypoScript files. The syntax is inspired by SASS
+imports and works as follows:
+
+.. code-block:: typoscript
+
+       # Import a single file
+       @import 'EXT:myproject/TypoScript/Configuration/randomfile.typoscript'
+       
+       # Import multiple files in a single directory, sorted by file name
+       @import 'EXT:myproject/TypoScript/Configuration/*.typoscript'
+       
+       # Import all files in a directory
+       @import 'EXT:myproject/TypoScript/Configuration/'
+       
+       # It's possible to omit the file ending, then "typoscript" is automatically added
+       @import 'EXT:myproject/TypoScript/Configuration/'
+
+The main benefits of `@import` over using `<INCLUDE_TYPOSCRIPT>` are:
+- Less error-prone when adding statements to TypoScript
+- Easier to read what should be included (files, folders - instead of `FILE:` and `DIR:` syntax)
+- @import is more speaking than a pseudo-XML syntax
+
+The following rules apply:
+- If multiple files are found, the file name is important in which order the files (sorted
+alphabetically by filename)
+- Recursive inclusion of files (@import within @import is possible)
+- It is not possible to use a condition as possible with <INCLUDE_TYPOSCRIPT condition=""> as its
+sole purpose is to include files, which happens before the actual real condition matching happens,
+and the INCLUDE_TYPOSCRIPT condition syntax is a conceptual mistake, and should be avoided.
+- Both `<INCLUDE_TYPOSCRIPT>` and `@import` can work side-by-side in the same project
+- If a directory is included, it will not include files recursively
+- Quoting of the filename is necessary, both double quotes (") and single tickst (') can be used
+
+The syntax is designed to stay, and @import is not intended to be extended with more logic in the
+future. However, it may be possible that TypoScript will get more preparsing logic via the @ annotation.
+
+Under the hood, Symfony Finder is used to detect files. This makes globbing (* syntax) possible.
+
+.. index:: TypoScript
\ No newline at end of file
index 49a3422..28d889e 100644 (file)
@@ -47,6 +47,12 @@ class FileStreamWrapperTest extends \TYPO3\TestingFramework\Core\Unit\UnitTestCa
                 'fileadmin' => [
                     'ext_typoscript_setup.txt' => 'test.Core.TypoScript = 1',
                     'test' => ['Foo.bar' => 'Baz'],
+                    'setup.typoscript' => 'test.TYPO3Forever.TypoScript = 1
+',
+                    'recursive_includes_setup.typoscript' => '@import \'EXT:core/Tests/Unit/TypoScript/Fixtures/setup.typoscript\'
+',
+                    'badfilename.php' => 'good.bad = ugly
+'
                 ],
             ],
         ];
diff --git a/typo3/sysext/core/Tests/Unit/TypoScript/Fixtures/badfilename.php b/typo3/sysext/core/Tests/Unit/TypoScript/Fixtures/badfilename.php
new file mode 100644 (file)
index 0000000..84c2285
--- /dev/null
@@ -0,0 +1 @@
+good.bad = ugly
diff --git a/typo3/sysext/core/Tests/Unit/TypoScript/Fixtures/recursive_includes_setup.typoscript b/typo3/sysext/core/Tests/Unit/TypoScript/Fixtures/recursive_includes_setup.typoscript
new file mode 100644 (file)
index 0000000..4c5a49b
--- /dev/null
@@ -0,0 +1 @@
+@import 'EXT:core/Tests/Unit/TypoScript/Fixtures/setup.typoscript'
diff --git a/typo3/sysext/core/Tests/Unit/TypoScript/Fixtures/setup.typoscript b/typo3/sysext/core/Tests/Unit/TypoScript/Fixtures/setup.typoscript
new file mode 100644 (file)
index 0000000..c2fc8b9
--- /dev/null
@@ -0,0 +1 @@
+test.TYPO3Forever.TypoScript = 1
index 929720f..d5c9d19 100644 (file)
@@ -328,6 +328,191 @@ class TypoScriptParserTest extends \TYPO3\TestingFramework\Core\Unit\UnitTestCas
     }
 
     /**
+     * @return array
+     */
+    public function importFilesDataProvider()
+    {
+        return [
+            'Found include file is imported' => [
+                // Input TypoScript
+                'bennilove = before
+@import "EXT:core/Tests/Unit/TypoScript/Fixtures/ext_typoscript_setup.txt"
+'
+                ,
+                // Expected
+                '
+bennilove = before
+### @import \'EXT:core/Tests/Unit/TypoScript/Fixtures/ext_typoscript_setup.txt\' begin ###
+test.Core.TypoScript = 1
+### @import \'EXT:core/Tests/Unit/TypoScript/Fixtures/ext_typoscript_setup.txt\' end ###
+'
+            ],
+            'Not found file is not imported' => [
+                // Input TypoScript
+                'bennilove = before
+@import "EXT:core/Tests/Unit/TypoScript/Fixtures/notfoundfile.txt"
+'
+                ,
+                // Expected
+                '
+bennilove = before
+
+###
+### ERROR: No file or folder found for importing TypoScript on "EXT:core/Tests/Unit/TypoScript/Fixtures/notfoundfile.txt".
+###
+'
+            ],
+            'All files with glob are imported' => [
+                // Input TypoScript
+                'bennilove = before
+@import "EXT:core/Tests/Unit/TypoScript/Fixtures/*.txt"
+'
+                ,
+                // Expected
+                '
+bennilove = before
+### @import \'EXT:core/Tests/Unit/TypoScript/Fixtures/ext_typoscript_setup.txt\' begin ###
+test.Core.TypoScript = 1
+### @import \'EXT:core/Tests/Unit/TypoScript/Fixtures/ext_typoscript_setup.txt\' end ###
+'
+            ],
+            'Specific file with typoscript ending is imported' => [
+                // Input TypoScript
+                'bennilove = before
+@import "EXT:core/Tests/Unit/TypoScript/Fixtures/setup.typoscript"
+'
+                ,
+                // Expected
+                '
+bennilove = before
+### @import \'EXT:core/Tests/Unit/TypoScript/Fixtures/setup.typoscript\' begin ###
+test.TYPO3Forever.TypoScript = 1
+
+### @import \'EXT:core/Tests/Unit/TypoScript/Fixtures/setup.typoscript\' end ###
+'
+            ],
+            'All files in folder are imported, sorted by name' => [
+                // Input TypoScript
+                'bennilove = before
+@import "EXT:core/Tests/Unit/TypoScript/Fixtures/"
+'
+                ,
+                // Expected
+                '
+bennilove = before
+### @import \'EXT:core/Tests/Unit/TypoScript/Fixtures/badfilename.php\' begin ###
+
+###
+### ERROR: File "EXT:core/Tests/Unit/TypoScript/Fixtures/badfilename.php" was not included since it is not allowed due to fileDenyPattern.
+###
+
+### @import \'EXT:core/Tests/Unit/TypoScript/Fixtures/badfilename.php\' end ###
+### @import \'EXT:core/Tests/Unit/TypoScript/Fixtures/ext_typoscript_setup.txt\' begin ###
+test.Core.TypoScript = 1
+### @import \'EXT:core/Tests/Unit/TypoScript/Fixtures/ext_typoscript_setup.txt\' end ###
+### @import \'EXT:core/Tests/Unit/TypoScript/Fixtures/recursive_includes_setup.typoscript\' begin ###
+
+### @import \'EXT:core/Tests/Unit/TypoScript/Fixtures/setup.typoscript\' begin ###
+test.TYPO3Forever.TypoScript = 1
+
+### @import \'EXT:core/Tests/Unit/TypoScript/Fixtures/setup.typoscript\' end ###
+
+### @import \'EXT:core/Tests/Unit/TypoScript/Fixtures/recursive_includes_setup.typoscript\' end ###
+### @import \'EXT:core/Tests/Unit/TypoScript/Fixtures/setup.typoscript\' begin ###
+test.TYPO3Forever.TypoScript = 1
+
+### @import \'EXT:core/Tests/Unit/TypoScript/Fixtures/setup.typoscript\' end ###
+'
+            ],
+            'All files ending with typoscript in folder are imported' => [
+                // Input TypoScript
+                'bennilove = before
+@import "EXT:core/Tests/Unit/TypoScript/Fixtures/*typoscript"
+'
+                ,
+                // Expected
+                '
+bennilove = before
+### @import \'EXT:core/Tests/Unit/TypoScript/Fixtures/recursive_includes_setup.typoscript\' begin ###
+
+### @import \'EXT:core/Tests/Unit/TypoScript/Fixtures/setup.typoscript\' begin ###
+test.TYPO3Forever.TypoScript = 1
+
+### @import \'EXT:core/Tests/Unit/TypoScript/Fixtures/setup.typoscript\' end ###
+
+### @import \'EXT:core/Tests/Unit/TypoScript/Fixtures/recursive_includes_setup.typoscript\' end ###
+### @import \'EXT:core/Tests/Unit/TypoScript/Fixtures/setup.typoscript\' begin ###
+test.TYPO3Forever.TypoScript = 1
+
+### @import \'EXT:core/Tests/Unit/TypoScript/Fixtures/setup.typoscript\' end ###
+'
+            ],
+            'All typoscript files in folder are imported' => [
+                // Input TypoScript
+                'bennilove = before
+@import "EXT:core/Tests/Unit/TypoScript/Fixtures/*.typoscript"
+'
+                ,
+                // Expected
+                '
+bennilove = before
+### @import \'EXT:core/Tests/Unit/TypoScript/Fixtures/recursive_includes_setup.typoscript\' begin ###
+
+### @import \'EXT:core/Tests/Unit/TypoScript/Fixtures/setup.typoscript\' begin ###
+test.TYPO3Forever.TypoScript = 1
+
+### @import \'EXT:core/Tests/Unit/TypoScript/Fixtures/setup.typoscript\' end ###
+
+### @import \'EXT:core/Tests/Unit/TypoScript/Fixtures/recursive_includes_setup.typoscript\' end ###
+### @import \'EXT:core/Tests/Unit/TypoScript/Fixtures/setup.typoscript\' begin ###
+test.TYPO3Forever.TypoScript = 1
+
+### @import \'EXT:core/Tests/Unit/TypoScript/Fixtures/setup.typoscript\' end ###
+'
+            ],
+            'All typoscript files in folder with glob are not imported due to recursion level=0' => [
+                // Input TypoScript
+                'bennilove = before
+@import "EXT:core/Tests/Unit/**/*.typoscript"
+'
+                ,
+                // Expected
+                '
+bennilove = before
+
+###
+### ERROR: No file or folder found for importing TypoScript on "EXT:core/Tests/Unit/**/*.typoscript".
+###
+'
+            ],            'TypoScript file ending is automatically added' => [
+                // Input TypoScript
+                'bennilove = before
+@import "EXT:core/Tests/Unit/TypoScript/Fixtures/setup"
+'
+                ,
+                // Expected
+                '
+bennilove = before
+### @import \'EXT:core/Tests/Unit/TypoScript/Fixtures/setup.typoscript\' begin ###
+test.TYPO3Forever.TypoScript = 1
+
+### @import \'EXT:core/Tests/Unit/TypoScript/Fixtures/setup.typoscript\' end ###
+'
+            ],
+        ];
+    }
+
+    /**
+     * @test
+     * @dataProvider importFilesDataProvider
+     */
+    public function importFiles($typoScript, $expected)
+    {
+        $resolvedIncludeLines = TypoScriptParser::checkIncludeLines($typoScript);
+        $this->assertEquals($expected, $resolvedIncludeLines);
+    }
+
+    /**
      * @param string $typoScript
      * @param array $expected
      * @dataProvider typoScriptIsParsedToArrayDataProvider