[FEATURE] Introduce DataProcessors for splitting values 06/40506/14
authorBenjamin Mack <benni@typo3.org>
Sun, 21 Jun 2015 12:14:22 +0000 (14:14 +0200)
committerFrans Saris <franssaris@gmail.com>
Wed, 8 Jul 2015 08:34:39 +0000 (10:34 +0200)
In order to allow powerful processing for FLUIDTEMPLATE and other
cObjects, two new DataProcessors are added to ensure flexibility
with comma-separated values and split values listings.

Resolves: #67658
Releases: master
Change-Id: Ib6f36ed2b815b08721eb6a29b216821f271d24f2
Reviewed-on: http://review.typo3.org/40506
Reviewed-by: Christian Kuhn <lolli@schwarzbu.ch>
Tested-by: Christian Kuhn <lolli@schwarzbu.ch>
Reviewed-by: Frans Saris <franssaris@gmail.com>
Tested-by: Frans Saris <franssaris@gmail.com>
typo3/sysext/core/Classes/Utility/CsvUtility.php [new file with mode: 0644]
typo3/sysext/core/Documentation/Changelog/master/Feature-67658-IntroduceDataProcessorsForSplittingValues.rst [new file with mode: 0644]
typo3/sysext/core/Tests/Unit/Utility/CsvUtilityTest.php [new file with mode: 0644]
typo3/sysext/frontend/Classes/DataProcessing/CommaSeparatedValueProcessor.php [new file with mode: 0644]
typo3/sysext/frontend/Classes/DataProcessing/SplitProcessor.php [new file with mode: 0644]

diff --git a/typo3/sysext/core/Classes/Utility/CsvUtility.php b/typo3/sysext/core/Classes/Utility/CsvUtility.php
new file mode 100644 (file)
index 0000000..a744c05
--- /dev/null
@@ -0,0 +1,69 @@
+<?php
+namespace TYPO3\CMS\Core\Utility;
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+/**
+ * Class with helper functions for CSV handling
+ */
+class CsvUtility {
+
+       /**
+        * Convert a string, formatted as CSV, into an multidimensional array
+        *
+        * @param string $input The CSV input
+        * @param string $fieldDelimiter The field delimiter
+        * @param string $fieldEnclosure The field enclosure
+        * @param string $rowDelimiter The row delimiter
+        * @param int $maximumColumns The maximum amount of columns
+        * @return array
+        */
+       static public function csvToArray($input, $fieldDelimiter = ',', $fieldEnclosure = '"', $rowDelimiter = LF, $maximumColumns = 0) {
+               $multiArray = array();
+               $maximumCellCount = 0;
+
+               // explode() would not work with enclosed newlines
+               $rows = str_getcsv($input, $rowDelimiter);
+
+               foreach ($rows as $row) {
+                       $cells = str_getcsv($row, $fieldDelimiter, $fieldEnclosure);
+
+                       $maximumCellCount = max(count($cells), $maximumCellCount);
+
+                       $multiArray[] = $cells;
+               }
+
+               if ($maximumColumns > $maximumCellCount) {
+                       $maximumCellCount = $maximumColumns;
+               }
+
+               foreach ($multiArray as &$row) {
+                       for ($key = 0; $key < $maximumCellCount; $key++) {
+                               if (
+                                       $maximumColumns > 0
+                                       && $maximumColumns < $maximumCellCount
+                                       && $key >= $maximumColumns
+                               ) {
+                                       if (isset($row[$key])) {
+                                               unset($row[$key]);
+                                       }
+                               } elseif (!isset($row[$key])) {
+                                       $row[$key] = '';
+                               }
+                       }
+               }
+
+               return $multiArray;
+       }
+}
\ No newline at end of file
diff --git a/typo3/sysext/core/Documentation/Changelog/master/Feature-67658-IntroduceDataProcessorsForSplittingValues.rst b/typo3/sysext/core/Documentation/Changelog/master/Feature-67658-IntroduceDataProcessorsForSplittingValues.rst
new file mode 100644 (file)
index 0000000..2d737db
--- /dev/null
@@ -0,0 +1,73 @@
+===============================================================
+Feature: #67658 - Introduce DataProcessors for splitting values
+===============================================================
+
+Description
+===========
+
+Two new DataProcessors are added to allow flexible processing for comma-separated
+values. To use e.g. with the FLUIDTEMPLATE content object.
+
+The SplitProcessor allows to split values separated with a delimiter inside a single database field
+into an array to loop over it.
+
+The CommaSeparatedValueProcessor allows to split values into a two-dimensional array used for
+CSV files or tt_content records of CType "table".
+
+Using the SplitProcessor the following scenario is possible:
+
+.. code-block:: typoscript
+
+       page.10 = FLUIDTEMPLATE
+       page.10.file = EXT:site_default/Resources/Private/Template/Default.html
+       page.10.dataProcessing.2 = TYPO3\CMS\Frontend\DataProcessing\SplitProcessor
+       page.10.dataProcessing.2 {
+               if.isTrue.field = bodytext
+               delimiter = ,
+               fieldName = bodytext
+               removeEmptyEntries = 1
+               filterIntegers = 1
+               filterUnique = 1
+               as = keywords
+       }
+
+
+In the Fluid template then iterate over the split data:
+
+.. code-block:: html
+
+       <f:for each="{keywords}" as="keyword">
+               <li>Keyword: {keyword}</li>
+       </f:for>
+
+
+Using the CommaSeparatedValueProcessor the following scenario is possible:
+
+.. code-block:: typoscript
+
+       page.10 = FLUIDTEMPLATE
+       page.10.file = EXT:site_default/Resources/Private/Template/Default.html
+       page.10.dataProcessing.4 = TYPO3\CMS\Frontend\DataProcessing\CommaSeparatedValueProcessor
+       page.10.dataProcessing.4 {
+               if.isTrue.field = bodytext
+               fieldName = bodytext
+               fieldDelimiter = |
+               fieldEnclosure =
+               maximumColumns = 2
+               as = table
+       }
+
+
+In the Fluid template then iterate over the processed data:
+
+.. code-block:: html
+
+       <table>
+               <f:for each="{table}" as="columns">
+                       <tr>
+                               <f:for each="{columns}" as="column">
+                                       <td>{column}</td>
+                               </f:for>
+                       <tr>
+               </f:for>
+       </table>
diff --git a/typo3/sysext/core/Tests/Unit/Utility/CsvUtilityTest.php b/typo3/sysext/core/Tests/Unit/Utility/CsvUtilityTest.php
new file mode 100644 (file)
index 0000000..094e7c3
--- /dev/null
@@ -0,0 +1,110 @@
+<?php
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+use TYPO3\CMS\Core\Tests\UnitTestCase;
+use TYPO3\CMS\Core\Utility\CsvUtility;
+
+/**
+ * Test cases of CsvUtility
+ */
+class CsvUtilityTest extends UnitTestCase {
+
+       /**
+        * @return array
+        */
+       public function csvToArrayDataProvider() {
+               return array(
+                       'Valid data' => array(
+                               'input'  => 'Column A, Column B, Column C' . LF . 'Value, Value2, Value 3',
+                               'fieldDelimiter' => ',',
+                               'fieldEnclosure' => '',
+                               'rowDelimiter' => LF,
+                               'maximumColumns' => 0,
+                               'expectedResult' => array(
+                                       array('Column A', ' Column B', ' Column C'),
+                                       array('Value', ' Value2', ' Value 3')
+                               )
+                       ),
+
+                       'Valid data with enclosed "' => array(
+                               'input'  => '"Column A", "Column B", "Column C"' . LF . '"Value", "Value2", "Value 3"',
+                               'fieldDelimiter' => ',',
+                               'fieldEnclosure' => '"',
+                               'rowDelimiter' => LF,
+                               'maximumColumns' => 0,
+                               'expectedResult' => array(
+                                       array('Column A', 'Column B', 'Column C'),
+                                       array('Value', 'Value2', 'Value 3')
+                               )
+                       ),
+
+                       'Valid data with semicolons and enclosed "' => array(
+                               'input'  => '"Column A"; "Column B"; "Column C"' . LF . '"Value"; "Value2"; "Value 3"',
+                               'fieldDelimiter' => ';',
+                               'fieldEnclosure' => '"',
+                               'rowDelimiter' => LF,
+                               'maximumColumns' => 0,
+                               'expectedResult' => array(
+                                       array('Column A', 'Column B', 'Column C'),
+                                       array('Value', 'Value2', 'Value 3')
+                               )
+                       ),
+
+                       'Valid data with semicolons and enclosed " and two columns' => array(
+                               'input'  => '"Column A"; "Column B"; "Column C"; "Column D"' . LF . '"Value"; "Value2"; "Value 3"',
+                               'fieldDelimiter' => ';',
+                               'fieldEnclosure' => '"',
+                               'rowDelimiter' => LF,
+                               'maximumColumns' => 2,
+                               'expectedResult' => array(
+                                       array('Column A', 'Column B'),
+                                       array('Value', 'Value2')
+                               )
+                       ),
+
+                       'Data with comma but configured with semicolons and enclosed "' => array(
+                               'input'  => '"Column A", "Column B", "Column C"' . LF . '"Value", "Value2", "Value 3"',
+                               'fieldDelimiter' => ';',
+                               'fieldEnclosure' => '"',
+                               'rowDelimiter' => LF,
+                               'maximumColumns' => 0,
+                               'expectedResult' => array(
+                                       array('Column A, "Column B", "Column C"'),
+                                       array('Value, "Value2", "Value 3"')
+                               )
+                       ),
+
+                       'Data with comma as field delimiter and semicolons as row delimiter' => array(
+                               'input'  => '"Column A", "Column B", "Column C";"Value", "Value2", "Value 3"',
+                               'fieldDelimiter' => ',',
+                               'fieldEnclosure' => '"',
+                               'rowDelimiter' => ';',
+                               'maximumColumns' => 0,
+                               'expectedResult' => array(
+                                       array('Column A', 'Column B', 'Column C'),
+                                       array('Value', 'Value2', 'Value 3')
+                               )
+                       )
+               );
+       }
+
+       /**
+        * @dataProvider csvToArrayDataProvider
+        * @test
+        */
+       public function csvToArraySplitsAsExpected($input, $fieldDelimiter, $fieldEnclosure, $rowDelimiter, $maximumColumns, $expectedResult) {
+               $this->assertEquals($expectedResult, CsvUtility::csvToArray($input, $fieldDelimiter, $fieldEnclosure, $rowDelimiter, $maximumColumns));
+       }
+}
\ No newline at end of file
diff --git a/typo3/sysext/frontend/Classes/DataProcessing/CommaSeparatedValueProcessor.php b/typo3/sysext/frontend/Classes/DataProcessing/CommaSeparatedValueProcessor.php
new file mode 100644 (file)
index 0000000..3fe236e
--- /dev/null
@@ -0,0 +1,107 @@
+<?php
+namespace TYPO3\CMS\Frontend\DataProcessing;
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+use TYPO3\CMS\Core\Utility\CsvUtility;
+use TYPO3\CMS\Extbase\Utility\DebuggerUtility;
+use TYPO3\CMS\Frontend\ContentObject\ContentObjectRenderer;
+use TYPO3\CMS\Frontend\ContentObject\DataProcessorInterface;
+
+/**
+ * This data processor will take field data formatted as a string, where each line, separated by line feed,
+ * represents a row. By default columns are separated by the delimiter character "comma ,",
+ * and can be enclosed by the character 'quotation mark "', like the default in a regular CSV file.
+ *
+ * An example of such a field is "bodytext" in the CType "table".
+ *
+ * The table data is transformed to a multi dimensional array, taking the delimiter and enclosure into account,
+ * before it is passed to the view.
+ *
+ * Example field data:
+ *
+ * This is row 1 column 1|This is row 1 column 2|This is row 1 column 3
+ * This is row 2 column 1|This is row 2 column 2|This is row 2 column 3
+ * This is row 3 column 1|This is row 3 column 2|This is row 3 column 3
+ *
+ * Example TypoScript configuration:
+ *
+ * 10 = \TYPO3\CMS\Frontend\ContentObject\DataProcessing\CommaSeparatedValueProcessor
+ * 10 {
+ *   if.isTrue.field = bodytext
+ *   fieldName = bodytext
+ *   fieldDelimiter = |
+ *   fieldEnclosure =
+ *   maximumColumns = 2
+ *   as = table
+ * }
+ *
+ * whereas "table" can be used as a variable {table} inside Fluid for iteration.
+ *
+ * Using maximumColumns limits the amount of columns in the multi dimensional array.
+ * In the example, field data of the last column will be stripped off.
+ *
+ * Multi line cells are taken into account.
+ */
+class CommaSeparatedValueProcessor implements DataProcessorInterface {
+
+       /**
+        * Process CSV field data to split into a multi dimensional array
+        *
+        * @param ContentObjectRenderer $cObj The data of the content element or page
+        * @param array $contentObjectConfiguration The configuration of Content Object
+        * @param array $processorConfiguration The configuration of this processor
+        * @param array $processedData Key/value store of processed data (e.g. to be passed to a Fluid View)
+        * @return array the processed data as key/value store
+        */
+       public function process(ContentObjectRenderer $cObj, array $contentObjectConfiguration, array $processorConfiguration, array $processedData) {
+
+               if (isset($processorConfiguration['if.']) && !$cObj->checkIf($processorConfiguration['if.'])) {
+                       return $processedData;
+               }
+
+               // The field name to process
+               $fieldName = $cObj->stdWrapValue('fieldName', $processorConfiguration);
+               if (empty($fieldName)) {
+                       return $processedData;
+               }
+
+               $originalValue = $cObj->data[$fieldName];
+
+               // Set the target variable
+               $targetVariableName = $cObj->stdWrapValue('as', $processorConfiguration, $fieldName);
+
+               // Set the maximum amount of columns
+               $maximumColumns = $cObj->stdWrapValue('maximumColumns', $processorConfiguration, 0);
+
+               // Set the field delimiter which is "," by default
+               $fieldDelimiter = $cObj->stdWrapValue('fieldDelimiter', $processorConfiguration, ',');
+
+               // Set the field enclosure which is " by default
+               $fieldEnclosure = $cObj->stdWrapValue('fieldEnclosure', $processorConfiguration, '"');
+
+               // Set the row delimiter which is "LF" by default
+               $rowDelimiter = $cObj->stdWrapValue('rowDelimiter', $processorConfiguration, LF);
+
+               $processedData[$targetVariableName] = CsvUtility::csvToArray(
+                       $originalValue,
+                       $fieldDelimiter,
+                       $fieldEnclosure,
+                       $rowDelimiter,
+                       (int)$maximumColumns
+               );
+
+               return $processedData;
+       }
+}
diff --git a/typo3/sysext/frontend/Classes/DataProcessing/SplitProcessor.php b/typo3/sysext/frontend/Classes/DataProcessing/SplitProcessor.php
new file mode 100644 (file)
index 0000000..91a1d53
--- /dev/null
@@ -0,0 +1,95 @@
+<?php
+namespace TYPO3\CMS\Frontend\DataProcessing;
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Frontend\ContentObject\ContentObjectRenderer;
+use TYPO3\CMS\Frontend\ContentObject\DataProcessorInterface;
+
+/**
+ * This data processor can be used for processing data for the content elements which have split contents in one field
+ * like e.g. "bullets". It will split the field data in an array ready to be iterated over in Fluid.
+ *
+ * Example field data:
+ *
+ * This is bullet 1, This is bullet 2, This is bullet 3
+ *
+ * Example TypoScript configuration:
+ *
+ * 10 = \TYPO3\CMS\Frontend\ContentObject\DataProcessing\SplitProcessor
+ * 10 {
+ *   if.isTrue.field = bodytext
+ *   delimiter = ,
+ *   fieldName = bodytext
+ *   removeEmptyEntries = 1
+ *   filterIntegers = 1
+ *   filterUnique = 1
+ *   as = bullets
+ * }
+ *
+ * whereas "bullets" can be used as a variable {bullets} inside Fluid for iteration.
+ */
+class SplitProcessor implements DataProcessorInterface {
+
+       /**
+        * Process field data to split in an array
+        *
+        * @param ContentObjectRenderer $cObj The data of the content element or page
+        * @param array $contentObjectConfiguration The configuration of Content Object
+        * @param array $processorConfiguration The configuration of this processor
+        * @param array $processedData Key/value store of processed data (e.g. to be passed to a Fluid View)
+        * @return array the processed data as key/value store
+        */
+       public function process(ContentObjectRenderer $cObj, array $contentObjectConfiguration, array $processorConfiguration, array $processedData) {
+               if (isset($processorConfiguration['if.']) && !$cObj->checkIf($processorConfiguration['if.'])) {
+                       return $processedData;
+               }
+
+               // The field name to process
+               $fieldName = $cObj->stdWrapValue('fieldName', $processorConfiguration);
+               if (empty($fieldName)) {
+                       return $processedData;
+               }
+
+               $originalValue = $cObj->data[$fieldName];
+
+               // Set the target variable
+               $targetVariableName = $cObj->stdWrapValue('as', $processorConfiguration, $fieldName);
+
+               // Set the delimiter which is "LF" by default
+               $delimiter = $cObj->stdWrapValue('delimiter', $processorConfiguration, LF);
+
+               // Filter integers
+               $filterIntegers = (bool)$cObj->stdWrapValue('filterIntegers', $processorConfiguration, FALSE);
+
+               // Filter unique
+               $filterUnique = (bool)$cObj->stdWrapValue('filterUnique', $processorConfiguration, FALSE);
+
+               // Remove empty entries
+               $removeEmptyEntries = (bool)$cObj->stdWrapValue('removeEmptyEntries', $processorConfiguration, FALSE);
+
+               if ($filterIntegers === TRUE) {
+                       $processedData[$targetVariableName] = GeneralUtility::intExplode($delimiter, $originalValue, $removeEmptyEntries);
+               } else {
+                       $processedData[$targetVariableName] = GeneralUtility::trimExplode($delimiter, $originalValue, $removeEmptyEntries);
+               }
+
+               if ($filterUnique === TRUE) {
+                       $processedData[$targetVariableName] = array_unique($processedData[$targetVariableName]);
+               }
+
+               return $processedData;
+       }
+}