[!!!][TASK] Remove renderCharset option 79/46979/4
authorBenni Mack <benni@typo3.org>
Tue, 1 Mar 2016 07:48:58 +0000 (08:48 +0100)
committerAnja Leichsenring <aleichsenring@ab-softlab.de>
Thu, 3 Mar 2016 15:14:18 +0000 (16:14 +0100)
The renderCharset TypoScript option is used for internal
parsing of data coming from various sources (TypoScript,
Language Labels, Database, POST variables). As TYPO3
can only operate with UTF-8 in language files for the backend, and expects a UTF-8 compliant database,
the option is now always set to "utf-8" and cannot be changed.

Resolves: #73794
Releases: master
Change-Id: I4405e701567e65d31d290d0c9d922c38c7fc8cf3
Reviewed-on: https://review.typo3.org/46979
Reviewed-by: Daniel Maier <dani-maier@gmx.de>
Tested-by: Daniel Maier <dani-maier@gmx.de>
Reviewed-by: Anja Leichsenring <aleichsenring@ab-softlab.de>
Tested-by: Anja Leichsenring <aleichsenring@ab-softlab.de>
23 files changed:
typo3/sysext/core/Classes/Resource/Driver/LocalDriver.php
typo3/sysext/core/Classes/Utility/File/BasicFileUtility.php
typo3/sysext/core/Classes/Utility/GeneralUtility.php
typo3/sysext/core/Documentation/Changelog/master/Breaking-73794-RenderCharsetOptionRemoved.rst [new file with mode: 0644]
typo3/sysext/core/Documentation/Changelog/master/Deprecation-73794-SearchFormController-utf8_to_currentCharsetAndTSFE-renderCharset.rst [new file with mode: 0644]
typo3/sysext/extbase/Classes/Utility/LocalizationUtility.php
typo3/sysext/fluid/Classes/ViewHelpers/Format/CropViewHelper.php
typo3/sysext/form/Classes/Domain/Filter/AbstractFilter.php
typo3/sysext/form/Classes/PostProcess/MailPostProcessor.php
typo3/sysext/form/Tests/Unit/Filter/LowerCaseFilterTest.php
typo3/sysext/form/Tests/Unit/Filter/TitleCaseFilterTest.php
typo3/sysext/form/Tests/Unit/Filter/UpperCaseFilterTest.php
typo3/sysext/frontend/Classes/ContentObject/ContentObjectRenderer.php
typo3/sysext/frontend/Classes/Controller/TypoScriptFrontendController.php
typo3/sysext/frontend/Classes/Http/RequestHandler.php
typo3/sysext/frontend/Classes/Plugin/AbstractPlugin.php
typo3/sysext/frontend/Tests/Unit/ContentObject/CaseContentObjectTest.php
typo3/sysext/frontend/Tests/Unit/ContentObject/ContentObjectRendererTest.php
typo3/sysext/frontend/Tests/Unit/ContentObject/FilesContentObjectTest.php
typo3/sysext/indexed_search/Classes/Controller/SearchController.php
typo3/sysext/indexed_search/Classes/Controller/SearchFormController.php
typo3/sysext/t3editor/Resources/Private/tsref.xml
typo3/sysext/t3editor/Resources/Public/JavaScript/parse_typoscript/tokenizetyposcript.js

index d354158..6c389d1 100644 (file)
@@ -300,30 +300,18 @@ class LocalDriver extends AbstractHierarchicalFilesystemDriver
      * Previously in \TYPO3\CMS\Core\Utility\File\BasicFileUtility::cleanFileName()
      *
      * @param string $fileName Input string, typically the body of a fileName
-     * @param string $charset Charset of the a fileName (defaults to current charset; depending on context)
+     * @param string $charset Charset of the a fileName (defaults to utf-8)
      * @return string Output string with any characters not matching [.a-zA-Z0-9_-] is substituted by '_' and trailing dots removed
      * @throws Exception\InvalidFileNameException
      */
-    public function sanitizeFileName($fileName, $charset = '')
+    public function sanitizeFileName($fileName, $charset = 'utf-8')
     {
         // Handle UTF-8 characters
         if ($GLOBALS['TYPO3_CONF_VARS']['SYS']['UTF8filesystem']) {
             // Allow ".", "-", 0-9, a-z, A-Z and everything beyond U+C0 (latin capital letter a with grave)
             $cleanFileName = preg_replace('/[' . self::UNSAFE_FILENAME_CHARACTER_EXPRESSION . ']/u', '_', trim($fileName));
         } else {
-            // Define character set
-            if (!$charset) {
-                if (TYPO3_MODE === 'FE') {
-                    $charset = $GLOBALS['TSFE']->renderCharset;
-                } else {
-                    // default for Backend
-                    $charset = 'utf-8';
-                }
-            }
-            // If a charset was found, convert fileName
-            if ($charset) {
-                $fileName = $this->getCharsetConversion()->specCharsToASCII($charset, $fileName);
-            }
+            $fileName = $this->getCharsetConversion()->specCharsToASCII($charset, $fileName);
             // Replace unwanted characters by underscores
             $cleanFileName = preg_replace('/[' . self::UNSAFE_FILENAME_CHARACTER_EXPRESSION . '\\xC0-\\xFF]/', '_', trim($fileName));
         }
index e1f5606..0a6b7ba 100644 (file)
@@ -386,15 +386,9 @@ class BasicFileUtility
             // allow ".", "-", 0-9, a-z, A-Z and everything beyond U+C0 (latin capital letter a with grave)
             $cleanFileName = preg_replace('/[' . self::UNSAFE_FILENAME_CHARACTER_EXPRESSION . ']/u', '_', trim($fileName));
         } else {
-            // Define character set
-            if (is_object($GLOBALS['TSFE'])) {
-                $charset = $GLOBALS['TSFE']->renderCharset;
-            } else {
-                $charset = 'utf-8';
-            }
             /** @var CharsetConverter $charsetConverter */
             $charsetConverter = GeneralUtility::makeInstance(CharsetConverter::class);
-            $fileName = $charsetConverter->specCharsToASCII($charset, $fileName);
+            $fileName = $charsetConverter->specCharsToASCII('utf-8', $fileName);
 
             // Replace unwanted characters by underscores
             $cleanFileName = preg_replace('/[' . self::UNSAFE_FILENAME_CHARACTER_EXPRESSION . '\\xC0-\\xFF]/', '_', trim($fileName));
index 8838265..e203e56 100755 (executable)
@@ -274,14 +274,9 @@ class GeneralUtility
      */
     public static function fixed_lgd_cs($string, $chars, $appendString = '...')
     {
-        if (is_object($GLOBALS['TSFE'])) {
-            $charset = $GLOBALS['TSFE']->renderCharset != '' ? $GLOBALS['TSFE']->renderCharset : $GLOBALS['TSFE']->defaultCharSet;
-        } else {
-            $charset = 'utf-8';
-        }
         /** @var CharsetConverter $charsetConverter */
         $charsetConverter = self::makeInstance(\TYPO3\CMS\Core\Charset\CharsetConverter::class);
-        return $charsetConverter->crop($charset, $string, $chars, $appendString);
+        return $charsetConverter->crop('utf-8', $string, $chars, $appendString);
     }
 
     /**
diff --git a/typo3/sysext/core/Documentation/Changelog/master/Breaking-73794-RenderCharsetOptionRemoved.rst b/typo3/sysext/core/Documentation/Changelog/master/Breaking-73794-RenderCharsetOptionRemoved.rst
new file mode 100644 (file)
index 0000000..bc094de
--- /dev/null
@@ -0,0 +1,33 @@
+===============================================
+Breaking: #73794 - renderCharset option removed
+===============================================
+
+Description
+===========
+
+The TypoScript option ``config.renderCharset`` which was used as character set for internal conversion within a frontend
+request has been removed, as internal conversions are always set to UTF-8 now.
+The property ``$TSFE->renderCharset`` is now always set to ``utf-8``, and is not used within the TYPO3 Core anymore.
+
+Please note that it is still possible to define the character set of the returned content via ``config.metaCharset``.
+
+
+Impact
+======
+
+Using TYPO3 with a different datasource which is not UTF-8 (e.g. a database with latin1) might return unexpected results.
+Non-UTF-8 databases work as expected if the connection charset is still UTF-8, as the DBMS takes
+care of converting the data to UTF-8.
+
+
+Affected Installations
+======================
+
+Any installation that uses the ``config.renderCharset`` TypoScript option with a different value than ``utf-8``.
+
+
+Migration
+=========
+
+Remove the TypoScript option from any TypoScript settings. If data sources (files, database input) are used that are
+different than UTF-8, a manual conversion via the CharsetConverter PHP class is needed.
\ No newline at end of file
diff --git a/typo3/sysext/core/Documentation/Changelog/master/Deprecation-73794-SearchFormController-utf8_to_currentCharsetAndTSFE-renderCharset.rst b/typo3/sysext/core/Documentation/Changelog/master/Deprecation-73794-SearchFormController-utf8_to_currentCharsetAndTSFE-renderCharset.rst
new file mode 100644 (file)
index 0000000..57dfbb1
--- /dev/null
@@ -0,0 +1,29 @@
+==========================================================================================
+Deprecation: #73794 - SearchFormController->utf8_to_currentCharset and TSFE->renderCharset
+==========================================================================================
+
+Description
+===========
+
+The public method ``SearchFormController->utf8_to_currentCharset()`` within indexed search has been marked as
+deprecated.
+
+The public property ``TypoScriptFrontendController->renderCharset`` has been marked as deprecated.
+
+
+Impact
+======
+
+Calling the method above will result in a deprecation log entry.
+
+
+Affected Installations
+======================
+
+Any installation that extends Indexed Search and uses this method, and any extension that accesses ``TSFE->renderCharset`` via TypoScript or PHP directly.
+
+
+Migration
+=========
+
+Remove any calls to the method and the public property.
\ No newline at end of file
index 0d48bb5..109e6f7 100644 (file)
@@ -153,14 +153,13 @@ class LocalizationUtility
         }
         $locallangPathAndFilename = 'EXT:' . GeneralUtility::camelCaseToLowerCaseUnderscored($extensionName) . '/' . self::$locallangPath . 'locallang.xlf';
         self::setLanguageKeys();
-        $renderCharset = TYPO3_MODE === 'FE' ? self::getTypoScriptFrontendController()->renderCharset : 'utf-8';
 
         /** @var $languageFactory LocalizationFactory */
         $languageFactory = GeneralUtility::makeInstance(LocalizationFactory::class);
 
-        self::$LOCAL_LANG[$extensionName] = $languageFactory->getParsedData($locallangPathAndFilename, self::$languageKey, $renderCharset);
+        self::$LOCAL_LANG[$extensionName] = $languageFactory->getParsedData($locallangPathAndFilename, self::$languageKey, 'utf-8');
         foreach (self::$alternativeLanguageKeys as $language) {
-            $tempLL = $languageFactory->getParsedData($locallangPathAndFilename, $language, $renderCharset);
+            $tempLL = $languageFactory->getParsedData($locallangPathAndFilename, $language, 'utf-8');
             if (self::$languageKey !== 'default' && isset($tempLL[$language])) {
                 self::$LOCAL_LANG[$extensionName][$language] = $tempLL[$language];
             }
index 2e0828a..c709e40 100644 (file)
@@ -129,7 +129,7 @@ class CropViewHelper extends AbstractViewHelper
     }
 
     /**
-     * Sets the global variables $GLOBALS['TSFE']->csConvObj and $GLOBALS['TSFE']->renderCharset in Backend mode
+     * Sets the global variable $GLOBALS['TSFE']->csConvObj in Backend mode
      * This somewhat hacky work around is currently needed because the crop() and cropHTML() functions of
      * ContentObjectRenderer rely on those variables to be set
      *
@@ -143,10 +143,6 @@ class CropViewHelper extends AbstractViewHelper
         if (!is_object($GLOBALS['TSFE']->csConvObj)) {
             $GLOBALS['TSFE']->csConvObj = GeneralUtility::makeInstance(CharsetConverter::class);
         }
-        // preparing renderCharset
-        if (!is_object($GLOBALS['TSFE']->renderCharset)) {
-            $GLOBALS['TSFE']->renderCharset = 'utf-8';
-        }
     }
 
     /**
index e93b547..073158e 100644 (file)
@@ -23,9 +23,8 @@ abstract class AbstractFilter
 {
     protected function convertCase($value, $case)
     {
-        $frontendController = $this->getTypoScriptFrontendController();
         return $this->getCharsetConverter()->conv_case(
-            $frontendController->renderCharset,
+            'utf-8',
             $value,
             $case
         );
@@ -38,12 +37,4 @@ abstract class AbstractFilter
     {
         return GeneralUtility::makeInstance(CharsetConverter::class);
     }
-
-    /**
-     * @return \TYPO3\CMS\Frontend\Controller\TypoScriptFrontendController
-     */
-    protected function getTypoScriptFrontendController()
-    {
-        return $GLOBALS['TSFE'];
-    }
 }
index a1cde3e..57063cd 100644 (file)
@@ -339,7 +339,7 @@ class MailPostProcessor extends AbstractPostProcessor implements PostProcessorIn
      * Set the default character set used
      *
      * Respect formMailCharset if it was set, otherwise use metaCharset for mail
-     * if different from renderCharset
+     * if different than utf-8
      *
      * @return void
      */
@@ -348,7 +348,7 @@ class MailPostProcessor extends AbstractPostProcessor implements PostProcessorIn
         $characterSet = null;
         if ($GLOBALS['TSFE']->config['config']['formMailCharset']) {
             $characterSet = $GLOBALS['TSFE']->csConvObj->parse_charset($GLOBALS['TSFE']->config['config']['formMailCharset']);
-        } elseif ($GLOBALS['TSFE']->metaCharset != $GLOBALS['TSFE']->renderCharset) {
+        } elseif ($GLOBALS['TSFE']->metaCharset !== 'utf-8') {
             $characterSet = $GLOBALS['TSFE']->metaCharset;
         }
         if ($characterSet) {
index 61f8ae4..820a1c4 100644 (file)
@@ -29,7 +29,6 @@ class LowerCaseFilterTest extends \TYPO3\CMS\Core\Tests\UnitTestCase
         $this->subject = new \TYPO3\CMS\Form\Domain\Filter\LowerCaseFilter();
         $GLOBALS['TSFE'] = new \stdClass();
         $GLOBALS['TSFE']->csConvObj = new \TYPO3\CMS\Core\Charset\CharsetConverter();
-        $GLOBALS['TSFE']->renderCharset = 'utf-8';
     }
 
     public function dataProvider()
index cd73649..f245149 100644 (file)
@@ -35,7 +35,6 @@ class TitleCaseFilterTest extends \TYPO3\CMS\Core\Tests\UnitTestCase
         $this->subject = new TitleCaseFilter();
         $GLOBALS['TSFE'] = new \stdClass();
         $GLOBALS['TSFE']->csConvObj = new CharsetConverter();
-        $GLOBALS['TSFE']->renderCharset = 'utf-8';
     }
 
     /**
index b616f5e..673acb3 100644 (file)
@@ -32,7 +32,6 @@ class UpperCaseFilterTest extends \TYPO3\CMS\Core\Tests\UnitTestCase
         $this->subject = new \TYPO3\CMS\Form\Domain\Filter\UpperCaseFilter();
         $GLOBALS['TSFE'] = new \stdClass();
         $GLOBALS['TSFE']->csConvObj = new \TYPO3\CMS\Core\Charset\CharsetConverter();
-        $GLOBALS['TSFE']->renderCharset = 'utf-8';
     }
 
     public function stringProvider()
index ee21e9b..dc770ab 100644 (file)
@@ -15,6 +15,7 @@ namespace TYPO3\CMS\Frontend\ContentObject;
  */
 
 use TYPO3\CMS\Core\Cache\CacheManager;
+use TYPO3\CMS\Core\Charset\CharsetConverter;
 use TYPO3\CMS\Core\FrontendEditing\FrontendEditingController;
 use TYPO3\CMS\Core\Html\HtmlParser;
 use TYPO3\CMS\Core\Log\LogManager;
@@ -2531,7 +2532,14 @@ class ContentObjectRenderer
      */
     public function stdWrap_csConv($content = '', $conf = array())
     {
-        return $this->getTypoScriptFrontendController()->csConv($content, $conf['csConv']);
+        if (!empty($conf['csConv'])) {
+            /** @var CharsetConverter $charsetConverter */
+            $charsetConverter = GeneralUtility::makeInstance(CharsetConverter::class);
+            $output = $charsetConverter->conv($content, $charsetConverter->parse_charset($conf['csConv']), 'utf-8');
+            return $output ?: $content;
+        } else {
+            return $content;
+        }
     }
 
     /**
@@ -2723,7 +2731,10 @@ class ContentObjectRenderer
         $content = (string)$content === '' ? $GLOBALS['EXEC_TIME'] : (int)$content;
         $content = $conf['strftime.']['GMT'] ? gmstrftime($conf['strftime'], $content) : strftime($conf['strftime'], $content);
         if (!empty($conf['strftime.']['charset'])) {
-            $content = $this->getTypoScriptFrontendController()->csConv($content, $conf['strftime.']['charset']);
+            /** @var CharsetConverter $charsetConverter */
+            $charsetConverter = GeneralUtility::makeInstance(CharsetConverter::class);
+            $output = $charsetConverter->conv($content, $charsetConverter->parse_charset($conf['strftime.']['charset']), 'utf-8');
+            return $output ?: $content;
         }
         return $content;
     }
@@ -3819,9 +3830,9 @@ class ContentObjectRenderer
         $tsfe = $this->getTypoScriptFrontendController();
         $options = GeneralUtility::intExplode(',', $options . ',');
         if ($options[1]) {
-            return $tsfe->csConvObj->substr($tsfe->renderCharset, $content, $options[0], $options[1]);
+            return $tsfe->csConvObj->substr('utf-8', $content, $options[0], $options[1]);
         } else {
-            return $tsfe->csConvObj->substr($tsfe->renderCharset, $content, $options[0]);
+            return $tsfe->csConvObj->substr('utf-8', $content, $options[0]);
         }
     }
 
@@ -3841,17 +3852,18 @@ class ContentObjectRenderer
         $afterstring = trim($options[1]);
         $crop2space = trim($options[2]);
         if ($chars) {
-            $tsfe = $this->getTypoScriptFrontendController();
-            if ($tsfe->csConvObj->strlen($tsfe->renderCharset, $content) > abs($chars)) {
+            /** @var CharsetConverter $charsetConverter */
+            $charsetConverter = GeneralUtility::makeInstance(CharsetConverter::class);
+            if ($charsetConverter->strlen('utf-8', $content) > abs($chars)) {
                 $truncatePosition = false;
                 if ($chars < 0) {
-                    $content = $tsfe->csConvObj->substr($tsfe->renderCharset, $content, $chars);
+                    $content = $charsetConverter->substr('utf-8', $content, $chars);
                     if ($crop2space) {
                         $truncatePosition = strpos($content, ' ');
                     }
                     $content = $truncatePosition ? $afterstring . substr($content, $truncatePosition) : $afterstring . $content;
                 } else {
-                    $content = $tsfe->csConvObj->substr($tsfe->renderCharset, $content, 0, $chars);
+                    $content = $charsetConverter->substr('utf-8', $content, 0, $chars);
                     if ($crop2space) {
                         $truncatePosition = strrpos($content, ' ');
                     }
@@ -3933,12 +3945,13 @@ class ContentObjectRenderer
         $strLen = 0;
         // This is the offset of the content item which was cropped.
         $croppedOffset = null;
-        $tsfe = $this->getTypoScriptFrontendController();
+        /** @var CharsetConverter $charsetConverter */
+        $charsetConverter = GeneralUtility::makeInstance(CharsetConverter::class);
         $countSplittedContent = count($splittedContent);
         for ($offset = 0; $offset < $countSplittedContent; $offset++) {
             if ($offset % 2 === 0) {
-                $tempContent = $tsfe->csConvObj->conv($splittedContent[$offset], $tsfe->renderCharset, 'utf-8');
-                $thisStrLen = $tsfe->csConvObj->strlen('utf-8', html_entity_decode($tempContent, ENT_COMPAT, 'UTF-8'));
+                $tempContent = $splittedContent[$offset];
+                $thisStrLen = $charsetConverter->strlen('utf-8', html_entity_decode($tempContent, ENT_COMPAT, 'UTF-8'));
                 if ($strLen + $thisStrLen > $absChars) {
                     $croppedOffset = $offset;
                     $cropPosition = $absChars - $strLen;
@@ -3960,7 +3973,7 @@ class ContentObjectRenderer
                             }
                         }
                     }
-                    $splittedContent[$offset] = $tsfe->csConvObj->conv($tempContent, 'utf-8', $tsfe->renderCharset);
+                    $splittedContent[$offset] = $tempContent;
                     break;
                 } else {
                     $strLen += $thisStrLen;
@@ -6762,19 +6775,19 @@ class ContentObjectRenderer
         $tsfe = $this->getTypoScriptFrontendController();
         switch (strtolower($case)) {
             case 'upper':
-                $theValue = $tsfe->csConvObj->conv_case($tsfe->renderCharset, $theValue, 'toUpper');
+                $theValue = $tsfe->csConvObj->conv_case('utf-8', $theValue, 'toUpper');
                 break;
             case 'lower':
-                $theValue = $tsfe->csConvObj->conv_case($tsfe->renderCharset, $theValue, 'toLower');
+                $theValue = $tsfe->csConvObj->conv_case('utf-8', $theValue, 'toLower');
                 break;
             case 'capitalize':
                 $theValue = ucwords($theValue);
                 break;
             case 'ucfirst':
-                $theValue = $tsfe->csConvObj->convCaseFirst($tsfe->renderCharset, $theValue, 'toUpper');
+                $theValue = $tsfe->csConvObj->convCaseFirst('utf-8', $theValue, 'toUpper');
                 break;
             case 'lcfirst':
-                $theValue = $tsfe->csConvObj->convCaseFirst($tsfe->renderCharset, $theValue, 'toLower');
+                $theValue = $tsfe->csConvObj->convCaseFirst('utf-8', $theValue, 'toLower');
                 break;
             case 'uppercamelcase':
                 $theValue = GeneralUtility::underscoredToUpperCamelCase($theValue);
index 6d01b33..9e0f10e 100644 (file)
@@ -730,20 +730,22 @@ class TypoScriptFrontendController
 
     /**
      * The default charset used in the frontend if nothing else is set.
+     * @deprecated since TYPO3 v8, will be removed in TYPO3 v9
      * @var string
      */
     public $defaultCharSet = 'utf-8';
 
     /**
      * Internal charset of the frontend during rendering. (Default: UTF-8)
+     * @deprecated since TYPO3 v8, will be removed in TYPO3 v9
      * @var string
      */
     public $renderCharset = 'utf-8';
 
     /**
      * Output charset of the websites content. This is the charset found in the
-     * header, meta tag etc. If different from $renderCharset a conversion
-     * happens before output to browser. Defaults to ->renderCharset if not set.
+     * header, meta tag etc. If different than utf-8 a conversion
+     * happens before output to browser. Defaults to utf-8.
      * @var string
      */
     public $metaCharset = 'utf-8';
@@ -2987,8 +2989,6 @@ class TypoScriptFrontendController
                If this message does not disappear within ' . $seconds . ' seconds, please reload.';
             $message = $this->config['config']['message_page_is_being_generated'];
             if ((string)$message !== '') {
-                // This page is always encoded as UTF-8
-                $message = $this->csConvObj->conv($message, $this->renderCharset, 'utf-8');
                 $message = str_replace('###TITLE###', $title, $message);
                 $message = str_replace('###REQUEST_URI###', $request_uri, $message);
             } else {
@@ -3243,8 +3243,8 @@ class TypoScriptFrontendController
         // Convert char-set for output: (should be BEFORE indexing of the content (changed 22/4 2005)),
         // because otherwise indexed search might convert from the wrong charset!
         // One thing is that the charset mentioned in the HTML header would be wrong since the output charset (metaCharset)
-        // has not been converted to from renderCharset. And indexed search will internally convert from metaCharset
-        // to renderCharset so the content MUST be in metaCharset already!
+        // has not been converted to from utf-8. And indexed search will internally convert from metaCharset
+        // to utf-8 so the content MUST be in metaCharset already!
         $this->content = $this->convOutputCharset($this->content);
         // Hook for indexing pages
         if (is_array($this->TYPO3_CONF_VARS['SC_OPTIONS']['tslib/class.tslib_fe.php']['pageIndexing'])) {
@@ -4259,7 +4259,7 @@ class TypoScriptFrontendController
 
         $localLanguage = array();
         foreach ($languages as $language) {
-            $tempLL = $languageFactory->getParsedData($fileRef, $language, $this->renderCharset);
+            $tempLL = $languageFactory->getParsedData($fileRef, $language, 'utf-8');
             $localLanguage['default'] = $tempLL['default'];
             if (!isset($localLanguage[$this->lang])) {
                 $localLanguage[$this->lang] = $localLanguage['default'];
@@ -4318,15 +4318,15 @@ class TypoScriptFrontendController
             }
         }
 
-        // Setting charsets:
-        $this->renderCharset = $this->csConvObj->parse_charset($this->config['config']['renderCharset'] ? $this->config['config']['renderCharset'] : 'utf-8');
         // Rendering charset of HTML page.
-        $this->metaCharset = $this->csConvObj->parse_charset($this->config['config']['metaCharset'] ? $this->config['config']['metaCharset'] : $this->renderCharset);
+        if ($this->config['config']['metaCharset']) {
+            $this->metaCharset = $this->csConvObj->parse_charset($this->config['config']['metaCharset']);
+        }
     }
 
     /**
      * Converts the charset of the input string if applicable.
-     * The "to" charset is determined by the currently used charset for the page which is "utf-8" by default or set by $GLOBALS['TSFE']->config['config']['renderCharset']
+     * The "to" charset is determined by the currently used charset for the page which is "utf-8" by default
      * Only if there is a difference between the two charsets will a conversion be made
      * The conversion is done real-time - no caching for performance at this point!
      *
@@ -4338,7 +4338,7 @@ class TypoScriptFrontendController
     public function csConv($str, $from = '')
     {
         if ($from) {
-            $output = $this->csConvObj->conv($str, $this->csConvObj->parse_charset($from), $this->renderCharset, 1);
+            $output = $this->csConvObj->conv($str, $this->csConvObj->parse_charset($from), 'utf-8');
             return $output ?: $str;
         } else {
             return $str;
@@ -4346,28 +4346,28 @@ class TypoScriptFrontendController
     }
 
     /**
-     * Converts input string from renderCharset to metaCharset IF the two charsets are different.
+     * Converts input string from utf-8 to metaCharset IF the two charsets are different.
      *
      * @param string $content Content to be converted.
      * @return string Converted content string.
      */
     public function convOutputCharset($content)
     {
-        if ($this->renderCharset != $this->metaCharset) {
-            $content = $this->csConvObj->conv($content, $this->renderCharset, $this->metaCharset, true);
+        if ($this->metaCharset !== 'utf-8') {
+            $content = $this->csConvObj->conv($content, 'utf-8', $this->metaCharset, true);
         }
         return $content;
     }
 
     /**
-     * Converts the $_POST array from metaCharset (page HTML charset from input form) to renderCharset (internal processing) IF the two charsets are different.
+     * Converts the $_POST array from metaCharset (page HTML charset from input form) to utf-8 (internal processing) IF the two charsets are different.
      *
      * @return void
      */
     public function convPOSTCharset()
     {
-        if ($this->renderCharset != $this->metaCharset && is_array($_POST) && !empty($_POST)) {
-            $this->csConvObj->convArray($_POST, $this->metaCharset, $this->renderCharset);
+        if ($this->metaCharset !== 'utf-8' && is_array($_POST) && !empty($_POST)) {
+            $this->csConvObj->convArray($_POST, $this->metaCharset, 'utf-8');
             $GLOBALS['HTTP_POST_VARS'] = $_POST;
         }
     }
index 5e0dfeb..04eb3e6 100644 (file)
@@ -185,7 +185,7 @@ class RequestHandler implements RequestHandlerInterface
         $this->controller->settingLocale();
         $this->timeTracker->pull();
 
-        // Convert POST data to internal "renderCharset" if different from the metaCharset
+        // Convert POST data to utf-8 for internal processing if metaCharset is different
         $this->controller->convPOSTCharset();
 
         $this->controller->initializeRedirectUrlHandlers();
index 338f4b7..3f709ce 100644 (file)
@@ -967,7 +967,7 @@ class AbstractPlugin
             /** @var $languageFactory LocalizationFactory */
             $languageFactory = GeneralUtility::makeInstance(LocalizationFactory::class);
             // Read the strings in the required charset (since TYPO3 4.2)
-            $this->LOCAL_LANG = $languageFactory->getParsedData($languageFilePath, $this->LLkey, $this->frontendController->renderCharset);
+            $this->LOCAL_LANG = $languageFactory->getParsedData($languageFilePath, $this->LLkey, 'utf-8');
             $alternativeLanguageKeys = GeneralUtility::trimExplode(',', $this->altLLkey, true);
             foreach ($alternativeLanguageKeys as $languageKey) {
                 $tempLL = $languageFactory->getParsedData($languageFilePath, $languageKey);
index 048898a..700e037 100644 (file)
@@ -44,7 +44,6 @@ class CaseContentObjectTest extends UnitTestCase
         $tsfe->page = array();
         $tsfe->sys_page = $this->getMock(PageRepository::class, array('getRawRecord'));
         $tsfe->csConvObj = new CharsetConverter();
-        $tsfe->renderCharset = 'utf-8';
         $GLOBALS['TSFE'] = $tsfe;
 
         $contentObjectRenderer = new ContentObjectRenderer();
index 3bb4b89..93076db 100755 (executable)
@@ -104,7 +104,6 @@ class ContentObjectRendererTest extends \TYPO3\CMS\Core\Tests\UnitTestCase
         $this->typoScriptFrontendControllerMock->page = array();
         $this->typoScriptFrontendControllerMock->sys_page = $pageRepositoryMock;
         $this->typoScriptFrontendControllerMock->csConvObj = new CharsetConverter();
-        $this->typoScriptFrontendControllerMock->renderCharset = 'utf-8';
         $GLOBALS['TSFE'] = $this->typoScriptFrontendControllerMock;
         $GLOBALS['TYPO3_DB'] = $this->getMock(\TYPO3\CMS\Core\Database\DatabaseConnection::class, array());
 
@@ -142,17 +141,16 @@ class ContentObjectRendererTest extends \TYPO3\CMS\Core\Tests\UnitTestCase
     }
 
     /**
-     * Converts the subject and the expected result into the target charset.
+     * Converts the subject and the expected result into utf-8.
      *
-     * @param string $charset the target charset
      * @param string $subject the subject, will be modified
      * @param string $expected the expected result, will be modified
      */
-    protected function handleCharset($charset, &$subject, &$expected)
+    protected function handleCharset(&$subject, &$expected)
     {
-        $GLOBALS['TSFE']->renderCharset = $charset;
-        $subject = $GLOBALS['TSFE']->csConvObj->conv($subject, 'iso-8859-1', $charset);
-        $expected = $GLOBALS['TSFE']->csConvObj->conv($expected, 'iso-8859-1', $charset);
+        $charsetConverter = new CharsetConverter();
+        $subject = $charsetConverter->conv($subject, 'iso-8859-1', 'utf-8');
+        $expected = $charsetConverter->conv($expected, 'iso-8859-1', 'utf-8');
     }
 
     /////////////////////////////////////////////
@@ -486,276 +484,228 @@ class ContentObjectRendererTest extends \TYPO3\CMS\Core\Tests\UnitTestCase
         $plainText = 'Kasper Sk' . chr(229) . 'rh' . chr(248) . 'j implemented the original version of the crop function.';
         $textWithMarkup = '<strong><a href="mailto:kasper@typo3.org">Kasper Sk' . chr(229) . 'rh' . chr(248) . 'j</a>' . ' implemented</strong> the original version of the crop function.';
         $textWithEntities = 'Kasper Sk&aring;rh&oslash;j implemented the; original ' . 'version of the crop function.';
-        $charsets = array('iso-8859-1', 'utf-8', 'ascii', 'big5');
-        $data = array();
-        foreach ($charsets as $charset) {
-            $data = array_merge($data, array(
-                $charset . ' plain text; 11|...' => array(
-                    '11|...',
-                    $plainText,
-                    'Kasper Sk' . chr(229) . 'r...',
-                    $charset
-                ),
-                $charset . ' plain text; -58|...' => array(
-                    '-58|...',
-                    $plainText,
-                    '...h' . chr(248) . 'j implemented the original version of the crop function.',
-                    $charset
-                ),
-                $charset . ' plain text; 4|...|1' => array(
-                    '4|...|1',
-                    $plainText,
-                    'Kasp...',
-                    $charset
-                ),
-                $charset . ' plain text; 20|...|1' => array(
-                    '20|...|1',
-                    $plainText,
-                    'Kasper Sk' . chr(229) . 'rh' . chr(248) . 'j...',
-                    $charset
-                ),
-                $charset . ' plain text; -5|...|1' => array(
-                    '-5|...|1',
-                    $plainText,
-                    '...tion.',
-                    $charset
-                ),
-                $charset . ' plain text; -49|...|1' => array(
-                    '-49|...|1',
-                    $plainText,
-                    '...the original version of the crop function.',
-                    $charset
-                ),
-                $charset . ' text with markup; 11|...' => array(
-                    '11|...',
-                    $textWithMarkup,
-                    '<strong><a href="mailto:kasper@typo3.org">Kasper Sk' . chr(229) . 'r...</a></strong>',
-                    $charset
-                ),
-                $charset . ' text with markup; 13|...' => array(
-                    '13|...',
-                    $textWithMarkup,
-                    '<strong><a href="mailto:kasper@typo3.org">Kasper Sk' . chr(229) . 'rh' . chr(248) . '...</a></strong>',
-                    $charset
-                ),
-                $charset . ' text with markup; 14|...' => array(
-                    '14|...',
-                    $textWithMarkup,
-                    '<strong><a href="mailto:kasper@typo3.org">Kasper Sk' . chr(229) . 'rh' . chr(248) . 'j</a>...</strong>',
-                    $charset
-                ),
-                $charset . ' text with markup; 15|...' => array(
-                    '15|...',
-                    $textWithMarkup,
-                    '<strong><a href="mailto:kasper@typo3.org">Kasper Sk' . chr(229) . 'rh' . chr(248) . 'j</a> ...</strong>',
-                    $charset
-                ),
-                $charset . ' text with markup; 29|...' => array(
-                    '29|...',
-                    $textWithMarkup,
-                    '<strong><a href="mailto:kasper@typo3.org">Kasper Sk' . chr(229) . 'rh' . chr(248) . 'j</a> implemented</strong> th...',
-                    $charset
-                ),
-                $charset . ' text with markup; -58|...' => array(
-                    '-58|...',
-                    $textWithMarkup,
-                    '<strong><a href="mailto:kasper@typo3.org">...h' . chr(248) . 'j</a> implemented</strong> the original version of the crop function.',
-                    $charset
-                ),
-                $charset . ' text with markup 4|...|1' => array(
-                    '4|...|1',
-                    $textWithMarkup,
-                    '<strong><a href="mailto:kasper@typo3.org">Kasp...</a></strong>',
-                    $charset
-                ),
-                $charset . ' text with markup; 11|...|1' => array(
-                    '11|...|1',
-                    $textWithMarkup,
-                    '<strong><a href="mailto:kasper@typo3.org">Kasper...</a></strong>',
-                    $charset
-                ),
-                $charset . ' text with markup; 13|...|1' => array(
-                    '13|...|1',
-                    $textWithMarkup,
-                    '<strong><a href="mailto:kasper@typo3.org">Kasper...</a></strong>',
-                    $charset
-                ),
-                $charset . ' text with markup; 14|...|1' => array(
-                    '14|...|1',
-                    $textWithMarkup,
-                    '<strong><a href="mailto:kasper@typo3.org">Kasper Sk' . chr(229) . 'rh' . chr(248) . 'j</a>...</strong>',
-                    $charset
-                ),
-                $charset . ' text with markup; 15|...|1' => array(
-                    '15|...|1',
-                    $textWithMarkup,
-                    '<strong><a href="mailto:kasper@typo3.org">Kasper Sk' . chr(229) . 'rh' . chr(248) . 'j</a>...</strong>',
-                    $charset
-                ),
-                $charset . ' text with markup; 29|...|1' => array(
-                    '29|...|1',
-                    $textWithMarkup,
-                    '<strong><a href="mailto:kasper@typo3.org">Kasper Sk' . chr(229) . 'rh' . chr(248) . 'j</a> implemented</strong>...',
-                    $charset
-                ),
-                $charset . ' text with markup; -66|...|1' => array(
-                    '-66|...|1',
-                    $textWithMarkup,
-                    '<strong><a href="mailto:kasper@typo3.org">...Sk' . chr(229) . 'rh' . chr(248) . 'j</a> implemented</strong> the original version of the crop function.',
-                    $charset
-                ),
-                $charset . ' text with entities 9|...' => array(
-                    '9|...',
-                    $textWithEntities,
-                    'Kasper Sk...',
-                    $charset
-                ),
-                $charset . ' text with entities 10|...' => array(
-                    '10|...',
-                    $textWithEntities,
-                    'Kasper Sk&aring;...',
-                    $charset
-                ),
-                $charset . ' text with entities 11|...' => array(
-                    '11|...',
-                    $textWithEntities,
-                    'Kasper Sk&aring;r...',
-                    $charset
-                ),
-                $charset . ' text with entities 13|...' => array(
-                    '13|...',
-                    $textWithEntities,
-                    'Kasper Sk&aring;rh&oslash;...',
-                    $charset
-                ),
-                $charset . ' text with entities 14|...' => array(
-                    '14|...',
-                    $textWithEntities,
-                    'Kasper Sk&aring;rh&oslash;j...',
-                    $charset
-                ),
-                $charset . ' text with entities 15|...' => array(
-                    '15|...',
-                    $textWithEntities,
-                    'Kasper Sk&aring;rh&oslash;j ...',
-                    $charset
-                ),
-                $charset . ' text with entities 16|...' => array(
-                    '16|...',
-                    $textWithEntities,
-                    'Kasper Sk&aring;rh&oslash;j i...',
-                    $charset
-                ),
-                $charset . ' text with entities -57|...' => array(
-                    '-57|...',
-                    $textWithEntities,
-                    '...j implemented the; original version of the crop function.',
-                    $charset
-                ),
-                $charset . ' text with entities -58|...' => array(
-                    '-58|...',
-                    $textWithEntities,
-                    '...&oslash;j implemented the; original version of the crop function.',
-                    $charset
-                ),
-                $charset . ' text with entities -59|...' => array(
-                    '-59|...',
-                    $textWithEntities,
-                    '...h&oslash;j implemented the; original version of the crop function.',
-                    $charset
-                ),
-                $charset . ' text with entities 4|...|1' => array(
-                    '4|...|1',
-                    $textWithEntities,
-                    'Kasp...',
-                    $charset
-                ),
-                $charset . ' text with entities 9|...|1' => array(
-                    '9|...|1',
-                    $textWithEntities,
-                    'Kasper...',
-                    $charset
-                ),
-                $charset . ' text with entities 10|...|1' => array(
-                    '10|...|1',
-                    $textWithEntities,
-                    'Kasper...',
-                    $charset
-                ),
-                $charset . ' text with entities 11|...|1' => array(
-                    '11|...|1',
-                    $textWithEntities,
-                    'Kasper...',
-                    $charset
-                ),
-                $charset . ' text with entities 13|...|1' => array(
-                    '13|...|1',
-                    $textWithEntities,
-                    'Kasper...',
-                    $charset
-                ),
-                $charset . ' text with entities 14|...|1' => array(
-                    '14|...|1',
-                    $textWithEntities,
-                    'Kasper Sk&aring;rh&oslash;j...',
-                    $charset
-                ),
-                $charset . ' text with entities 15|...|1' => array(
-                    '15|...|1',
-                    $textWithEntities,
-                    'Kasper Sk&aring;rh&oslash;j...',
-                    $charset
-                ),
-                $charset . ' text with entities 16|...|1' => array(
-                    '16|...|1',
-                    $textWithEntities,
-                    'Kasper Sk&aring;rh&oslash;j...',
-                    $charset
-                ),
-                $charset . ' text with entities -57|...|1' => array(
-                    '-57|...|1',
-                    $textWithEntities,
-                    '...implemented the; original version of the crop function.',
-                    $charset
-                ),
-                $charset . ' text with entities -58|...|1' => array(
-                    '-58|...|1',
-                    $textWithEntities,
-                    '...implemented the; original version of the crop function.',
-                    $charset
-                ),
-                $charset . ' text with entities -59|...|1' => array(
-                    '-59|...|1',
-                    $textWithEntities,
-                    '...implemented the; original version of the crop function.',
-                    $charset
-                ),
-                $charset . ' text with dash in html-element 28|...|1' => array(
-                    '28|...|1',
-                    'Some text with a link to <link email.address@example.org - mail "Open email window">my email.address@example.org</link> and text after it',
-                    'Some text with a link to <link email.address@example.org - mail "Open email window">my...</link>',
-                    $charset
-                ),
-                $charset . ' html elements with dashes in attributes' => array(
-                    '9',
-                    '<em data-foo="x">foobar</em>foobaz',
-                    '<em data-foo="x">foobar</em>foo',
-                    $charset
-                ),
-                $charset . ' html elements with iframe embedded 24|...|1' => array(
-                    '24|...|1',
-                    'Text with iframe <iframe src="//what.ever/"></iframe> and text after it',
-                    'Text with iframe <iframe src="//what.ever/"></iframe> and...',
-                    $charset
-                ),
-                $charset . ' html elements with script tag embedded 24|...|1' => array(
-                    '24|...|1',
-                    'Text with script <script>alert(\'foo\');</script> and text after it',
-                    'Text with script <script>alert(\'foo\');</script> and...',
-                    $charset
-                ),
-            ));
-        }
+        $data = array(
+            'plain text; 11|...' => array(
+                '11|...',
+                $plainText,
+                'Kasper Sk' . chr(229) . 'r...'
+            ),
+            'plain text; -58|...' => array(
+                '-58|...',
+                $plainText,
+                '...h' . chr(248) . 'j implemented the original version of the crop function.'
+            ),
+            'plain text; 4|...|1' => array(
+                '4|...|1',
+                $plainText,
+                'Kasp...'
+            ),
+            'plain text; 20|...|1' => array(
+                '20|...|1',
+                $plainText,
+                'Kasper Sk' . chr(229) . 'rh' . chr(248) . 'j...'
+            ),
+            'plain text; -5|...|1' => array(
+                '-5|...|1',
+                $plainText,
+                '...tion.'
+            ),
+            'plain text; -49|...|1' => array(
+                '-49|...|1',
+                $plainText,
+                '...the original version of the crop function.'
+            ),
+            'text with markup; 11|...' => array(
+                '11|...',
+                $textWithMarkup,
+                '<strong><a href="mailto:kasper@typo3.org">Kasper Sk' . chr(229) . 'r...</a></strong>'
+            ),
+            'text with markup; 13|...' => array(
+                '13|...',
+                $textWithMarkup,
+                '<strong><a href="mailto:kasper@typo3.org">Kasper Sk' . chr(229) . 'rh' . chr(248) . '...</a></strong>'
+            ),
+            'text with markup; 14|...' => array(
+                '14|...',
+                $textWithMarkup,
+                '<strong><a href="mailto:kasper@typo3.org">Kasper Sk' . chr(229) . 'rh' . chr(248) . 'j</a>...</strong>'
+            ),
+            'text with markup; 15|...' => array(
+                '15|...',
+                $textWithMarkup,
+                '<strong><a href="mailto:kasper@typo3.org">Kasper Sk' . chr(229) . 'rh' . chr(248) . 'j</a> ...</strong>'
+            ),
+            'text with markup; 29|...' => array(
+                '29|...',
+                $textWithMarkup,
+                '<strong><a href="mailto:kasper@typo3.org">Kasper Sk' . chr(229) . 'rh' . chr(248) . 'j</a> implemented</strong> th...'
+            ),
+            'text with markup; -58|...' => array(
+                '-58|...',
+                $textWithMarkup,
+                '<strong><a href="mailto:kasper@typo3.org">...h' . chr(248) . 'j</a> implemented</strong> the original version of the crop function.'
+            ),
+            'text with markup 4|...|1' => array(
+                '4|...|1',
+                $textWithMarkup,
+                '<strong><a href="mailto:kasper@typo3.org">Kasp...</a></strong>'
+            ),
+            'text with markup; 11|...|1' => array(
+                '11|...|1',
+                $textWithMarkup,
+                '<strong><a href="mailto:kasper@typo3.org">Kasper...</a></strong>'
+            ),
+            'text with markup; 13|...|1' => array(
+                '13|...|1',
+                $textWithMarkup,
+                '<strong><a href="mailto:kasper@typo3.org">Kasper...</a></strong>'
+            ),
+            'text with markup; 14|...|1' => array(
+                '14|...|1',
+                $textWithMarkup,
+                '<strong><a href="mailto:kasper@typo3.org">Kasper Sk' . chr(229) . 'rh' . chr(248) . 'j</a>...</strong>'
+            ),
+            'text with markup; 15|...|1' => array(
+                '15|...|1',
+                $textWithMarkup,
+                '<strong><a href="mailto:kasper@typo3.org">Kasper Sk' . chr(229) . 'rh' . chr(248) . 'j</a>...</strong>'
+            ),
+            'text with markup; 29|...|1' => array(
+                '29|...|1',
+                $textWithMarkup,
+                '<strong><a href="mailto:kasper@typo3.org">Kasper Sk' . chr(229) . 'rh' . chr(248) . 'j</a> implemented</strong>...'
+            ),
+            'text with markup; -66|...|1' => array(
+                '-66|...|1',
+                $textWithMarkup,
+                '<strong><a href="mailto:kasper@typo3.org">...Sk' . chr(229) . 'rh' . chr(248) . 'j</a> implemented</strong> the original version of the crop function.'
+            ),
+            'text with entities 9|...' => array(
+                '9|...',
+                $textWithEntities,
+                'Kasper Sk...'
+            ),
+            'text with entities 10|...' => array(
+                '10|...',
+                $textWithEntities,
+                'Kasper Sk&aring;...'
+            ),
+            'text with entities 11|...' => array(
+                '11|...',
+                $textWithEntities,
+                'Kasper Sk&aring;r...'
+            ),
+            'text with entities 13|...' => array(
+                '13|...',
+                $textWithEntities,
+                'Kasper Sk&aring;rh&oslash;...'
+            ),
+            'text with entities 14|...' => array(
+                '14|...',
+                $textWithEntities,
+                'Kasper Sk&aring;rh&oslash;j...'
+            ),
+            'text with entities 15|...' => array(
+                '15|...',
+                $textWithEntities,
+                'Kasper Sk&aring;rh&oslash;j ...'
+            ),
+            'text with entities 16|...' => array(
+                '16|...',
+                $textWithEntities,
+                'Kasper Sk&aring;rh&oslash;j i...'
+            ),
+            'text with entities -57|...' => array(
+                '-57|...',
+                $textWithEntities,
+                '...j implemented the; original version of the crop function.'
+            ),
+            'text with entities -58|...' => array(
+                '-58|...',
+                $textWithEntities,
+                '...&oslash;j implemented the; original version of the crop function.'
+            ),
+            'text with entities -59|...' => array(
+                '-59|...',
+                $textWithEntities,
+                '...h&oslash;j implemented the; original version of the crop function.'
+            ),
+            'text with entities 4|...|1' => array(
+                '4|...|1',
+                $textWithEntities,
+                'Kasp...'
+            ),
+            'text with entities 9|...|1' => array(
+                '9|...|1',
+                $textWithEntities,
+                'Kasper...'
+            ),
+            'text with entities 10|...|1' => array(
+                '10|...|1',
+                $textWithEntities,
+                'Kasper...'
+            ),
+            'text with entities 11|...|1' => array(
+                '11|...|1',
+                $textWithEntities,
+                'Kasper...'
+            ),
+            'text with entities 13|...|1' => array(
+                '13|...|1',
+                $textWithEntities,
+                'Kasper...'
+            ),
+            'text with entities 14|...|1' => array(
+                '14|...|1',
+                $textWithEntities,
+                'Kasper Sk&aring;rh&oslash;j...'
+            ),
+            'text with entities 15|...|1' => array(
+                '15|...|1',
+                $textWithEntities,
+                'Kasper Sk&aring;rh&oslash;j...'
+            ),
+            'text with entities 16|...|1' => array(
+                '16|...|1',
+                $textWithEntities,
+                'Kasper Sk&aring;rh&oslash;j...'
+            ),
+            'text with entities -57|...|1' => array(
+                '-57|...|1',
+                $textWithEntities,
+                '...implemented the; original version of the crop function.'
+            ),
+            'text with entities -58|...|1' => array(
+                '-58|...|1',
+                $textWithEntities,
+                '...implemented the; original version of the crop function.'
+            ),
+            'text with entities -59|...|1' => array(
+                '-59|...|1',
+                $textWithEntities,
+                '...implemented the; original version of the crop function.'
+            ),
+            'text with dash in html-element 28|...|1' => array(
+                '28|...|1',
+                'Some text with a link to <link email.address@example.org - mail "Open email window">my email.address@example.org</link> and text after it',
+                'Some text with a link to <link email.address@example.org - mail "Open email window">my...</link>'
+            ),
+            'html elements with dashes in attributes' => array(
+                '9',
+                '<em data-foo="x">foobar</em>foobaz',
+                '<em data-foo="x">foobar</em>foo'
+            ),
+            'html elements with iframe embedded 24|...|1' => array(
+                '24|...|1',
+                'Text with iframe <iframe src="//what.ever/"></iframe> and text after it',
+                'Text with iframe <iframe src="//what.ever/"></iframe> and...'
+            ),
+            'html elements with script tag embedded 24|...|1' => array(
+                '24|...|1',
+                'Text with script <script>alert(\'foo\');</script> and text after it',
+                'Text with script <script>alert(\'foo\');</script> and...'
+            )
+        );
         return $data;
     }
 
@@ -767,12 +717,11 @@ class ContentObjectRendererTest extends \TYPO3\CMS\Core\Tests\UnitTestCase
      * @param string $settings
      * @param string $subject the string to crop
      * @param string $expected the expected cropped result
-     * @param string $charset the charset that will be set as renderCharset
      */
-    public function cropHtmlWithDataProvider($settings, $subject, $expected, $charset)
+    public function cropHtmlWithDataProvider($settings, $subject, $expected)
     {
-        $this->handleCharset($charset, $subject, $expected);
-        $this->assertEquals($expected, $this->subject->cropHTML($subject, $settings), 'cropHTML failed with settings: "' . $settings . '" and charset "' . $charset . '"');
+        $this->handleCharset($subject, $expected);
+        $this->assertEquals($expected, $this->subject->cropHTML($subject, $settings), 'cropHTML failed with settings: "' . $settings . '"');
     }
 
     /**
@@ -783,7 +732,6 @@ class ContentObjectRendererTest extends \TYPO3\CMS\Core\Tests\UnitTestCase
      */
     public function cropHtmlWorksWithComplexContent()
     {
-        $GLOBALS['TSFE']->renderCharset = 'iso-8859-1';
         $input =
             '<h1>Blog Example</h1>' . LF .
             '<hr>' . LF .
@@ -2646,7 +2594,7 @@ class ContentObjectRendererTest extends \TYPO3\CMS\Core\Tests\UnitTestCase
      */
     public function getDataWithTypeTsfe()
     {
-        $this->assertEquals($GLOBALS['TSFE']->renderCharset, $this->subject->getData('tsfe:renderCharset'));
+        $this->assertEquals($GLOBALS['TSFE']->metaCharset, $this->subject->getData('tsfe:metaCharset'));
     }
 
     /**
@@ -2769,7 +2717,7 @@ class ContentObjectRendererTest extends \TYPO3\CMS\Core\Tests\UnitTestCase
      */
     public function getDataWithTypeGlobal()
     {
-        $this->assertEquals($GLOBALS['TSFE']->renderCharset, $this->subject->getData('global:TSFE|renderCharset'));
+        $this->assertEquals($GLOBALS['TSFE']->metaCharset, $this->subject->getData('global:TSFE|metaCharset'));
     }
 
     /**
index ae95d04..a40203b 100644 (file)
@@ -57,7 +57,6 @@ class FilesContentObjectTest extends \TYPO3\CMS\Core\Tests\UnitTestCase
         $this->tsfe->sys_page = $sysPageMock;
         $GLOBALS['TSFE'] = $this->tsfe;
         $GLOBALS['TSFE']->csConvObj = new CharsetConverter();
-        $GLOBALS['TSFE']->renderCharset = 'utf-8';
 
         $contentObjectRenderer = new ContentObjectRenderer();
         $contentObjectRenderer->setContentObjectClassMap(array(
index 3d6ffd5..937116a 100644 (file)
@@ -672,7 +672,6 @@ class SearchController extends \TYPO3\CMS\Extbase\Mvc\Controller\ActionControlle
                 $outputStr = htmlspecialchars($outputStr);
             }
             $output = $outputStr ?: $markedSW;
-            $output = $GLOBALS['TSFE']->csConv($output, 'utf-8');
         } else {
             $output = '<span class="noResume">' . LocalizationUtility::translate('result.noResume', 'IndexedSearch') . '</span>';
         }
@@ -834,10 +833,10 @@ class SearchController extends \TYPO3\CMS\Extbase\Mvc\Controller\ActionControlle
                     array('|', 'OR'),
                     array('-', 'AND NOT'),
                     // Add operators for various languages
-                    // Converts the operators to UTF-8 and lowercase
-                    array($this->charsetConverter->conv_case('utf-8', $this->charsetConverter->conv(LocalizationUtility::translate('localizedOperandAnd', 'IndexedSearch'), $GLOBALS['TSFE']->renderCharset, 'utf-8'), 'toLower'), 'AND'),
-                    array($this->charsetConverter->conv_case('utf-8', $this->charsetConverter->conv(LocalizationUtility::translate('localizedOperandOr', 'IndexedSearch'), $GLOBALS['TSFE']->renderCharset, 'utf-8'), 'toLower'), 'OR'),
-                    array($this->charsetConverter->conv_case('utf-8', $this->charsetConverter->conv(LocalizationUtility::translate('localizedOperandNot', 'IndexedSearch'), $GLOBALS['TSFE']->renderCharset, 'utf-8'), 'toLower'), 'AND NOT')
+                    // Converts the operators to lowercase
+                    array($this->charsetConverter->conv_case('utf-8', LocalizationUtility::translate('localizedOperandAnd', 'IndexedSearch'), 'toLower'), 'AND'),
+                    array($this->charsetConverter->conv_case('utf-8', LocalizationUtility::translate('localizedOperandOr', 'IndexedSearch'), 'toLower'), 'OR'),
+                    array($this->charsetConverter->conv_case('utf-8', LocalizationUtility::translate('localizedOperandNot', 'IndexedSearch'), 'toLower'), 'AND NOT')
                 );
                 $swordArray = \TYPO3\CMS\IndexedSearch\Utility\IndexedSearchUtility::getExplodedSearchString($searchWords, $defaultOperator == 1 ? 'OR' : 'AND', $operatorTranslateTable);
                 if (is_array($swordArray)) {
index 4735484..27afe22 100755 (executable)
@@ -339,10 +339,10 @@ class SearchFormController extends \TYPO3\CMS\Frontend\Plugin\AbstractPlugin
             }
         }
         // Add operators for various languages
-        // Converts the operators to UTF-8 and lowercase
-        $this->operator_translate_table[] = array($this->charsetConverter->conv_case('utf-8', $this->charsetConverter->conv($this->pi_getLL('local_operator_AND'), $this->frontendController->renderCharset, 'utf-8'), 'toLower'), 'AND');
-        $this->operator_translate_table[] = array($this->charsetConverter->conv_case('utf-8', $this->charsetConverter->conv($this->pi_getLL('local_operator_OR'), $this->frontendController->renderCharset, 'utf-8'), 'toLower'), 'OR');
-        $this->operator_translate_table[] = array($this->charsetConverter->conv_case('utf-8', $this->charsetConverter->conv($this->pi_getLL('local_operator_NOT'), $this->frontendController->renderCharset, 'utf-8'), 'toLower'), 'AND NOT');
+        // Converts the operators to lowercase
+        $this->operator_translate_table[] = array($this->charsetConverter->conv_case('utf-8', $this->pi_getLL('local_operator_AND'), 'toLower'), 'AND');
+        $this->operator_translate_table[] = array($this->charsetConverter->conv_case('utf-8', $this->pi_getLL('local_operator_OR'), 'toLower'), 'OR');
+        $this->operator_translate_table[] = array($this->charsetConverter->conv_case('utf-8', $this->pi_getLL('local_operator_NOT'), 'toLower'), 'AND NOT');
         // This is the id of the site root. This value may be a commalist of integer (prepared for this)
         $this->wholeSiteIdList = (int)$this->frontendController->config['rootLine'][0]['uid'];
         // Creating levels for section menu:
@@ -1761,17 +1761,17 @@ class SearchFormController extends \TYPO3\CMS\Frontend\Plugin\AbstractPlugin
             if ($c) {
                 switch ($v['oper']) {
                     case 'OR':
-                        $searchingFor .= ' ' . $this->pi_getLL('searchFor_or', '', true) . ' ' . $this->wrapSW($this->utf8_to_currentCharset($v['sword']));
+                        $searchingFor .= ' ' . $this->pi_getLL('searchFor_or', '', true) . ' ' . $this->wrapSW($v['sword']);
                         break;
                     case 'AND NOT':
-                        $searchingFor .= ' ' . $this->pi_getLL('searchFor_butNot', '', true) . ' ' . $this->wrapSW($this->utf8_to_currentCharset($v['sword']));
+                        $searchingFor .= ' ' . $this->pi_getLL('searchFor_butNot', '', true) . ' ' . $this->wrapSW($v['sword']);
                         break;
                     default:
                         // AND...
-                        $searchingFor .= ' ' . $this->pi_getLL('searchFor_and', '', true) . ' ' . $this->wrapSW($this->utf8_to_currentCharset($v['sword']));
+                        $searchingFor .= ' ' . $this->pi_getLL('searchFor_and', '', true) . ' ' . $this->wrapSW($v['sword']);
                 }
             } else {
-                $searchingFor = $this->pi_getLL('searchFor', '', true) . ' ' . $this->wrapSW($this->utf8_to_currentCharset($v['sword']));
+                $searchingFor = $this->pi_getLL('searchFor', '', true) . ' ' . $this->wrapSW($v['sword']);
             }
             $c++;
         }
@@ -1954,7 +1954,7 @@ class SearchFormController extends \TYPO3\CMS\Frontend\Plugin\AbstractPlugin
                 $outputStr = $this->charsetConverter->crop('utf-8', $row['item_description'], $lgd, $this->conf['results.']['summaryCropSignifier']);
                 $outputStr = htmlspecialchars($outputStr);
             }
-            $output = $this->utf8_to_currentCharset($outputStr ?: $markedSW);
+            $output = $outputStr ?: $markedSW;
         } else {
             $output = '<span class="noResume">' . $this->pi_getLL('res_noResume', '', true) . '</span>';
         }
@@ -2051,7 +2051,7 @@ class SearchFormController extends \TYPO3\CMS\Frontend\Plugin\AbstractPlugin
             }
         }
         $outputString = $this->charsetConverter->crop('utf-8', $row['item_title'], $this->conf['results.']['titleCropAfter'], $this->conf['results.']['titleCropSignifier']);
-        return $this->utf8_to_currentCharset($outputString) . $add;
+        return $outputString . $add;
     }
 
     /**
@@ -2310,9 +2310,11 @@ class SearchFormController extends \TYPO3\CMS\Frontend\Plugin\AbstractPlugin
      *
      * @param string String to convert (utf-8)
      * @return string Converted string (backend charset if different from utf-8)
+     * @deprecated since TYPO3 v8, will be removed in TYPO3 v9
      */
     public function utf8_to_currentCharset($str)
     {
+        GeneralUtility::logDeprecatedFunction();
         return $this->frontendController->csConv($str, 'utf-8');
     }
 
index 6a1b65b..849d675 100644 (file)
@@ -594,8 +594,8 @@ config.message_preview_workspace = <div class="previewbox">Displaying workspace
 
 Is used for a) HTML meta-tag, b) HTTP header (unless disabled with .disableCharsetHeader) and c) xhtml prologues (if available)
 
-If renderCharset and metaCharset are different the output content is automatically converted to metaCharset before output and likewise are values posted back to the page converted from metaCharset to renderCharset for internal processing. This conversion takes time of course so there is another good reason to use the same charset for both.]]></description>
-                       <default><![CDATA[value of ".renderCharset"]]></default>
+If metaCharset is different than utf-8 the output content is automatically converted to metaCharset before output and likewise are values posted back to the page converted from metaCharset to utf-8 for internal processing. This conversion takes time of course so there is another good reason to use the same charset for both.]]></description>
+                       <default><![CDATA[utf-8]]></default>
                </property>
                <property name="moveJsFromHeaderToFooter" type="boolean">
                        <description><![CDATA[
@@ -748,15 +748,6 @@ Example:
 $GLOBALS['TYPO3_CONF_VARS']['FE']['cssConcatenateHandler'] = \TYPO3\CMS\Core\Utility\ExtensionManagementUtility::extPath($_EXTKEY) . 'Classes/class.tx_myext_cssConcatenateHandler.php:tx_myext_cssConcatenateHandler->concatenateCss';]]></description>
                        <default><![CDATA[false]]></default>
                </property>
-               <property name="renderCharset" type="string">
-                       <description><![CDATA[Charset used for rendering internally of the page content. It is highly recommended that this value is the same as the charset of the content coming from the main data source (eg. the database). Thus you don't need to do any other conversion.
-All strings from locallang files and locale strings are (and should be) converted to "renderCharset" during rendering.
-
-If you need another output charset than the render charset, see "metaCharset" below.
-
-]]></description>
-                       <default><![CDATA[UTF-8]]></default>
-               </property>
                <property name="sendCacheHeaders" type="boolean">
                        <description><![CDATA[If set, TYPO3 will output cache-control headers to the client based mainly on ¬†whether the page was cached internally. This feature allows client browsers and/or reverse proxies to take load of TYPO3 websites.
 
@@ -1005,7 +996,7 @@ output = only the output code just before it's echoed out.]]></description>
                <property name="xmlprologue" type="string">
                        <description><![CDATA[If empty (not set) then the default XML 1.0 prologue is set, when the doctype is set to a known keyword (eg xhtml_11):
 
-<?xml version="1.0" encoding="[config.renderCharset]">
+<?xml version="1.0" encoding="utf-8">
 
 If set to one of the know keywords then a standard prologue will be set:
 "xml_10" XML 1.0 prologue (see above)
@@ -6948,7 +6939,7 @@ bytes.labels = " | K| M| G"]]></description>
 - lower: Converts to lower case
 - capitalize: Capitalize words
 
-Uses "renderCharset" for the operation.]]></description>
+Uses utf-8 for the operation.]]></description>
                        <default><![CDATA[
 ]]></default>
                </property>
@@ -6967,7 +6958,7 @@ Examples:
 -20 | ... => max 20 characters. If more, the value will be truncated to last 20 chars and appended with "..."
 20 | ... | 1 => max 20 characters. If more, the value will be truncated to last 20 chars and appended with "...". If the division is in the middle of a word, the remains of that word is removed.
 
-Uses "renderCharset" for the operation.]]></description>
+Uses utf-8 for the operation.]]></description>
                        <default><![CDATA[
 ]]></default>
                </property>
@@ -6979,7 +6970,7 @@ Note that stdWrap.crop should not be used if stdWrap.cropHTML is already used.]]
 ]]></default>
                </property>
                <property name="csConv" type="string">
-                       <description><![CDATA[Convert the charset of the string from the charset given as value to the current rendering charset of the frontend (renderCharset).]]></description>
+                       <description><![CDATA[Convert the charset of the string from the charset given as value to utf-8.]]></description>
                        <default><![CDATA[
 ]]></default>
                </property>
@@ -7494,7 +7485,7 @@ If set, a clear gif is not used by rather a <div> tag with a style-attribute set
 This formatting is useful if the locale is set in advance in the CONFIG-object. See this.
 
 Properties:
-.charset  : Can be set to the charset of the output string if you need to convert it to renderCharset. Default is to take the intelligently guessed charset from \TYPO3\CMS\Core\Charset\CharsetConverter.]]></description>
+.charset  : Can be set to the charset of the output string if you need to convert it to utf-8. Default is to take the intelligently guessed charset from \TYPO3\CMS\Core\Charset\CharsetConverter.]]></description>
                        <default><![CDATA[
 ]]></default>
                </property>
@@ -7506,7 +7497,7 @@ Properties:
                <property name="substring" type="stringList">
                        <description><![CDATA[Returns the substring with [p1] and [p2] send as the 2nd and 3rd parameter to the PHP substring function.
 
-Uses "renderCharset" for the operation.]]></description>
+Uses utf-8 as character set for the operation.]]></description>
                        <default><![CDATA[
 ]]></default>
                </property>
index c134a6f..6c9ae9c 100644 (file)
@@ -702,7 +702,6 @@ var typoscriptWords = {
        'removePrependedNumbers': 'reserved',
        'removeTags': 'reserved',
        'removeWrapping': 'reserved',
-       'renderCharset': 'reserved',
        'renderWrap': 'reserved',
        'reset': 'reserved',
        'resources': 'reserved',
@@ -2006,7 +2005,6 @@ var typoscriptWords = {
        'removePrependedNumbers': 'reserved',
        'removeTags': 'reserved',
        'removeWrapping': 'reserved',
-       'renderCharset': 'reserved',
        'renderWrap': 'reserved',
        'reset': 'reserved',
        'resources': 'reserved',