[BUGFIX] Words with special characters not added to personal dictionary
authorStanislas Rolland <typo3@sjbr.ca>
Fri, 6 Jul 2012 20:03:41 +0000 (16:03 -0400)
committerStanislas Rolland <typo3@sjbr.ca>
Fri, 6 Jul 2012 20:08:11 +0000 (22:08 +0200)
Problem: Words containing special characters (f. e. German umlauts)
can´t be added to the user´s personal dictionary.
Solution: Work around Aspell issue.

Change-Id: I4382cabc8d4d5190d5c27a0bcdb4ab212b2855eb
Releases: 4.5, 4.6, 4.7, 6.0
Resolves: #38653
Reviewed-on: http://review.typo3.org/12679
Reviewed-by: Stanislas Rolland
Tested-by: Stanislas Rolland
typo3/sysext/rtehtmlarea/pi1/class.tx_rtehtmlarea_pi1.php

index 5e216b5..1ee25d1 100644 (file)
  * @author Stanislas Rolland <typo3(arobas)sjbr.ca>
  */
 class tx_rtehtmlarea_pi1 {
-
+               // Instance of t3lib_cs
        protected $csConvObj;
-       var $extKey = 'rtehtmlarea'; // The extension key.
+               // The extension key
+       var $extKey = 'rtehtmlarea';
        var $siteUrl;
        var $charset = 'utf-8';
        var $parserCharset = 'utf-8';
@@ -51,7 +52,10 @@ class tx_rtehtmlarea_pi1 {
        var $filePrefix = 'rtehtmlarea_';
                // Pre-FAL backward compatibility
        protected $uploadFolder = 'uploads/tx_rtehtmlarea/';
-       var $personalDictsArg = '';
+               // Path to main dictionary
+       protected $mainDictionaryPath;
+               // Path to personal dictionary
+       protected $personalDictionaryPath;
        var $xmlCharacterData = '';
 
        /**
@@ -135,40 +139,16 @@ class tx_rtehtmlarea_pi1 {
                if($this->pspell_is_available && !$this->forceCommandMode) {
                        $this->pspell_link = pspell_new($this->dictionary, '', '', $this->parserCharset, $pspellModeFlag);
                }
-
-                       // Setting the path to user personal dicts, if any
-               if (t3lib_div::_POST('enablePersonalDicts') == 'true' && TYPO3_MODE == 'BE' && is_object($GLOBALS['BE_USER'])) {
-                       if ($GLOBALS['BE_USER']->user['uid']) {
-                               $personalDictionaryFolderName = 'BE_' . $GLOBALS['BE_USER']->user['uid'];
-                                       // Check for pre-FAL personal dictionary folder
-                               try {
-                                       $personalDictionaryFolder = t3lib_file_Factory::getInstance()->getFolderObjectFromCombinedIdentifier(
-                                               PATH_site . $this->uploadFolder . $personalDictionaryFolderName
-                                       );
-                               } catch (Exception $e) {
-                                       $personalDictionaryFolder = FALSE;
-                               }
-                                       // The personal dictionary folder is created in the user's default upload folder
-                                       // and named BE_(uid)_personaldictionary
-                               if (!$personalDictionaryFolder) {
-                                       $personalDictionaryFolderName .= '_personaldictionary';
-                                       $backendUserDefaultFolder = $GLOBALS['BE_USER']->getDefaultUploadFolder();
-                                       if ($backendUserDefaultFolder->hasFolder($personalDictionaryFolderName)) {
-                                               $personalDictionaryFolder = $backendUserDefaultFolder->getSubfolder($personalDictionaryFolderName);
-                                       } else {
-                                               $personalDictionaryFolder = $backendUserDefaultFolder->createFolder($personalDictionaryFolderName);
-                                       }
-                               }
-                               $personalDictionaryPath = PATH_site . rtrim($personalDictionaryFolder->getPublicUrl(), '/');
-                                       // Escape here for later use
-                               $this->personalDictsArg = ' --home-dir=' . escapeshellarg($personalDictionaryPath);
-                       }
-               }
+                       // Setting the path to main dictionary
+               $this->setMainDictionaryPath();
+                       // Setting the path to user personal dictionary, if any
+               $this->setPersonalDictionaryPath();
+               $this->fixPersonalDictionaryCharacterSet();
 
                $cmd = t3lib_div::_POST('cmd');
                if ($cmd == 'learn') {
                                // Only availble for BE_USERS, die silently if someone has gotten here by accident
-                       if (TYPO3_MODE !='BE' || !is_object($GLOBALS['BE_USER'])) {
+                       if (TYPO3_MODE !='BE' || !is_object($GLOBALS['BE_USER'])) {
                                die('');
                        }
                                // Updating the personal word list
@@ -178,35 +158,44 @@ class tx_rtehtmlarea_pi1 {
                        $to_r_list = $to_r_list ? $to_r_list : array();
                        header('Content-Type: text/plain; charset=' . strtoupper($this->parserCharset));
                        header('Pragma: no-cache');
-                       if($to_p_dict || $to_r_list) {
+                       if ($to_p_dict || $to_r_list) {
                                $tmpFileName = t3lib_div::tempnam($this->filePrefix);
-                               if($filehandle = fopen($tmpFileName, 'wb')) {
+                               $filehandle = fopen($tmpFileName, 'wb');
+                               if ($filehandle) {
+                                               // Get the character set of the main dictionary
+                                               // We need to convert the input into the character set of the main dictionary
+                                       $mainDictionaryCharacterSet = $this->getMainDictionaryCharacterSet();
+                                               // Write the personal words addition commands to the temporary file
                                        foreach ($to_p_dict as $personal_word) {
-                                               $cmd = '&' . $personal_word . LF;
-                                               echo $cmd;
+                                               $cmd = '&' . $this->csConvObj->conv($personal_word, $this->parserCharset, $mainDictionaryCharacterSet) . LF;
                                                fwrite($filehandle, $cmd, strlen($cmd));
                                        }
+                                               // Write the replacent pairs addition commands to the temporary file
                                        foreach ($to_r_list as $replace_pair) {
-                                               $cmd = '$$ra ' . $replace_pair[0] . ' , ' . $replace_pair[1] . LF;
-                                               echo $cmd;
+                                               $cmd = '$$ra ' . $this->csConvObj->conv($replace_pair[0], $this->parserCharset, $mainDictionaryCharacterSet) . ' , ' . $this->csConvObj->conv($replace_pair[1], $this->parserCharset, $mainDictionaryCharacterSet) . LF;
                                                fwrite($filehandle, $cmd, strlen($cmd));
                                        }
-                                       $cmd = "#\n";
-                                       echo $cmd;
+                                       $cmd = '#' . LF;
                                        fwrite($filehandle, $cmd, strlen($cmd));
+                                               // Assemble the Aspell command
+                                       $AspellCommand = ((TYPO3_OS === 'WIN') ? 'type ' : 'cat ') . escapeshellarg($tmpFileName) . ' | '
+                                               . $this->AspellDirectory
+                                               . ' -a --mode=none'
+                                               . ($this->personalDictionaryPath ? ' --home-dir=' . escapeshellarg($this->personalDictionaryPath) : '')
+                                               . ' --lang=' . escapeshellarg($this->dictionary)
+                                               . ' --encoding=' . escapeshellarg($mainDictionaryCharacterSet)
+                                               . ' 2>&1';
+                                       $AspellAnswer = shell_exec($AspellCommand);
+                                               // Close and delete the temporary file
                                        fclose($filehandle);
-                                               // $this->personalDictsArg has already been escapeshellarg()'ed above, it is an optional paramter and might be empty here
-                                       $catCommand = (TYPO3_OS == 'WIN') ? 'type' : 'cat';
-                                       $AspellCommand = $catCommand . ' ' . escapeshellarg($tmpFileName) . ' | ' . $this->AspellDirectory . ' -a --mode=none' . $this->personalDictsArg . ' --lang=' . escapeshellarg($this->dictionary) . ' --encoding=' . escapeshellarg($this->aspellEncoding) . ' 2>&1';
-                                       print $AspellCommand . LF;
-                                       print shell_exec($AspellCommand);
                                        t3lib_div::unlink_tempfile($tmpFileName);
-                                       echo('Personal word list was updated.');
+                                       echo 'Personal word list was updated.';
                                } else {
-                                       echo('SpellChecker tempfile open error.');
+                                       t3lib_div::sysLog('SpellChecker tempfile open error: ' . $tmpFileName, $this->extKey, t3lib_div::SYSLOG_SEVERITY_ERROR);
+                                       echo 'SpellChecker tempfile open error.';
                                }
                        } else {
-                               echo('Nothing to add to the personal word list.');
+                               echo 'Nothing to add to the personal word list.';
                        }
                        flush();
                        exit();
@@ -279,7 +268,98 @@ var selectedDictionary = "' . $this->dictionary . '";
                        echo $this->result;
                }
 
-       }  // end of function main
+       }
+
+       /**
+        * Sets the path to the main dictionary
+        *
+        * @return string path to the main dictionary
+        */
+       protected function setMainDictionaryPath() {
+               $this->mainDictionaryPath = trim(shell_exec($this->AspellDirectory . ' config dict-dir'));
+               return $this->mainDictionaryPath;
+       }
+
+       /**
+        * Gets the character set the main dictionary
+        *
+        * @return string character set the main dictionary
+        */
+       protected function getMainDictionaryCharacterSet() {
+               $characterSet = '';
+                       // Read the options of the dictionary
+               $dictionaryHandle = fopen($this->mainDictionaryPath . '/' . $this->dictionary . '.dat', 'rb');
+               $dictionaryContent = fread($dictionaryHandle, 500);
+               fclose($dictionaryHandle);
+                       // Get the line that contains the character set option
+               $dictionaryContent = preg_split('/charset\s*/', $dictionaryContent, 2);
+               if ($dictionaryContent[1]) {
+                               // Isolate the character set
+                       $dictionaryContent = t3lib_div::trimExplode(LF, $dictionaryContent[1]);
+                       $characterSet = $dictionaryContent[0];
+               }
+                       // Fix Aspell character set oddity (i.e. iso8859-1)
+               $characterSet = str_replace('iso', 'iso-', $characterSet);
+               $characterSet = str_replace('--', '-', $characterSet);
+               return $characterSet;
+       }
+
+       /**
+        * Sets the path to the personal dictionary
+        *
+        * @return string path to the personal dictionary
+        */
+       protected function setPersonalDictionaryPath() {
+               $this->personalDictionaryPath = '';
+               if (t3lib_div::_POST('enablePersonalDicts') == 'true' && TYPO3_MODE == 'BE' && is_object($GLOBALS['BE_USER'])) {
+                       if ($GLOBALS['BE_USER']->user['uid']) {
+                               $personalDictionaryFolderName = 'BE_' . $GLOBALS['BE_USER']->user['uid'];
+                                       // Check for pre-FAL personal dictionary folder
+                               try {
+                                       $personalDictionaryFolder = t3lib_file_Factory::getInstance()->getFolderObjectFromCombinedIdentifier(
+                                               PATH_site . $this->uploadFolder . $personalDictionaryFolderName
+                                       );
+                               } catch (Exception $e) {
+                                       $personalDictionaryFolder = FALSE;
+                               }
+                                       // The personal dictionary folder is created in the user's default upload folder and named BE_(uid)_personaldictionary
+                               if (!$personalDictionaryFolder) {
+                                       $personalDictionaryFolderName .= '_personaldictionary';
+                                       $backendUserDefaultFolder = $GLOBALS['BE_USER']->getDefaultUploadFolder();
+                                       if ($backendUserDefaultFolder->hasFolder($personalDictionaryFolderName)) {
+                                               $personalDictionaryFolder = $backendUserDefaultFolder->getSubfolder($personalDictionaryFolderName);
+                                       } else {
+                                               $personalDictionaryFolder = $backendUserDefaultFolder->createFolder($personalDictionaryFolderName);
+                                       }
+                               }
+                               $this->personalDictionaryPath = PATH_site . rtrim($personalDictionaryFolder->getPublicUrl(), '/');
+                       }
+               }
+               return $this->personalDictionaryPath;
+       }
+
+       /**
+        * Ensures that the personal dictionary is utf-8 encoded
+        *
+        * @return void
+        */
+       protected function fixPersonalDictionaryCharacterSet() {
+                       // Fix the options of the personl word list and of the replacement pairs files
+               $fileNames = array();
+               $fileNames[0] = $this->personalDictionaryPath . '/' . '.aspell.' . $this->dictionary . '.pws';
+               $fileNames[1] = $this->personalDictionaryPath . '/' . '.aspell.' . $this->dictionary . '.prepl';
+               foreach ($fileNames as $fileName) {
+                       if (file_exists($fileName)) {
+                               $fileContent = file_get_contents($fileName);
+                               $fileContent = explode(LF, $fileContent);
+                               if (strpos($fileContent[0], 'utf-8') === FALSE) {
+                                       $fileContent[0] .= ' utf-8';
+                               }
+                               $fileContent = implode(LF, $fileContent);
+                               file_put_contents($fileName, $fileContent);
+                       }
+               }
+       }
 
        function startHandler($xml_parser, $tag, $attributes) {
 
@@ -383,7 +463,7 @@ var selectedDictionary = "' . $this->dictionary . '";
                                                echo('SpellChecker tempfile close error');
                                        }
                                        $catCommand = (TYPO3_OS == 'WIN') ? 'type' : 'cat';
-                                       $AspellCommand = $catCommand . ' ' . escapeshellarg($tmpFileName) . ' | ' . $this->AspellDirectory . ' -a check --mode=none --sug-mode=' . escapeshellarg($this->pspellMode) . $this->personalDictsArg . ' --lang=' . escapeshellarg($this->dictionary) . ' --encoding=' . escapeshellarg($this->aspellEncoding) . ' 2>&1';
+                                       $AspellCommand = $catCommand . ' ' . escapeshellarg($tmpFileName) . ' | ' . $this->AspellDirectory . ' -a check --mode=none --sug-mode=' . escapeshellarg($this->pspellMode) . ($this->personalDictionaryPath ? ' --home-dir=' . escapeshellarg($this->personalDictionaryPath) : '') . ' --lang=' . escapeshellarg($this->dictionary) . ' --encoding=' . escapeshellarg($this->aspellEncoding) . ' 2>&1';
                                        $AspellAnswer = shell_exec($AspellCommand);
                                        $AspellResultLines = array();
                                        $AspellResultLines = t3lib_div::trimExplode(LF, $AspellAnswer, 1);