Commit 2412c79d authored by Benni Mack's avatar Benni Mack Committed by Georg Ringer
Browse files

[FEATURE] Add unified Locale class (IETF RFC 5646)

A new locale class is added in order to migrate the
code base and the configuration towards real locales in
form of RFC 5646 language tag (en-AT) and optional
script code ("Hans") and optional country / region
based on ISO 3166-1.

This unifies handling of locales instead of dealing
with "default" or other TYPO3-specific namings in
user-land code and configuration.

This locale functionality at first serves
to handle "label files" (XLF), but will
be further extended to also work with the locale
to be used in the SiteLanguage object to simplify
configuration and Date/Time Formatting based
on php-intl.

Thus, the first and foremost topic is
to allow to create custom "LanguageService"
objects out of a defined Locale instead of
a string.

The locale class contains the actual
locale (such as "de-AT" or "en") but
also allows to use the backwards-compatibility
for the labels internally. It also contains the
dependencies, so they do not need to be evaluated
in various places and makes the public facing API
much easier to understand.

Next to the introduction of the Locale
object, this patch also adapts various places which touch
current LanguageService instantiations to use the Locale.

Next Steps:
* SiteLanguage.locale should use the object, as Locale uses a
   \Stringable interface.
* Reduce optional settings in Site Language object and editing interface
* A new languageService ($GLOBALS[LANG]) could and
   should be instantiated where it is needed, and not
   by re-using the same $GLOBALS[LANG] as this is needed
   This Reduce usages on $GLOBALS[LANG] by building
   a new LanguageService object based on the Context everywhere.
* Ideally deprecate $GLOBALS[LANG] in TYPO3 v12 LTS.

Resolves: #99694
Releases: main
Change-Id: I9e2464699a2e53f4e3b136e0b66351f9f3aaf71f
Reviewed-on: https://review.typo3.org/c/Packages/TYPO3.CMS/+/77558


Tested-by: core-ci's avatarcore-ci <typo3@b13.com>
Tested-by: Georg Ringer's avatarGeorg Ringer <georg.ringer@gmail.com>
Tested-by: Andreas Fernandez's avatarAndreas Fernandez <a.fernandez@scripting-base.de>
Reviewed-by: Georg Ringer's avatarGeorg Ringer <georg.ringer@gmail.com>
Reviewed-by: Andreas Fernandez's avatarAndreas Fernandez <a.fernandez@scripting-base.de>
Reviewed-by: Christian Kuhn's avatarChristian Kuhn <lolli@schwarzbu.ch>
parent 87a40a01
......@@ -202,7 +202,7 @@ class LoginController
// If we found a $preferredBrowserLanguage, which is not the default language, while no user is logged in,
// initialize $this->getLanguageService()
if (empty($backendUser->user['uid'])) {
$languageService->init($preferredBrowserLanguage);
$languageService->init($this->locales->createLocale($preferredBrowserLanguage));
}
$this->setUpBasicPageRendererForBackend($this->pageRenderer, $this->extensionConfiguration, $request, $languageService);
......
......@@ -49,19 +49,12 @@ class LanguageService
*/
public string $lang = 'default';
protected ?Locale $locale = null;
/**
* If true, will show the key/location of labels in the backend.
*/
public bool $debugKey = false;
/**
* List of language dependencies for an actual language. This setting is used for local variants of a language
* that depend on their "main" language, like Brazilian Portuguese or Canadian French.
*
* @var array<int, string>
*/
protected array $languageDependencies = [];
/**
* @var string[][]
*/
......@@ -96,19 +89,17 @@ class LanguageService
* ```
*
* @throws \RuntimeException
* @param string $languageKey The language key (two character string from backend users profile)
* @param Locale|string $languageKey The language key (two character string from backend users profile)
* @internal use one of the factory methods instead
*/
public function init(string $languageKey): void
public function init(Locale|string $languageKey): void
{
// Find the requested language in this list based on the $languageKey
// Language is found. Configure it:
if ($this->locales->isValidLanguageKey($languageKey)) {
// The current language key
$this->lang = $languageKey;
$this->languageDependencies = array_merge([$languageKey], $this->locales->getLocaleDependencies($languageKey));
$this->languageDependencies = array_reverse($this->languageDependencies);
if ($languageKey instanceof Locale) {
$this->locale = $languageKey;
} else {
$this->locale = $this->locales->createLocale($languageKey);
}
$this->lang = $this->getTypo3LanguageKey();
}
/**
......@@ -181,7 +172,7 @@ class LanguageService
return $input;
}
$cacheIdentifier = 'labels_' . $this->lang . '_' . md5($input . '_' . (int)$this->debugKey);
$cacheIdentifier = 'labels_' . (string)$this->locale . '_' . md5($input . '_' . (int)$this->debugKey);
$cacheEntry = $this->runtimeCache->get($cacheIdentifier);
if ($cacheEntry !== false) {
return $cacheEntry;
......@@ -261,24 +252,36 @@ class LanguageService
*/
protected function readLLfile(string $fileRef): array
{
$cacheIdentifier = 'labels_file_' . md5($fileRef . $this->lang . json_encode($this->languageDependencies));
$cacheIdentifier = 'labels_file_' . md5($fileRef . (string)$this->locale);
$cacheEntry = $this->runtimeCache->get($cacheIdentifier);
if (is_array($cacheEntry)) {
return $cacheEntry;
}
$languages = $this->lang === 'default' ? ['default'] : $this->languageDependencies;
$mainLanguageKey = $this->getTypo3LanguageKey();
$localLanguage = [];
foreach ($languages as $language) {
$tempLL = $this->localizationFactory->getParsedData($fileRef, $language);
$allLocales = array_merge([$mainLanguageKey], $this->locale->getDependencies());
$allLocales = array_reverse($allLocales);
foreach ($allLocales as $locale) {
$tempLL = $this->localizationFactory->getParsedData($fileRef, $locale);
$localLanguage['default'] = $tempLL['default'];
if (!isset($localLanguage[$this->lang])) {
$localLanguage[$this->lang] = $localLanguage['default'];
if (!isset($localLanguage[$mainLanguageKey])) {
$localLanguage[$mainLanguageKey] = $localLanguage['default'];
}
if ($this->lang !== 'default' && isset($tempLL[$language])) {
// Merge current language labels onto labels from previous language
// This way we have a labels with fall back applied
ArrayUtility::mergeRecursiveWithOverrule($localLanguage[$this->lang], $tempLL[$language], true, false);
if ($mainLanguageKey !== 'default') {
// Fallback as long as TYPO3 supports "da_DK" and "da-DK"
if ((!isset($tempLL[$locale]) || $tempLL[$locale] === []) && str_contains($locale, '-')) {
$underscoredLocale = str_replace('-', '_', $locale);
$tempLL = $this->localizationFactory->getParsedData($fileRef, $underscoredLocale);
if (isset($tempLL[$underscoredLocale])) {
$tempLL[$locale] = $tempLL[$underscoredLocale];
}
}
if (isset($tempLL[$locale])) {
// Merge current language labels onto labels from previous language
// This way we have a labels with fall back applied
ArrayUtility::mergeRecursiveWithOverrule($localLanguage[$mainLanguageKey], $tempLL[$locale], true, false);
}
}
}
......@@ -295,27 +298,31 @@ class LanguageService
$localLanguage = [
'default' => $labels['default'] ?? [],
];
if ($this->lang !== 'default') {
foreach ($this->languageDependencies as $language) {
$mainLanguageKey = $this->getTypo3LanguageKey();
if ($mainLanguageKey !== 'default') {
$allLocales = array_merge([$mainLanguageKey], $this->locale->getDependencies());
$allLocales = array_reverse($allLocales);
foreach ($allLocales as $language) {
// Populate the initial values with default, if no labels for the current language are given
if (!isset($localLanguage[$this->lang])) {
$localLanguage[$this->lang] = $localLanguage['default'];
if (!isset($localLanguage[$mainLanguageKey])) {
$localLanguage[$mainLanguageKey] = $localLanguage['default'];
}
if ($this->lang !== 'default' && isset($labels[$language])) {
$localLanguage[$this->lang] = array_replace_recursive($localLanguage[$this->lang], $labels[$language]);
if (isset($labels[$language])) {
$localLanguage[$mainLanguageKey] = array_replace_recursive($localLanguage[$mainLanguageKey], $labels[$language]);
}
}
}
$this->overrideLabels[$fileRef] = $localLanguage;
}
/**
* This is needed as Extbase LocalizationUtility allows to set custom dependencies.
* @internal This is not public API and might be removed at any time.
*/
public function setDependencies(array $dependencies): void
private function getTypo3LanguageKey(): string
{
$this->languageDependencies = array_merge([$this->lang], $dependencies);
$this->languageDependencies = array_reverse($this->languageDependencies);
if ($this->locale === null) {
return 'default';
}
if ($this->locale->getName() === 'en') {
return 'default';
}
return $this->locale->getName();
}
}
......@@ -40,25 +40,28 @@ class LanguageServiceFactory
/**
* Factory method to create a language service object.
*
* @param string $locale the locale (= the TYPO3-internal locale given)
* @param Locale|string $locale the locale
*/
public function create(string $locale): LanguageService
public function create(Locale|string $locale): LanguageService
{
$obj = new LanguageService($this->locales, $this->localizationFactory, $this->runtimeCache);
$obj->init($locale);
$obj->init($locale instanceof Locale ? $locale : $this->locales->createLocale($locale));
return $obj;
}
public function createFromUserPreferences(?AbstractUserAuthentication $user): LanguageService
{
if ($user && ($user->user['lang'] ?? false)) {
return $this->create($user->user['lang']);
return $this->create($this->locales->createLocale($user->user['lang']));
}
return $this->create('default');
return $this->create('en');
}
public function createFromSiteLanguage(SiteLanguage $language): LanguageService
{
return $this->create($language->getTypo3Language());
$languageService = $this->create($language->getLocale() ?: $language->getTypo3Language());
// Always disable debugging for frontend
$languageService->debugKey = false;
return $languageService;
}
}
<?php
declare(strict_types=1);
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
namespace TYPO3\CMS\Core\Localization;
/**
* A representation of
* language key (based on ISO 639-1 / ISO 639-2)
* - the optional four-letter script code that can follow the language code according to the Unicode ISO 15924 Registry (e.g. HANS in zh_HANS)
* - region / country (based on ISO 3166-1)
* separated with a "-".
*
* This conforms to IETF - RFC 5646 (see https://datatracker.ietf.org/doc/rfc5646/) in a simplified form.
*/
class Locale implements \Stringable
{
protected string $locale;
protected string $languageCode;
protected ?string $languageScript = null;
protected ?string $countryCode = null;
/**
* List of language dependencies for an actual language. This setting is used for local variants of a language
* that depend on their "main" language, like Brazilian Portuguese or Canadian French.
*
* @var array<int, string>
*/
protected array $dependencies = [];
public function __construct(
string $locale = 'en',
array $dependencies = []
) {
$locale = $this->normalize($locale);
if (str_contains($locale, '-')) {
[$this->languageCode, $tail] = explode('-', $locale, 2);
if (str_contains($tail, '-')) {
[$this->languageScript, $this->countryCode] = explode('-', $tail);
} elseif (strlen($tail) === 4) {
$this->languageScript = $tail;
} else {
$this->countryCode = $tail ?: null;
}
$this->languageCode = strtolower($this->languageCode);
$this->languageScript = $this->languageScript ? ucfirst(strtolower($this->languageScript)) : null;
$this->countryCode = $this->countryCode ? strtoupper($this->countryCode) : null;
} else {
$this->languageCode = strtolower($locale);
}
$this->locale = $this->languageCode . ($this->languageScript ? '-' . $this->languageScript : '') . ($this->countryCode ? '-' . $this->countryCode : '');
$this->dependencies = array_map(fn ($dep) => $this->normalize($dep), $dependencies);
}
public function getName(): string
{
return $this->locale;
}
/**
* @return string
*/
public function getLanguageCode(): string
{
return $this->languageCode;
}
public function getLanguageScriptCode(): ?string
{
return $this->languageScript;
}
public function getCountryCode(): ?string
{
return $this->countryCode;
}
public function getDependencies(): array
{
return $this->dependencies;
}
protected function normalize(string $locale): string
{
if ($locale === 'default') {
return 'en';
}
if (str_contains($locale, '_')) {
$locale = str_replace('_', '-', $locale);
}
if (str_contains($locale, '.')) {
[$locale] = explode('.', $locale);
}
return $locale;
}
public function __toString(): string
{
return $this->locale;
}
}
......@@ -177,6 +177,19 @@ class Locales implements SingletonInterface
}
}
public function createLocale(string $localeKey): Locale
{
if (strpos($localeKey, '.')) {
[$sanitizedLocaleKey] = explode('.', $localeKey);
}
// Find the requested language in this list based on the $languageKey
// Language is found. Configure it:
if ($localeKey === 'en' || $this->isValidLanguageKey($sanitizedLocaleKey ?? $localeKey)) {
return new Locale($localeKey, $this->getLocaleDependencies($sanitizedLocaleKey ?? $localeKey));
}
return new Locale();
}
/**
* Returns the locales.
* @return array<int, non-empty-string>
......
.. include:: /Includes.rst.txt
.. _feature-99694-1674552209:
=====================================================================
Feature: #99694 - Unified Locale handling for translation files (XLF)
=====================================================================
See :issue:`99694`
Description
===========
TYPO3 internally now uses a "locale" format following the IETF RFC 5646 language
tag standard (https://www.rfc-editor.org/rfc/rfc5646.html).
A locale supported by TYPO3 consists of the following parts (tags and subtags):
* ISO 639-1 / ISO 639-2 compatible Language Key in lower-case (such as "fr" French, or "de" for German)
* optionally the ISO 15924 compatible language script system (4 letter, such as "Hans" as in "zh_Hans")
* optionally the region / country code according to ISO 3166-1 standard in upper camelcase such as "AT" for Austria.
Examples for a locale string are
* "en" for English
* "pt" for Portuguese
* "da-DK" for Danish as used in Denmark
* "de-CH" for German as used in Switzerland
* "zh-Hans-CN" for Chinese with the simplified script as spoken in China (mainland)
A new PHP object "Locale" automatically separates each tag and subtag into
these parts.
The Locale object can now be used to instantiate a new LanguageService object for
translating labels. Previous, TYPO3 used the "default" language key, instead of
the locale "en" to identify the english language. Both are supported, but it is
encouraged to use "en-US" or "en-GB" with the region subtag to identify the chosen
language more precisely.
Impact
======
Example for using the Locale for creating a "LanguageService" object for translations:
.. code-block:: php
$languageService = $languageServiceFactory->create(new Locale('de-AT'));
$myTranslatedString = $languageService->sL('LLL:EXT:my_extension/Resources/Private/Language/myfile.xlf:my-label');
This is highly recommended, as the wrappers php:`$GLOBALS['LANG']->sL()` and
:php:`$GLOBALS['TSFE']->sL()`will be deprecated in the future.
.. index:: PHP-API, ext:core
<?php
declare(strict_types=1);
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
namespace TYPO3\CMS\Core\Tests\Unit\Localization;
use TYPO3\CMS\Core\Localization\Locale;
use TYPO3\TestingFramework\Core\Unit\UnitTestCase;
class LocaleTest extends UnitTestCase
{
/**
* @test
*/
public function localeWithJustLanguageCodeSanitizesIncomingValuesProperly(): void
{
$subject = new Locale('en');
self::assertNull($subject->getLanguageScriptCode());
self::assertNull($subject->getCountryCode());
self::assertEquals('en', $subject->getLanguageCode());
self::assertEquals('en', (string)$subject);
// Also with mixed case
$subject = new Locale('eN');
self::assertNull($subject->getLanguageScriptCode());
self::assertNull($subject->getCountryCode());
self::assertEquals('en', $subject->getLanguageCode());
self::assertEquals('en', (string)$subject);
}
/**
* @test
*/
public function localeWithLanguageAndScriptCodeSanitizesIncomingValuesProperly(): void
{
$subject = new Locale('zh_HANS');
self::assertEquals('Hans', $subject->getLanguageScriptCode());
self::assertNull($subject->getCountryCode());
self::assertEquals('zh', $subject->getLanguageCode());
self::assertEquals('zh-Hans', (string)$subject);
}
/**
* @test
*/
public function localeWithLanguageAndScriptCodeAndCountryCodeSanitizesIncomingValuesProperly(): void
{
$subject = new Locale('zh_HANS_CN');
self::assertEquals('Hans', $subject->getLanguageScriptCode());
self::assertEquals('CN', $subject->getCountryCode());
self::assertEquals('zh', $subject->getLanguageCode());
self::assertEquals('zh-Hans-CN', (string)$subject);
}
/**
* @test
*/
public function variousCombinationsOfLanguageAndCountryCodeReturnsSanitizedValues(): void
{
$subject = new Locale('fr_CA');
self::assertNull($subject->getLanguageScriptCode());
self::assertEquals('CA', $subject->getCountryCode());
self::assertEquals('fr', $subject->getLanguageCode());
self::assertEquals('fr-CA', (string)$subject);
$subject = new Locale('de-AT');
self::assertNull($subject->getLanguageScriptCode());
self::assertEquals('AT', $subject->getCountryCode());
self::assertEquals('de', $subject->getLanguageCode());
self::assertEquals('de-AT', (string)$subject);
}
/**
* @test
*/
public function dependenciesAreSetAndRetrievedCorrectly(): void
{
$subject = new Locale('fr_CA', ['fr', 'en']);
self::assertNull($subject->getLanguageScriptCode());
self::assertEquals('CA', $subject->getCountryCode());
self::assertEquals('fr', $subject->getLanguageCode());
self::assertEquals(['fr', 'en'], $subject->getDependencies());
self::assertEquals('fr-CA', (string)$subject);
$subject = new Locale('en-US', ['en-UK', 'en']);
self::assertNull($subject->getLanguageScriptCode());
self::assertEquals('US', $subject->getCountryCode());
self::assertEquals('en', $subject->getLanguageCode());
self::assertEquals(['en-UK', 'en'], $subject->getDependencies());
self::assertEquals('en-US', (string)$subject);
}
}
......@@ -23,6 +23,7 @@ use TYPO3\CMS\Core\Cache\Frontend\FrontendInterface;
use TYPO3\CMS\Core\Http\ApplicationType;
use TYPO3\CMS\Core\Localization\LanguageService;
use TYPO3\CMS\Core\Localization\LanguageServiceFactory;
use TYPO3\CMS\Core\Localization\Locale;
use TYPO3\CMS\Core\Site\Entity\SiteLanguage;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Extbase\Configuration\ConfigurationManagerInterface;
......@@ -120,10 +121,8 @@ class LocalizationUtility
$languageKeyHash = sha1(json_encode(array_merge([$languageKey], $alternativeLanguageKeys, [$languageFilePath])));
$cache = self::getRuntimeCache();
if (!$cache->get($languageKeyHash)) {
$languageService = GeneralUtility::makeInstance(LanguageServiceFactory::class)->create($languageKey);
if ($alternativeLanguageKeys !== []) {
$languageService->setDependencies($alternativeLanguageKeys);
}
$locale = new Locale($languageKey, $alternativeLanguageKeys);
$languageService = GeneralUtility::makeInstance(LanguageServiceFactory::class)->create($locale);
$languageService->includeLLFile($languageFilePath);
$cache->set($languageKeyHash, $languageService);
}
......
......@@ -39,6 +39,7 @@ use TYPO3\CMS\Core\Html\SanitizerBuilderFactory;
use TYPO3\CMS\Core\Html\SanitizerInitiator;
use TYPO3\CMS\Core\Imaging\ImageManipulation\Area;
use TYPO3\CMS\Core\Imaging\ImageManipulation\CropVariantCollection;
use TYPO3\CMS\Core\Localization\LanguageServiceFactory;
use TYPO3\CMS\Core\Localization\Locales;
use TYPO3\CMS\Core\Log\LogManager;
use TYPO3\CMS\Core\Page\DefaultJavaScriptAssetTrait;
......@@ -4035,7 +4036,8 @@ class ContentObjectRenderer implements LoggerAwareInterface
}
break;
case 'lll':
$retVal = $tsfe->sL('LLL:' . $key);
$languageService = GeneralUtility::makeInstance(LanguageServiceFactory::class)->createFromSiteLanguage($this->getTypoScriptFrontendController()->getLanguage());
$retVal = $languageService->sL('LLL:' . $key);
break;
case 'path':
try {
......
......@@ -527,8 +527,6 @@ class TypoScriptFrontendController implements LoggerAwareInterface
$this->uniqueString = md5(microtime());
$this->initPageRenderer();
$this->initCaches();
// Initialize LLL behaviour
$this->setOutputLanguage();
}
private function initializeContext(Context $context): void
......@@ -2652,20 +2650,12 @@ class TypoScriptFrontendController implements LoggerAwareInterface
*/
public function sL($input)
{
if ($this->languageService === null) {
$this->languageService = GeneralUtility::makeInstance(LanguageServiceFactory::class)->createFromSiteLanguage($this->language);
}
return $this->languageService->sL($input);
}
/**
* Sets all internal measures what language the page should be rendered.
* This is not for records, but rather the HTML / charset and the locallang labels
*/
protected function setOutputLanguage()
{
$this->languageService = GeneralUtility::makeInstance(LanguageServiceFactory::class)->createFromSiteLanguage($this->language);
// Always disable debugging for TSFE
$this->languageService->debugKey = false;
}
/**
* Returns the originally requested page uid when TSFE was instantiated initially.
*/
......
......@@ -40,6 +40,8 @@ use TYPO3\CMS\Core\ExpressionLanguage\ProviderConfigurationLoader;
use TYPO3\CMS\Core\Http\ServerRequest;
use TYPO3\CMS\Core\LinkHandling\LinkService;
use TYPO3\CMS\Core\LinkHandling\TypoLinkCodecService;
use TYPO3\CMS\Core\Localization\LanguageService;
use TYPO3\CMS\Core\Localization\LanguageServiceFactory;
use TYPO3\CMS\Core\Log\Logger;
use TYPO3\CMS\Core\Package\PackageManager;
use TYPO3\CMS\Core\Page\PageRenderer;
......@@ -1468,7 +1470,11 @@ class ContentObjectRendererTest extends UnitTestCase
{
$key = StringUtility::getUniqueId('someKey');
$value = StringUtility::getUniqueId('someValue');
$GLOBALS['TSFE']->expects(self::once())->method('sL')->with('LLL:' . $key)->willReturn($value);
$languageServiceFactory = $this->createMock(LanguageServiceFactory::class);
$languageServiceMock = $this->createMock(LanguageService::class);
$languageServiceFactory->expects(self::once())->method('createFromSiteLanguage')->with(self::anything())->willReturn($languageServiceMock);
GeneralUtility::