[TASK] Use ISO-8601 dates for display and processing 24/47124/16
authorAndreas Wolf <andreas.wolf@typo3.org>
Sat, 5 Mar 2016 22:51:56 +0000 (23:51 +0100)
committerOliver Hader <oliver.hader@typo3.org>
Fri, 2 Dec 2016 17:11:05 +0000 (18:11 +0100)
This changes the internal date processing for FormEngine and DataHandler
to use ISO-8601 dates including a proper timezone. Dates are converted
to ISO-8601 as early as possible and converted back to UNIX timestamps
as late as possible.

As before, the database values are always values in the server’s
timezone, interpreted as UTC. Also, the client side inputs are
interpreted as UTC.

The main advantage is that once and for all we get rid of the timezone
issues that may potentially arise if the server and client use different
timezones. Additionally, the values are human readable (which is of
course not so much of an issue for hidden fields) and we can directly
use Moment.js for all heavy lifting.

Another big advantage is that we can make the date formats configurable
and decouple displayed dates and internally stored dates (by putting the
Moment objects to the input fields).

Change-Id: I3461915c2beaa96cd29c52c60e65a2e5005065b7
Resolves: #77702
Releases: master
Reviewed-on: https://review.typo3.org/47124
Reviewed-by: Susanne Moog <susanne.moog@typo3.org>
Tested-by: Susanne Moog <susanne.moog@typo3.org>
Tested-by: TYPO3com <no-reply@typo3.com>
Reviewed-by: Oliver Hader <oliver.hader@typo3.org>
Tested-by: Oliver Hader <oliver.hader@typo3.org>
typo3/sysext/backend/Classes/Form/Element/InputTextElement.php
typo3/sysext/backend/Classes/Form/FormDataProvider/DatabaseRowDateTimeFields.php
typo3/sysext/backend/Resources/Public/JavaScript/DateTimePicker.js
typo3/sysext/backend/Resources/Public/JavaScript/FormEngineValidation.js
typo3/sysext/backend/Tests/JavaScript/FormEngineValidationTest.js
typo3/sysext/backend/Tests/Unit/Form/Element/InputTextElementTest.php
typo3/sysext/backend/Tests/Unit/Form/FormDataProvider/DatabaseRowDateTimeFieldsTest.php
typo3/sysext/core/Classes/DataHandling/DataHandler.php
typo3/sysext/core/Documentation/Changelog/master/Important-77702-CustomRenderTypesForDateAndDatetimeFieldsMustUseISO-8601.rst [new file with mode: 0644]

index 3f16f34..029f68e 100644 (file)
@@ -79,9 +79,17 @@ class InputTextElement extends AbstractFormElement
             } elseif (in_array('date', $evalList)) {
                 $attributes['data-date-type'] = 'date';
             }
-            if (((int)$parameterArray['itemFormElValue']) !== 0) {
-                $parameterArray['itemFormElValue'] += date('Z', $parameterArray['itemFormElValue']);
+
+            // convert timestamp to proper ISO-8601 date so we get rid of timezone issues on the client.
+            // This only handles integer timestamps; if the field is a date(time), it already was converted to an
+            // ISO-8601 date by DatabaseRowDateTimeFields.
+            if (MathUtility::canBeInterpretedAsInteger($parameterArray['itemFormElValue']) && $parameterArray['itemFormElValue'] != 0) {
+                // output date as a ISO-8601 date; the stored value is the server time zone, so we need to treat it as such.
+                $timestamp = $parameterArray['itemFormElValue'];
+                $timestamp += date('Z', $timestamp);
+                $parameterArray['itemFormElValue'] = gmdate('c', $timestamp);
             }
+
             if (isset($config['range']['lower'])) {
                 $attributes['data-date-minDate'] = (int)$config['range']['lower'];
             }
index 4d09920..b05209c 100644 (file)
@@ -38,12 +38,17 @@ class DatabaseRowDateTimeFields implements FormDataProviderInterface
                 if (!empty($result['databaseRow'][$column])
                     &&  $result['databaseRow'][$column] !== $dateTimeFormats[$columnConfig['config']['dbType']]['empty']
                 ) {
-                    // Create a timestamp from current field data
-                    $result['databaseRow'][$column] = strtotime($result['databaseRow'][$column]);
+                    // Create an ISO-8601 date from current field data; the database always contains UTC
+                    // The field value is something like "2016-01-01" or "2016-01-01 10:11:12", so appending "UTC"
+                    // makes date() treat it as a UTC date (which is what we store in the database).
+                    $result['databaseRow'][$column] = date('c', strtotime($result['databaseRow'][$column] . ' UTC'));
                 } else {
                     // Set to 0 timestamp
                     $result['databaseRow'][$column] = 0;
                 }
+            } else {
+                // its a UNIX timestamp! We do not modify this here, as it will only be treated as a datetime because
+                // of eval being set to "date" or "datetime". This is handled in InputTextElement then.
             }
         }
         return $result;
index 934e23b..ea0a294 100644 (file)
@@ -111,7 +111,7 @@ define(['jquery'], function($) {
                                                $hiddenField.val('');
                                        } else {
                                                var format = $element.data('DateTimePicker').format();
-                                               var date = moment($element.val(), format);
+                                               var date = moment.utc($element.val(), format);
                                                var calculateTimeZoneOffset = $element.data('date-offset');
                                                var timeZoneOffset;
 
@@ -123,20 +123,22 @@ define(['jquery'], function($) {
 
                                                if (date.isValid()) {
                                                        var value;
+                                                       // Format the value for the hidden field that is passed on to the backend, i.e. most likely DataHandler.
+                                                       // The format for that is the timestamp for time fields, and a full-blown ISO-8601 timestamp for all
+                                                       // date-related fields.
                                                        switch ($element.data('dateType')) {
                                                                case 'time':
-                                                                       value = parseInt(date.format('H')) * 3600 + parseInt(date.format('m')) * 60;
+                                                                       value = date.format('HH:mm');
                                                                        break;
                                                                case 'timesec':
-                                                                       value = parseInt(date.format('H')) * 3600 + parseInt(date.format('m')) * 60 + parseInt(date.format('s'));
+                                                                       value = date.format('HH:mm:ss');
                                                                        break;
                                                                default:
-                                                                       value = date.unix() - timeZoneOffset;
+                                                                       value = date.format();
                                                        }
                                                        $hiddenField.val(value);
                                                } else {
-                                                       date = moment($hiddenField.val() + timeZoneOffset, 'X');
-                                                       $element.val(date.format(format));
+                                                       $element.val(moment($hiddenField.val()).utc().format(format));
                                                }
                                        }
                                });
@@ -149,7 +151,7 @@ define(['jquery'], function($) {
                                        if ($element.val() === '') {
                                                $hiddenField.val('');
                                        } else {
-                                               var date = event.date;
+                                               var date = event.date.utc();
 
                                                var calculateTimeZoneOffset = $element.data('date-offset');
                                                var timeZoneOffset, value;
@@ -161,13 +163,13 @@ define(['jquery'], function($) {
 
                                                switch ($element.data('dateType')) {
                                                        case 'time':
-                                                               value = parseInt(date.format('H')) * 3600 + parseInt(date.format('m')) * 60;
+                                                               value = date.format('HH:mm');
                                                                break;
                                                        case 'timesec':
-                                                               value = parseInt(date.format('H')) * 3600 + parseInt(date.format('m')) * 60 + parseInt(date.format('s'));
+                                                               value = date.format('HH:mm:ss');
                                                                break;
                                                        default:
-                                                               value = date.unix() - timeZoneOffset;
+                                                               value = date.format();
                                                }
                                                $hiddenField.val(value);
                                        }
index aed472e..c50ac7c 100644 (file)
@@ -16,7 +16,7 @@
  * Contains all JS functions related to TYPO3 TCEforms/FormEngineValidation
  * @internal
  */
-define(['jquery', 'TYPO3/CMS/Backend/FormEngine'], function ($, FormEngine) {
+define(['jquery', 'TYPO3/CMS/Backend/FormEngine', 'moment'], function ($, FormEngine, moment) {
 
        /**
         * The main FormEngineValidation object
@@ -147,41 +147,60 @@ define(['jquery', 'TYPO3/CMS/Backend/FormEngine'], function ($, FormEngine) {
         *
         * @param {String} type
         * @param {String} value
-        * @param {array} config
+        * @param {Object} config
         * @returns {String}
         */
        FormEngineValidation.formatValue = function(type, value, config) {
                var theString = '';
                switch (type) {
                        case 'date':
-                               var parsedInt = parseInt(value);
-                               if (!parsedInt) {
-                                       return '';
-                               }
-                               theTime = new Date(parsedInt * 1000);
-                               if (FormEngineValidation.USmode) {
-                                       theString = (theTime.getUTCMonth() + 1) + '-' + theTime.getUTCDate() + '-' + this.getYear(theTime);
+                               // poor man’s ISO-8601 detection: if we have a "-" in it, it apparently is not an integer.
+                               if (value.toString().indexOf('-') > 0) {
+                                       var date = moment(value).utc();
+                                       if (FormEngineValidation.USmode) {
+                                               theString = date.format('MM-DD-YYYY');
+                                       } else {
+                                               theString = date.format('DD-MM-YYYY');
+                                       }
                                } else {
-                                       theString = theTime.getUTCDate() + '-' + (theTime.getUTCMonth() + 1) + '-' + this.getYear(theTime);
+                                       var parsedInt = parseInt(value);
+                                       if (!parsedInt) {
+                                               return '';
+                                       }
+                                       theTime = new Date(parsedInt * 1000);
+                                       if (FormEngineValidation.USmode) {
+                                               theString = (theTime.getUTCMonth() + 1) + '-' + theTime.getUTCDate() + '-' + this.getYear(theTime);
+                                       } else {
+                                               theString = theTime.getUTCDate() + '-' + (theTime.getUTCMonth() + 1) + '-' + this.getYear(theTime);
+                                       }
                                }
                                break;
                        case 'datetime':
-                               if (!parseInt(value)) {
+                               if (value.toString().indexOf('-') <= 0 && !parseInt(value)) {
                                        return '';
                                }
                                theString = FormEngineValidation.formatValue('time', value, config) + ' ' + FormEngineValidation.formatValue('date', value, config);
                                break;
                        case 'time':
                        case 'timesec':
-                               var parsedInt = parseInt(value);
-                               if (!parsedInt && value.toString() !== '0') {
-                                       return '';
+                               if (value.toString().indexOf('-') > 0) {
+                                       var date = moment(value).utc();
+                                       if (type == 'timesec') {
+                                               theString = date.format('HH:mm:ss');
+                                       } else {
+                                               theString = date.format('HH:mm');
+                                       }
+                               } else {
+                                       var parsedInt = parseInt(value);
+                                       if (!parsedInt && value.toString() !== '0') {
+                                               return '';
+                                       }
+                                       var theTime = new Date(parsedInt * 1000);
+                                       var h = theTime.getUTCHours();
+                                       var m = theTime.getUTCMinutes();
+                                       var s = theTime.getUTCSeconds();
+                                       theString = h + ':' + ((m < 10) ? '0' : '') + m + ((type == 'timesec') ? ':' + ((s < 10) ? '0' : '') + s : '');
                                }
-                               var theTime = new Date(parsedInt * 1000);
-                               var h = theTime.getUTCHours();
-                               var m = theTime.getUTCMinutes();
-                               var s = theTime.getUTCSeconds();
-                               theString = h + ':' + ((m < 10) ? '0' : '') + m + ((type == 'timesec') ? ':' + ((s < 10) ? '0' : '') + s : '');
                                break;
                        case 'password':
                                theString = (value) ? FormEngineValidation.passwordDummy : '';
index 9a30349..2fd9e7b 100644 (file)
@@ -21,6 +21,13 @@ define(['jquery', 'TYPO3/CMS/Backend/FormEngineValidation'], function($, FormEng
                                'result': '26-4-1970'
                        },
                        {
+                               'description': 'works for type date with iso date',
+                               'type': 'date',
+                               'value': '2016-12-02T11:16:06+00:00',
+                               'config': [],
+                               'result': '02-12-2016'
+                       },
+                       {
                                'description': 'works for type datetime',
                                'type': 'datetime',
                                'value': 0,
@@ -35,6 +42,27 @@ define(['jquery', 'TYPO3/CMS/Backend/FormEngineValidation'], function($, FormEng
                                'result': '17:46 26-4-1970'
                        },
                        {
+                               'description': 'works for type datetime with iso date',
+                               'type': 'datetime',
+                               'value': '2016-12-02T11:16:06+00:00',
+                               'config': [],
+                               'result': '11:16 02-12-2016'
+                       },
+                       {
+                               'description': 'resolves to empty result for zero value',
+                               'type': 'datetime',
+                               'value': 0,
+                               'config': [],
+                               'result': ''
+                       },
+                       {
+                               'description': 'resolves to empty result for invalid value',
+                               'type': 'datetime',
+                               'value': 'invalid',
+                               'config': [],
+                               'result': ''
+                       },
+                       {
                                'description': 'works for type time',
                                'type': 'time',
                                'value': 0,
@@ -47,6 +75,13 @@ define(['jquery', 'TYPO3/CMS/Backend/FormEngineValidation'], function($, FormEng
                                'value': 10000000,
                                'config': [],
                                'result': '17:46'
+                       },
+                       {
+                               'description': 'works for type time with iso date',
+                               'type': 'time',
+                               'value': '2016-12-02T11:16:06+00:00',
+                               'config': [],
+                               'result': '11:16'
                        }
                ];
 
@@ -57,6 +92,69 @@ define(['jquery', 'TYPO3/CMS/Backend/FormEngineValidation'], function($, FormEng
                describe('tests for formatValue', function() {
                        using(formatValueDataProvider, function(testCase) {
                                it(testCase.description, function() {
+                                       FormEngineValidation.USmode = 0;
+                                       var result = FormEngineValidation.formatValue(testCase.type, testCase.value, testCase.config);
+                                       expect(result).toBe(testCase.result);
+                               });
+                       });
+               });
+
+               /**
+                * @type {*[]}
+                */
+               var formatValueUsModeDataProvider = [
+                       {
+                               'description': 'works for type date with timestamp in US mode',
+                               'type': 'date',
+                               'value': 10000000,
+                               'config': [],
+                               'result': '4-26-1970'
+                       },
+                       {
+                               'description': 'works for type date with iso date in US mode',
+                               'type': 'date',
+                               'value': '2016-12-02T11:16:06+00:00',
+                               'config': [],
+                               'result': '12-02-2016'
+                       },
+                       {
+                               'description': 'works for type datetime with timestamp in US mode',
+                               'type': 'datetime',
+                               'value': 10000000,
+                               'config': [],
+                               'result': '17:46 4-26-1970'
+                       },
+                       {
+                               'description': 'works for type datetime with iso date in US mode',
+                               'type': 'datetime',
+                               'value': '2016-12-02T11:16:06+00:00',
+                               'config': [],
+                               'result': '11:16 12-02-2016'
+                       },
+                       {
+                               'description': 'works for type time with timestamp in US mode',
+                               'type': 'time',
+                               'value': 10000000,
+                               'config': [],
+                               'result': '17:46'
+                       },
+                       {
+                               'description': 'works for type time with iso date in US mode',
+                               'type': 'time',
+                               'value': '2016-12-02T11:16:06+00:00',
+                               'config': [],
+                               'result': '11:16'
+                       }
+               ];
+
+               /**
+                * @dataProvider formatValueUsModeDataProvider
+                * @test
+                */
+               describe('tests for formatValue in US Mode', function() {
+                       using(formatValueUsModeDataProvider, function(testCase) {
+                               it(testCase.description, function() {
+                                       FormEngineValidation.USmode = 1;
                                        var result = FormEngineValidation.formatValue(testCase.type, testCase.value, testCase.config);
                                        expect(result).toBe(testCase.result);
                                });
index 9b8bfd7..2534259 100644 (file)
@@ -59,17 +59,17 @@ class InputTextElementTest extends UnitTestCase
         return [
             // German standard time (without DST) is one hour ahead of UTC
             'date in 2016 in German timezone' => [
-                1457103519, 'Europe/Berlin', 1457103519 + 3600
+                1457103519, 'Europe/Berlin', '2016-03-04T15:58:39+00:00'
             ],
             'date in 1969 in German timezone' => [
-                -7200, 'Europe/Berlin', -3600
+                -7200, 'Europe/Berlin', '1969-12-31T23:00:00+00:00'
             ],
             // Los Angeles is 8 hours behind UTC
             'date in 2016 in Los Angeles timezone' => [
-                1457103519, 'America/Los_Angeles', 1457103519 - 28800
+                1457103519, 'America/Los_Angeles', '2016-03-04T06:58:39+00:00'
             ],
             'date in UTC' => [
-                1457103519, 'UTC', 1457103519
+                1457103519, 'UTC', '2016-03-04T14:58:39+00:00'
             ]
         ];
     }
index 3b01e99..cb4ff41 100644 (file)
@@ -89,7 +89,7 @@ class DatabaseRowDateTimeFieldsTest extends UnitTestCase
             ],
         ];
         $expected = $input;
-        $expected['databaseRow']['aField'] = 1437955200; // 27.07.2015 0:00 UTC
+        $expected['databaseRow']['aField'] = '2015-07-27T00:00:00+00:00';
         $this->assertEquals($expected, (new DatabaseRowDateTimeFields())->addData($input));
         date_default_timezone_set($oldTimezone);
     }
@@ -117,7 +117,7 @@ class DatabaseRowDateTimeFieldsTest extends UnitTestCase
             ],
         ];
         $expected = $input;
-        $expected['databaseRow']['aField'] = 1438010732; // 27.07.2015 15:25:32 UTC
+        $expected['databaseRow']['aField'] = '2015-07-27T15:25:32+00:00';
         $this->assertEquals($expected, (new DatabaseRowDateTimeFields())->addData($input));
         date_default_timezone_set($oldTimezone);
     }
index 88bcbd5..7d3cf1d 100644 (file)
@@ -988,17 +988,6 @@ class DataHandler
                 }
                 $theRealPid = null;
 
-                // Handle native date/time fields
-                $dateTimeFormats = QueryHelper::getDateTimeFormats();
-                foreach ($GLOBALS['TCA'][$table]['columns'] as $column => $config) {
-                    if (isset($incomingFieldArray[$column])) {
-                        if (isset($config['config']['dbType']) && ($config['config']['dbType'] === 'date' || $config['config']['dbType'] === 'datetime')) {
-                            $emptyValue = $dateTimeFormats[$config['config']['dbType']]['empty'];
-                            $format = $dateTimeFormats[$config['config']['dbType']]['format'];
-                            $incomingFieldArray[$column] = $incomingFieldArray[$column] && $incomingFieldArray[$column] !== $emptyValue ? gmdate($format, $incomingFieldArray[$column]) : $emptyValue;
-                        }
-                    }
-                }
                 // Hook: processDatamap_preProcessFieldArray
                 foreach ($hookObjectsArr as $hookObj) {
                     if (method_exists($hookObj, 'processDatamap_preProcessFieldArray')) {
@@ -1751,18 +1740,19 @@ class DataHandler
         $isDateOrDateTimeField = false;
         $format = '';
         $emptyValue = '';
+        // normal integer "date" fields (timestamps) are handled in checkValue_input_Eval
         if (isset($tcaFieldConf['dbType']) && ($tcaFieldConf['dbType'] === 'date' || $tcaFieldConf['dbType'] === 'datetime')) {
             if (empty($value)) {
                 $value = 0;
             } else {
                 $isDateOrDateTimeField = true;
                 $dateTimeFormats = QueryHelper::getDateTimeFormats();
+                $format = $dateTimeFormats[$tcaFieldConf['dbType']]['format'];
+
                 // Convert the date/time into a timestamp for the sake of the checks
                 $emptyValue = $dateTimeFormats[$tcaFieldConf['dbType']]['empty'];
-                $format = $dateTimeFormats[$tcaFieldConf['dbType']]['format'];
-                // At this point in the processing, the timestamps are still based on UTC
-                $timeZone = new \DateTimeZone('UTC');
-                $dateTime = \DateTime::createFromFormat('!' . $format, $value, $timeZone);
+                // We store UTC timestamps in the database, which is what getTimestamp() returns.
+                $dateTime = new \DateTime($value);
                 $value = $value === $emptyValue ? 0 : $dateTime->getTimestamp();
             }
         }
@@ -2711,7 +2701,20 @@ class DataHandler
                     break;
                 case 'date':
                 case 'datetime':
-                    $value = (int)$value;
+                    // a hyphen as first character indicates a negative timestamp
+                    if (strpos($value, '-') === false || strpos($value, '-') === 0) {
+                        $value = (int)$value;
+                    } else {
+                        // ISO 8601 dates
+                        $dateTime = new \DateTime($value);
+                        // The returned timestamp is always UTC
+                        $value = $dateTime->getTimestamp();
+                    }
+                    // $value is a UTC timestamp here.
+                    // The value will be stored in the server’s local timezone, but treated as UTC, so we brute force
+                    // subtract the offset here. The offset is subtracted instead of added because the value is stored
+                    // in the timezone, but interpreted as UTC, so if we switched the server to UTC, the correct
+                    // value would be returned.
                     if ($value !== 0 && !$this->dontProcessTransformations) {
                         $value -= date('Z', $value);
                     }
diff --git a/typo3/sysext/core/Documentation/Changelog/master/Important-77702-CustomRenderTypesForDateAndDatetimeFieldsMustUseISO-8601.rst b/typo3/sysext/core/Documentation/Changelog/master/Important-77702-CustomRenderTypesForDateAndDatetimeFieldsMustUseISO-8601.rst
new file mode 100644 (file)
index 0000000..00ce644
--- /dev/null
@@ -0,0 +1,16 @@
+.. include:: ../../Includes.txt
+
+======================================================================================
+Important: #77702 - Custom render types for date and datetime fields must use ISO-8601
+======================================================================================
+
+See :issue:`77702`
+
+Description
+===========
+
+Historically, TYPO3 used its own special, localized formats for passing date and datetime values between server and client. To get rid of any possible problems with that, we now use ISO-8601, a standard format for date/time representations.
+
+Due to that, you need to adapt your **custom FormEngine render types** if you use them for any date/datetime fields, even those stored as integers in the database (eval=date/datetime).
+
+.. index:: Backend, Database
\ No newline at end of file