Implementing new structures for DateTime

The structures with signature `0x46` and `0x66` are being replaced by `0x49` and `0x69`. This new structures changes the meaning of seconds and nano seconds from `adjusted Unix epoch`  to `UTC`.

This changes have with goal of avoiding unexistent or ambiguos ZonedDateTime to be received or sent over Bolt.

Bolt v4.3 and v4.4 were patched to support this feature if the server supports the patch.

This is a backport of neo4j#948

Author: bigmontz

packages/bolt-connection/src/bolt/bolt-protocol-v4x3.js

@@ -19,6 +19,8 @@
 import BoltProtocolV42 from './bolt-protocol-v4x2'
 import RequestMessage from './request-message'
 import { RouteObserver } from './stream-observers'
+import RequestMessage from './request-message'
+import { LoginObserver } from './stream-observers'
 
 import { internal } from 'neo4j-driver-core'
 
@@ -65,4 +67,49 @@ export default class BoltProtocol extends BoltProtocolV42 {
 
     return observer
   }
+
+  /**
+   * Initialize a connection with the server
+   *
+   * @param {Object} param0 The params
+   * @param {string} param0.userAgent The user agent
+   * @param {any} param0.authToken The auth token
+   * @param {function(error)} param0.onError On error callback
+   * @param {function(onComplte)} param0.onComplete On complete callback
+   * @returns {LoginObserver} The Login observer
+   */
+   initialize ({ userAgent, authToken, onError, onComplete } = {}) {
+    const observer = new LoginObserver({
+      onError: error => this._onLoginError(error, onError),
+      onCompleted: metadata => {
+        if (metadata.patch_bolt !== undefined) {
+          this._applyPatches(metadata.patch_bolt)
+        }
+        return this._onLoginCompleted(metadata, onComplete)
+      }
+    })
+
+    this.write(
+      RequestMessage.hello(userAgent, authToken, this._serversideRouting, ['utc']),
+      observer,
+      true
+    )
+
+    return observer
+  }
+
+  /**
+   *
+   * @param {string[]} patches Patches to be applied to the protocol
+   */
+  _applyPatches (patches) {
+    if (patches.includes('utc')) {
+      this._applyUtcPatch()
+    }
+  }
+
+  _applyUtcPatch () {
+    this._packer.useUtc = true
+    this._unpacker.useUtc = true
+  }
 }

packages/bolt-connection/src/bolt/request-message.js

@@ -106,11 +106,14 @@ export default class RequestMessage {
    * @param {Object} optional server side routing, set to routing context to turn on server side routing (> 4.1)
    * @return {RequestMessage} new HELLO message.
    */
-  static hello (userAgent, authToken, routing = null) {
+  static hello (userAgent, authToken, routing = null, patchs = null) {
     const metadata = Object.assign({ user_agent: userAgent }, authToken)
     if (routing) {
       metadata.routing = routing
     }
+    if (patchs) {
+      metadata.patch_bolt = patchs
+    }
     return new RequestMessage(
       HELLO,
       [metadata],

packages/bolt-connection/src/packstream/packstream-utc.js

@@ -0,0 +1,323 @@
+/**
+ * Copyright (c) "Neo4j"
+ * Neo4j Sweden AB [http://neo4j.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ import {
+   DateTime,
+   isInt,
+   int,
+   internal
+ } from 'neo4j-driver-core'
+ 
+ 
+ import {
+   epochSecondAndNanoToLocalDateTime
+ } from './temporal-factory'
+ 
+ const {
+   temporalUtil: {
+     localDateTimeToEpochSecond
+   }
+ } = internal
+ 
+ export const DATE_TIME_WITH_ZONE_OFFSET = 0x49
+ const DATE_TIME_WITH_ZONE_OFFSET_STRUCT_SIZE = 3
+ 
+ export const DATE_TIME_WITH_ZONE_ID = 0x69
+ const DATE_TIME_WITH_ZONE_ID_STRUCT_SIZE = 3
+
+ /**
+ * Unpack date time with zone offset value using the given unpacker.
+ * @param {Unpacker} unpacker the unpacker to use.
+ * @param {number} structSize the retrieved struct size.
+ * @param {BaseBuffer} buffer the buffer to unpack from.
+ * @param {boolean} disableLosslessIntegers if integer properties in the result date-time should be native JS numbers.
+ * @return {DateTime} the unpacked date time with zone offset value.
+ */
+export function unpackDateTimeWithZoneOffset (
+  unpacker,
+  structSize,
+  buffer,
+  disableLosslessIntegers,
+  useBigInt
+) {
+  unpacker._verifyStructSize(
+    'DateTimeWithZoneOffset',
+    DATE_TIME_WITH_ZONE_OFFSET_STRUCT_SIZE,
+    structSize
+  )
+
+  const utcSecond = unpacker.unpackInteger(buffer)
+  const nano = unpacker.unpackInteger(buffer)
+  const timeZoneOffsetSeconds = unpacker.unpackInteger(buffer)
+
+  const epochSecond = int(utcSecond).add(timeZoneOffsetSeconds)
+  const localDateTime = epochSecondAndNanoToLocalDateTime(epochSecond, nano)
+  const result = new DateTime(
+    localDateTime.year,
+    localDateTime.month,
+    localDateTime.day,
+    localDateTime.hour,
+    localDateTime.minute,
+    localDateTime.second,
+    localDateTime.nanosecond,
+    timeZoneOffsetSeconds,
+    null
+  )
+  return convertIntegerPropsIfNeeded(result, disableLosslessIntegers, useBigInt)
+}
+
+/**
+ * Unpack date time with zone id value using the given unpacker.
+ * @param {Unpacker} unpacker the unpacker to use.
+ * @param {number} structSize the retrieved struct size.
+ * @param {BaseBuffer} buffer the buffer to unpack from.
+ * @param {boolean} disableLosslessIntegers if integer properties in the result date-time should be native JS numbers.
+ * @return {DateTime} the unpacked date time with zone id value.
+ */
+ export function unpackDateTimeWithZoneId (
+  unpacker,
+  structSize,
+  buffer,
+  disableLosslessIntegers,
+  useBigInt
+) {
+  unpacker._verifyStructSize(
+    'DateTimeWithZoneId',
+    DATE_TIME_WITH_ZONE_ID_STRUCT_SIZE,
+    structSize
+  )
+
+  const epochSecond = unpacker.unpackInteger(buffer)
+  const nano = unpacker.unpackInteger(buffer)
+  const timeZoneId = unpacker.unpack(buffer)
+
+  const localDateTime = getTimeInZoneId(timeZoneId, epochSecond, nano)
+ 
+  const result = new DateTime(
+    localDateTime.year,
+    localDateTime.month,
+    localDateTime.day,
+    localDateTime.hour,
+    localDateTime.minute,
+    localDateTime.second,
+    int(nano),
+    localDateTime.timeZoneOffsetSeconds,
+    timeZoneId
+  )
+  return convertIntegerPropsIfNeeded(result, disableLosslessIntegers, useBigInt)
+}
+
+/*
+* Pack given date time.
+* @param {DateTime} value the date time value to pack.
+* @param {Packer} packer the packer to use.
+*/
+export function packDateTime (value, packer) {
+ if (value.timeZoneId) {
+   packDateTimeWithZoneId(value, packer)
+ } else {
+   packDateTimeWithZoneOffset(value, packer)
+ }
+}
+
+/**
+ * Pack given date time with zone id.
+ * @param {DateTime} value the date time value to pack.
+ * @param {Packer} packer the packer to use.
+ */
+ function packDateTimeWithZoneId (value, packer) {
+
+  const epochSecond = localDateTimeToEpochSecond(
+    value.year,
+    value.month,
+    value.day,
+    value.hour,
+    value.minute,
+    value.second,
+    value.nanosecond
+  )
+
+  const offset = value.timeZoneOffsetSeconds != null
+    ? value.timeZoneOffsetSeconds
+    : getOffsetFromZoneId(value.timeZoneId, epochSecond, value.nanosecond)
+
+  const utc = epochSecond.subtract(offset)
+  const nano = int(value.nanosecond)
+  const timeZoneId = value.timeZoneId
+  
+  const packableStructFields = [
+    packer.packable(utc),
+    packer.packable(nano),
+    packer.packable(timeZoneId)
+  ]
+  packer.packStruct(DATE_TIME_WITH_ZONE_ID, packableStructFields)
+}
+
+/**
+ * Pack given date time with zone offset.
+ * @param {DateTime} value the date time value to pack.
+ * @param {Packer} packer the packer to use.
+ */
+ function packDateTimeWithZoneOffset (value, packer) {
+  const epochSecond = localDateTimeToEpochSecond(
+    value.year,
+    value.month,
+    value.day,
+    value.hour,
+    value.minute,
+    value.second,
+    value.nanosecond
+  )
+  const nano = int(value.nanosecond)
+  const timeZoneOffsetSeconds = int(value.timeZoneOffsetSeconds)
+  const utcSecond = epochSecond.subtract(timeZoneOffsetSeconds)
+
+  const packableStructFields = [
+    packer.packable(utcSecond),
+    packer.packable(nano),
+    packer.packable(timeZoneOffsetSeconds)
+  ]
+  packer.packStruct(DATE_TIME_WITH_ZONE_OFFSET, packableStructFields)
+}
+ 
+ 
+ /**
+  * Returns the offset for a given timezone id
+  *
+  * Javascript doesn't have support for direct getting the timezone offset from a given
+  * TimeZoneId and DateTime in the given TimeZoneId. For solving this issue,
+  *
+  * 1. The ZoneId is applied to the timestamp, so we could make the difference between the
+  * given timestamp and the new calculated one. This is the offset for the timezone
+  * in the utc is equal to epoch (some time in the future or past)
+  * 2. The offset is subtracted from the timestamp, so we have an estimated utc timestamp.
+  * 3. The ZoneId is applied to the new timestamp, se we could could make the difference
+  * between the new timestamp and the calculated one. This is the offset for the given timezone.
+  *
+  * Example:
+  *    Input: 2022-3-27 1:59:59 'Europe/Berlin'
+  *    Apply 1, 2022-3-27 1:59:59 => 2022-3-27 3:59:59 'Europe/Berlin' +2:00
+  *    Apply 2, 2022-3-27 1:59:59 - 2:00 => 2022-3-26 23:59:59
+  *    Apply 3, 2022-3-26 23:59:59 => 2022-3-27 00:59:59 'Europe/Berlin' +1:00
+  *  The offset is +1 hour.
+  *
+  * @param {string} timeZoneId The timezone id
+  * @param {Integer} epochSecond The epoch second in the timezone id
+  * @param {Integerable} nanosecond The nanoseconds in the timezone id
+  * @returns The timezone offset
+  */
+ function getOffsetFromZoneId (timeZoneId, epochSecond, nanosecond) {
+   const dateTimeWithZoneAppliedTwice = getTimeInZoneId(timeZoneId, epochSecond, nanosecond)
+ 
+   // The wallclock form the current date time
+   const epochWithZoneAppliedTwice = localDateTimeToEpochSecond(
+     dateTimeWithZoneAppliedTwice.year,
+     dateTimeWithZoneAppliedTwice.month,
+     dateTimeWithZoneAppliedTwice.day,
+     dateTimeWithZoneAppliedTwice.hour,
+     dateTimeWithZoneAppliedTwice.minute,
+     dateTimeWithZoneAppliedTwice.second,
+     nanosecond)
+ 
+   const offsetOfZoneInTheFutureUtc = epochWithZoneAppliedTwice.subtract(epochSecond)
+   const guessedUtc = epochSecond.subtract(offsetOfZoneInTheFutureUtc)
+ 
+   const zonedDateTimeFromGuessedUtc = getTimeInZoneId(timeZoneId, guessedUtc, nanosecond)
+ 
+   const zonedEpochFromGuessedUtc = localDateTimeToEpochSecond(
+     zonedDateTimeFromGuessedUtc.year,
+     zonedDateTimeFromGuessedUtc.month,
+     zonedDateTimeFromGuessedUtc.day,
+     zonedDateTimeFromGuessedUtc.hour,
+     zonedDateTimeFromGuessedUtc.minute,
+     zonedDateTimeFromGuessedUtc.second,
+     nanosecond)
+ 
+   const offset = zonedEpochFromGuessedUtc.subtract(guessedUtc)
+   return offset
+ }
+ 
+ function getTimeInZoneId (timeZoneId, epochSecond, nano) {
+   const formatter = new Intl.DateTimeFormat('en-US', {
+     timeZone: timeZoneId,
+     year: 'numeric',
+     month: 'numeric',
+     day: 'numeric',
+     hour: 'numeric',
+     minute: 'numeric',
+     second: 'numeric',
+     hour12: false
+   })
+ 
+   const l = epochSecondAndNanoToLocalDateTime(epochSecond, nano)
+   const utc = Date.UTC(
+     int(l.year).toNumber(),
+     int(l.month).toNumber() - 1,
+     int(l.day).toNumber(),
+     int(l.hour).toNumber(),
+     int(l.minute).toNumber(),
+     int(l.second).toNumber()
+   )
+ 
+   const formattedUtcParts = formatter.formatToParts(utc)
+ 
+   const localDateTime = formattedUtcParts.reduce((obj, currentValue) => {
+     if (currentValue.type !== 'literal') {
+       obj[currentValue.type] = int(currentValue.value)
+     }
+     return obj
+   }, {})
+ 
+   const epochInTimeZone = localDateTimeToEpochSecond(
+     localDateTime.year,
+     localDateTime.month,
+     localDateTime.day,
+     localDateTime.hour,
+     localDateTime.minute,
+     localDateTime.second,
+     localDateTime.nanosecond
+   )
+ 
+   localDateTime.timeZoneOffsetSeconds = epochInTimeZone.subtract(epochSecond)
+   localDateTime.hour = localDateTime.hour.modulo(24)
+ 
+   return localDateTime
+ }
+
+ 
+ function convertIntegerPropsIfNeeded (obj, disableLosslessIntegers, useBigInt) {
+   if (!disableLosslessIntegers && !useBigInt) {
+     return obj
+   }
+ 
+   const convert = value =>
+     useBigInt ? value.toBigInt() : value.toNumberOrInfinity()
+ 
+   const clone = Object.create(Object.getPrototypeOf(obj))
+   for (const prop in obj) {
+     if (Object.prototype.hasOwnProperty.call(obj, prop) === true) {
+       const value = obj[prop]
+       clone[prop] = isInt(value) ? convert(value) : value
+     }
+   }
+   Object.freeze(clone)
+   return clone
+ }
+ 
+