Improve the timestamp parsing

Add a timestampwrapper config utility, which should make this type
easier to use in a good way in config.

Make it accept RFC3339-like datetimes, but also shortened. We require a
timezone, which IMO is a good idea, to avoid ambiguity.

Cognite.Common/CogniteTime.cs

@@ -14,7 +14,7 @@ public static class CogniteTime
         /// DateTime object representing the Unix Epoch, midnight 1/1/1970, in UTC.
         /// </summary>
         public static DateTime DateTimeEpoch => new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc);
-        
+
         private static readonly long epochTicks = DateTimeEpoch.Ticks;
         private static readonly long maxTsValue = DateTime.MaxValue.ToUnixTimeMilliseconds();
         private static readonly long minTsValue = DateTime.MinValue.ToUnixTimeMilliseconds();
@@ -160,12 +160,29 @@ public static TimeSpan Min(TimeSpan t1, TimeSpan t2)
         private static readonly Regex timestampStringRegex = new Regex("^([0-9]+)(ms?|w|d|h|s)-ago$");
         private static readonly Regex timespanStringRegex = new Regex("^([0-9]+)(ms?|w|d|h|s)$");
 
+        static readonly string[] formats = {
+            "yyyy-MM-ddTHH:mm:sszzz",
+            "yyyy-MM-ddTHH:mm:sszz",
+            "yyyy-MM-ddTHH:mm:ssZ",
+            "yyyy-MM-ddTHH:mmzzz",
+            "yyyy-MM-ddTHH:mmzz",
+            "yyyy-MM-ddTHH:mmZ",
+            "yyyy-MM-ddTHHzzz",
+            "yyyy-MM-ddTHHzz",
+            "yyyy-MM-ddTHHZ",
+            "yyyy-MM-ddzzz",
+            "yyyy-MM-ddzz",
+            "yyyy-MM-ddZ",
+        };
+
         /// <summary>
         /// Parse Cognite timestamp string.
         /// The format is N[timeunit]-ago where timeunit is w,d,h,m,s. Example: '2d-ago'
         /// returns a timestamp two days ago.
         /// If the format is N[timeunit], without -ago, it is set to the future, or after <paramref name="relative"/>
         /// Without timeunit, it is converted to a datetime in milliseconds since epoch.
+        /// 
+        /// This also supports RFC3339-like timestamps, on the form 'yyyy-MM-dd[THH:mm:ss]Z'.
         /// </summary>
         /// <param name="t">Timestamp string</param>
         /// <param name="relative">Set time relative to this if -ago syntax is used</param>
@@ -190,6 +207,13 @@ public static TimeSpan Min(TimeSpan t1, TimeSpan t2)
                 if (span != null) return now.Add(span.Value);
             }
 
+            try
+            {
+                return DateTime.ParseExact(t, formats, CultureInfo.InvariantCulture, DateTimeStyles.AllowLeadingWhite | DateTimeStyles.AllowTrailingWhite)
+                    .ToUniversalTime();
+            }
+            catch { }
+
             try
             {
                 return FromUnixTimeMilliseconds(Convert.ToInt64(t, CultureInfo.InvariantCulture));
@@ -217,7 +241,7 @@ public static TimeSpan Min(TimeSpan t1, TimeSpan t2)
             {
                 var rawUnit = match.Groups[2].Value;
                 var rawValue = match.Groups[1].Value;
-                
+
                 long value = Convert.ToInt64(rawValue, CultureInfo.InvariantCulture);
 
                 return GetSpan(rawUnit, value);

Cognite.Config/TimestampWrapper.cs

@@ -0,0 +1,63 @@
+using System;
+using Cognite.Extractor.Common;
+using YamlDotNet.Core;
+using YamlDotNet.Core.Events;
+using YamlDotNet.Serialization;
+
+namespace Cognite.Extractor.Configuration
+{
+    /// <summary>
+    /// Wrapper around a timestamp, for configuration objects.
+    /// </summary>
+    public class TimestampWrapper
+    {
+        /// <summary>
+        /// Constructor
+        /// </summary>
+        /// <param name="value">Raw value</param>
+        /// <exception cref="ConfigurationException">If the provided value is non-null, and an invalid timestamp</exception>
+        public TimestampWrapper(string? value)
+        {
+            if (!string.IsNullOrWhiteSpace(value) && CogniteTime.ParseTimestampString(value) is null)
+            {
+                throw new ConfigurationException($"Invalid timestamp {value}, must be on the form 'yyyy-MM-dd[THH:mm:ss]Z', or N[ms|s|m|h|d](-ago)");
+            }
+            RawValue = value;
+        }
+
+        /// <summary>
+        /// The raw value of the timestamp, written to during deserialization.
+        /// </summary>
+        public string? RawValue { get; }
+        /// <summary>
+        /// Get the current datetime value.
+        /// </summary>
+        /// <returns></returns>
+        public DateTime? Get()
+        {
+            if (RawValue == null) return null;
+            // Should never fail, but we throw an error here to be safe.
+            return CogniteTime.ParseTimestampString(RawValue) ?? throw new ConfigurationException("Invalid timestamp");
+        }
+    }
+
+    internal class TimestampWrapperConverter : IYamlTypeConverter
+    {
+        public bool Accepts(Type type)
+        {
+            return type == typeof(TimestampWrapper);
+        }
+
+        public object? ReadYaml(IParser parser, Type type)
+        {
+            var scalar = parser.Consume<Scalar>();
+            return new TimestampWrapper(scalar.Value);
+        }
+
+        public void WriteYaml(IEmitter emitter, object? value, Type type)
+        {
+            var it = value as TimestampWrapper;
+            emitter.Emit(new Scalar(AnchorName.Empty, TagName.Empty, it?.RawValue ?? "", ScalarStyle.DoubleQuoted, false, true));
+        }
+    }
+}

Cognite.Config/YamlConfigBuilder.cs

@@ -74,6 +74,7 @@ public INamingConvention NamingConvention
         private List<IYamlTypeConverter> _typeConverters = new List<IYamlTypeConverter> {
             new ListOrStringConverter(),
             new YamlEnumConverter(),
+            new TimestampWrapperConverter(),
         };
 
         private Dictionary<string, Type> _tagMappings = new Dictionary<string, Type>

ExtractorUtils.Test/unit/CogniteTimeTest.cs

@@ -3,6 +3,7 @@
 using Cognite.Extractor.Common;
 using System.Threading;
 using System.Threading.Tasks;
+using Cognite.Extractor.Configuration;
 
 namespace ExtractorUtils.Test.Unit
 {
@@ -189,6 +190,7 @@ public static void TestSmallExtendContract()
         [InlineData("1234s-agoooo", true)]
         [InlineData("1234k-ago", true)]
         [InlineData("1209600000ms-ago", false)]
+        [InlineData("4y-ago", true)]
         public static void TestParseTime(string input, bool resultNull)
         {
             var time = DateTime.UtcNow;
@@ -217,6 +219,7 @@ public static void TestParseTime(string input, bool resultNull)
         [InlineData("1234s-", true)]
         [InlineData("1234k", true)]
         [InlineData("1209600000ms", false)]
+        [InlineData("4y", true)]
         public static void TestParseTimeFuture(string input, bool resultNull)
         {
             var time = DateTime.UtcNow;
@@ -242,6 +245,31 @@ public static void TestParseTimeAbsolute()
 
             Assert.Equal(raw, CogniteTime.ParseTimestampString(raw.ToString(), time)?.ToUnixTimeMilliseconds());
         }
+
+        [Theory]
+        [InlineData("2020-01-01T01:00:00+01:00", false)]
+        [InlineData("2020-01-01T00:00:00Z", false)]
+        [InlineData("2020-01-01T00:00:00+00:00", false)]
+        [InlineData("2020-01-01T01:00+01:00", false)]
+        [InlineData("2020-01-01T01+01:00", false)]
+        [InlineData("2020-01-01+00:00", false)]
+        [InlineData("2020-01-01Z", false)]
+        public static void TestParseTimeValue(string input, bool resultNull)
+        {
+            var time = new DateTime(2020, 1, 1, 0, 0, 0);
+            var converted = CogniteTime.ParseTimestampString(input, time);
+
+            if (resultNull)
+            {
+                Assert.Null(converted);
+            }
+            else
+            {
+                Assert.NotNull(converted);
+                Assert.Equal(time, converted);
+            }
+        }
+
         [Theory]
         [InlineData("2w", false)]
         [InlineData("2", false, "w")]
@@ -329,5 +357,24 @@ public static async Task TestCronTimeSpanWrapper()
             Assert.True(wrapper.Value <= TimeSpan.FromMinutes(1));
         }
 
+        [Fact]
+        public static void TestTimestampWrapper()
+        {
+            var config = @"
+foo: 2020-01-01T00:00:00Z
+";
+            var loaded = ConfigurationUtils.ReadString<TestConf>(config, false);
+            Assert.Equal(new DateTime(2020, 1, 1, 0, 0, 0, DateTimeKind.Utc), loaded.Foo.Get());
+            config = @"
+foo: some-bs
+";
+            Assert.Throws<ConfigurationException>(() => ConfigurationUtils.ReadString<TestConf>(config, false));
+        }
+
+        class TestConf
+        {
+            public TimestampWrapper Foo { get; set; }
+        }
+
     }
 }

version

@@ -1 +1 @@
-1.29.0
+1.30.0