From df4e9ca12bb142b857851b9d647be397d5255687 Mon Sep 17 00:00:00 2001 From: Emil Tullstedt Date: Tue, 4 Jan 2022 13:56:34 +0100 Subject: [PATCH] Datemath: Support fiscal years (#43558) --- go.mod | 2 +- go.sum | 4 +- pkg/extensions/main.go | 2 +- pkg/tsdb/legacydata/time_range.go | 164 +++++++++++++++++-------- pkg/tsdb/legacydata/time_range_test.go | 50 ++++++-- 5 files changed, 157 insertions(+), 65 deletions(-) diff --git a/go.mod b/go.mod index 81650ac2575..a6b10ddbae9 100644 --- a/go.mod +++ b/go.mod @@ -90,11 +90,11 @@ require ( github.com/russellhaering/goxmldsig v1.1.1 github.com/stretchr/testify v1.7.0 github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf - github.com/timberio/go-datemath v0.1.1-0.20200323150745-74ddef604fff github.com/ua-parser/uap-go v0.0.0-20211112212520-00c877edfe0f github.com/uber/jaeger-client-go v2.29.1+incompatible github.com/unknwon/com v1.0.1 github.com/urfave/cli/v2 v2.3.0 + github.com/vectordotdev/go-datemath v0.1.1-0.20211214152759-c03a58217724 github.com/weaveworks/common v0.0.0-20210913144402-035033b78a78 github.com/xorcare/pointer v1.1.0 github.com/yudai/gojsondiff v1.0.0 diff --git a/go.sum b/go.sum index 153e88634e5..8ee7b4e3291 100644 --- a/go.sum +++ b/go.sum @@ -2299,8 +2299,6 @@ github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU= github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/tinylru v1.0.2/go.mod h1:HDVL7TsWeezQ4g44Um84TOVBMFcq7Xa9giqNc805KJ8= github.com/tidwall/wal v0.1.4/go.mod h1:ww7Pd44/KnyETODJPUPKrzLlYjI72GZWlucNKt7pOt0= -github.com/timberio/go-datemath v0.1.1-0.20200323150745-74ddef604fff h1:QCdUBuN+iKWAB9HqPTkBwyKPPUHDobJ2AuELSNZwd4o= -github.com/timberio/go-datemath v0.1.1-0.20200323150745-74ddef604fff/go.mod h1:m7kjsbCuO4QKP3KLfnxiUZWiOiFXmxj30HeexjL3lc0= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg= @@ -2351,6 +2349,8 @@ github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBn github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +github.com/vectordotdev/go-datemath v0.1.1-0.20211214152759-c03a58217724 h1:v2/Zact72KD/a+HJNb0ES/8JV0Lhlp9PjPdZQGiaBoQ= +github.com/vectordotdev/go-datemath v0.1.1-0.20211214152759-c03a58217724/go.mod h1:PnwzbSst7KD3vpBzzlntZU5gjVa455Uqa5QPiKSYJzQ= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/vektra/mockery v0.0.0-20181123154057-e78b021dcbb5/go.mod h1:ppEjwdhyy7Y31EnHRDm1JkChoC7LXIJ7Ex0VYLWtZtQ= github.com/vishvananda/netlink v0.0.0-20171020171820-b2de5d10e38e/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= diff --git a/pkg/extensions/main.go b/pkg/extensions/main.go index 2a711249782..f3a27dabd60 100644 --- a/pkg/extensions/main.go +++ b/pkg/extensions/main.go @@ -22,7 +22,7 @@ import ( _ "github.com/robfig/cron/v3" _ "github.com/russellhaering/goxmldsig" _ "github.com/stretchr/testify/require" - _ "github.com/timberio/go-datemath" + _ "github.com/vectordotdev/go-datemath" _ "golang.org/x/time/rate" _ "gopkg.in/square/go-jose.v2" ) diff --git a/pkg/tsdb/legacydata/time_range.go b/pkg/tsdb/legacydata/time_range.go index d68ef900976..909c54cca5a 100644 --- a/pkg/tsdb/legacydata/time_range.go +++ b/pkg/tsdb/legacydata/time_range.go @@ -4,7 +4,7 @@ import ( "strconv" "time" - "github.com/timberio/go-datemath" + "github.com/vectordotdev/go-datemath" ) type DataTimeRange struct { @@ -21,31 +21,31 @@ func NewDataTimeRange(from, to string) DataTimeRange { } } -func (tr *DataTimeRange) GetFromAsMsEpoch() int64 { +func (tr DataTimeRange) GetFromAsMsEpoch() int64 { return tr.MustGetFrom().UnixNano() / int64(time.Millisecond) } -func (tr *DataTimeRange) GetFromAsSecondsEpoch() int64 { +func (tr DataTimeRange) GetFromAsSecondsEpoch() int64 { return tr.GetFromAsMsEpoch() / 1000 } -func (tr *DataTimeRange) GetFromAsTimeUTC() time.Time { +func (tr DataTimeRange) GetFromAsTimeUTC() time.Time { return tr.MustGetFrom().UTC() } -func (tr *DataTimeRange) GetToAsMsEpoch() int64 { +func (tr DataTimeRange) GetToAsMsEpoch() int64 { return tr.MustGetTo().UnixNano() / int64(time.Millisecond) } -func (tr *DataTimeRange) GetToAsSecondsEpoch() int64 { +func (tr DataTimeRange) GetToAsSecondsEpoch() int64 { return tr.GetToAsMsEpoch() / 1000 } -func (tr *DataTimeRange) GetToAsTimeUTC() time.Time { +func (tr DataTimeRange) GetToAsTimeUTC() time.Time { return tr.MustGetTo().UTC() } -func (tr *DataTimeRange) MustGetFrom() time.Time { +func (tr DataTimeRange) MustGetFrom() time.Time { res, err := tr.ParseFrom() if err != nil { return time.Unix(0, 0) @@ -53,7 +53,7 @@ func (tr *DataTimeRange) MustGetFrom() time.Time { return res } -func (tr *DataTimeRange) MustGetTo() time.Time { +func (tr DataTimeRange) MustGetTo() time.Time { res, err := tr.ParseTo() if err != nil { return time.Unix(0, 0) @@ -61,59 +61,127 @@ func (tr *DataTimeRange) MustGetTo() time.Time { return res } -func (tr DataTimeRange) ParseFrom() (time.Time, error) { - return parseTimeRange(tr.From, tr.Now, false, nil) +func (tr DataTimeRange) ParseFrom(options ...TimeRangeOption) (time.Time, error) { + options = append(options, WithNow(tr.Now)) + + pt := newParsableTime(tr.From, options...) + return pt.Parse() } -func (tr DataTimeRange) ParseTo() (time.Time, error) { - return parseTimeRange(tr.To, tr.Now, true, nil) -} +func (tr DataTimeRange) ParseTo(options ...TimeRangeOption) (time.Time, error) { + options = append(options, WithRoundUp(), WithNow(tr.Now)) -func (tr DataTimeRange) ParseFromWithLocation(location *time.Location) (time.Time, error) { - return parseTimeRange(tr.From, tr.Now, false, location) -} - -func (tr DataTimeRange) ParseToWithLocation(location *time.Location) (time.Time, error) { - return parseTimeRange(tr.To, tr.Now, true, location) + pt := newParsableTime(tr.To, options...) + return pt.Parse() } func (tr DataTimeRange) ParseFromWithWeekStart(location *time.Location, weekstart time.Weekday) (time.Time, error) { - return parseTimeRangeWithWeekStart(tr.From, tr.Now, false, location, weekstart) + return tr.ParseFrom(WithLocation(location), WithWeekstart(weekstart)) } -func (tr *DataTimeRange) ParseToWithWeekStart(location *time.Location, weekstart time.Weekday) (time.Time, error) { - return parseTimeRangeWithWeekStart(tr.To, tr.Now, true, location, weekstart) +func (tr DataTimeRange) ParseToWithWeekStart(location *time.Location, weekstart time.Weekday) (time.Time, error) { + return tr.ParseTo(WithLocation(location), WithWeekstart(weekstart)) } -func parseTimeRange(s string, now time.Time, withRoundUp bool, location *time.Location) (time.Time, error) { - return parseTimeRangeWithWeekStart(s, now, withRoundUp, location, -1) +func WithWeekstart(weekday time.Weekday) TimeRangeOption { + return func(timeRange parsableTime) parsableTime { + timeRange.weekstart = &weekday + return timeRange + } } -func parseTimeRangeWithWeekStart(s string, now time.Time, withRoundUp bool, location *time.Location, weekstart time.Weekday) (time.Time, error) { - if val, err := strconv.ParseInt(s, 10, 64); err == nil { - seconds := val / 1000 - nano := (val - seconds*1000) * 1000000 - return time.Unix(seconds, nano), nil +func WithLocation(loc *time.Location) TimeRangeOption { + return func(timeRange parsableTime) parsableTime { + timeRange.location = loc + return timeRange + } +} + +func WithFiscalStartMonth(month time.Month) TimeRangeOption { + return func(timeRange parsableTime) parsableTime { + timeRange.fiscalStartMonth = &month + return timeRange + } +} + +func WithNow(t time.Time) TimeRangeOption { + return func(timeRange parsableTime) parsableTime { + timeRange.now = t + return timeRange + } +} + +func WithRoundUp() TimeRangeOption { + return func(timeRange parsableTime) parsableTime { + timeRange.roundUp = true + return timeRange + } +} + +type parsableTime struct { + time string + now time.Time + location *time.Location + weekstart *time.Weekday + fiscalStartMonth *time.Month + roundUp bool +} + +type TimeRangeOption func(timeRange parsableTime) parsableTime + +func newParsableTime(t string, options ...TimeRangeOption) parsableTime { + p := parsableTime{ + time: t, + now: time.Now(), } - diff, err := time.ParseDuration("-" + s) - if err != nil { - options := []func(*datemath.Options){ - datemath.WithNow(now), - datemath.WithRoundUp(withRoundUp), - } - if location != nil { - options = append(options, datemath.WithLocation(location)) - } - if weekstart != -1 { - if weekstart > now.Weekday() { - weekstart = weekstart - 7 - } - options = append(options, datemath.WithStartOfWeek(weekstart)) - } - - return datemath.ParseAndEvaluate(s, options...) + for _, opt := range options { + p = opt(p) } - return now.Add(diff), nil + return p +} + +func (t parsableTime) Parse() (time.Time, error) { + // Milliseconds since Unix epoch. + if val, err := strconv.ParseInt(t.time, 10, 64); err == nil { + return time.UnixMilli(val), nil + } + + // Duration relative to current time. + if diff, err := time.ParseDuration("-" + t.time); err == nil { + return t.now.Add(diff), nil + } + + // Advanced time string, mimics the frontend's datemath library. + return datemath.ParseAndEvaluate(t.time, t.datemathOptions()...) +} + +func (t parsableTime) datemathOptions() []func(*datemath.Options) { + options := []func(*datemath.Options){ + datemath.WithNow(t.now), + datemath.WithRoundUp(t.roundUp), + } + if t.location != nil { + options = append(options, datemath.WithLocation(t.location)) + } + if t.weekstart != nil { + weekstart := *t.weekstart + if weekstart > t.now.Weekday() { + weekstart = weekstart - 7 + } + options = append(options, datemath.WithStartOfWeek(weekstart)) + } + if t.fiscalStartMonth != nil { + loc := time.UTC + if t.location != nil { + loc = t.location + } + options = append(options, datemath.WithStartOfFiscalYear( + // Year doesn't matter, and Grafana only supports setting the + // month that the fiscal year starts in. + time.Date(0, *t.fiscalStartMonth, 1, 0, 0, 0, 0, loc), + )) + } + return options } diff --git a/pkg/tsdb/legacydata/time_range_test.go b/pkg/tsdb/legacydata/time_range_test.go index 6d7cff54415..3837f35014d 100644 --- a/pkg/tsdb/legacydata/time_range_test.go +++ b/pkg/tsdb/legacydata/time_range_test.go @@ -3,9 +3,9 @@ package legacydata import ( "strconv" "testing" - "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -117,6 +117,30 @@ func TestTimeRange(t *testing.T) { }) }) + t.Run("Can parse now/fy, now/fQ for 1994-02-26T14:00:00.000Z with fiscal year starting in July", func(t *testing.T) { + tr := DataTimeRange{ + From: "now/fy", + To: "now/fQ", + Now: time.Date(1994, time.February, 26, 14, 0, 0, 0, time.UTC), + } + + start, err := tr.ParseFrom(WithFiscalStartMonth(time.July)) + require.NoError(t, err) + assert.Equal( + t, + time.Date(1993, time.July, 1, 0, 0, 0, 0, time.UTC), + start, + ) + + end, err := tr.ParseTo(WithFiscalStartMonth(time.July)) + require.NoError(t, err) + assert.Equal( + t, + time.Date(1994, time.April, 1, 0, 0, 0, 0, time.UTC).Add(-time.Millisecond), + end, + ) + }) + t.Run("Can parse 1960-02-01T07:00:00.000Z, 1965-02-03T08:00:00.000Z", func(t *testing.T) { tr := DataTimeRange{ From: "1960-02-01T07:00:00.000Z", @@ -205,7 +229,7 @@ func TestTimeRange(t *testing.T) { expected, err := time.Parse(time.RFC3339Nano, "2020-06-01T00:00:00.000-05:00") require.Nil(t, err) - res, err := tr.ParseFromWithLocation(location) + res, err := tr.ParseFrom(WithLocation(location)) require.Nil(t, err) require.True(t, expected.Equal(res)) }) @@ -214,7 +238,7 @@ func TestTimeRange(t *testing.T) { expected, err := time.Parse(time.RFC3339Nano, "2020-06-30T23:59:59.999-05:00") require.Nil(t, err) - res, err := tr.ParseToWithLocation(location) + res, err := tr.ParseTo(WithLocation(location)) require.Nil(t, err) require.True(t, expected.Equal(res)) }) @@ -233,7 +257,7 @@ func TestTimeRange(t *testing.T) { expected, err := time.Parse(time.RFC3339Nano, "2020-07-26T07:12:56.000-05:00") require.Nil(t, err) - res, err := tr.ParseFromWithLocation(location) + res, err := tr.ParseFrom(WithLocation(location)) require.Nil(t, err) require.True(t, expected.Equal(res)) }) @@ -242,7 +266,7 @@ func TestTimeRange(t *testing.T) { expected, err := time.Parse(time.RFC3339Nano, "2020-07-26T12:12:56.000-05:00") require.Nil(t, err) - res, err := tr.ParseToWithLocation(location) + res, err := tr.ParseTo(WithLocation(location)) require.Nil(t, err) require.True(t, expected.Equal(res)) }) @@ -261,7 +285,7 @@ func TestTimeRange(t *testing.T) { expected, err := time.Parse(time.RFC3339Nano, "2020-07-13T00:00:00.000Z") require.Nil(t, err) - res, err := tr.ParseFromWithWeekStart(nil, weekstart) + res, err := tr.ParseFrom(WithWeekstart(weekstart)) require.Nil(t, err) require.True(t, expected.Equal(res)) }) @@ -270,7 +294,7 @@ func TestTimeRange(t *testing.T) { expected, err := time.Parse(time.RFC3339Nano, "2020-07-19T23:59:59.999Z") require.Nil(t, err) - res, err := tr.ParseToWithWeekStart(nil, weekstart) + res, err := tr.ParseTo(WithWeekstart(weekstart)) require.Nil(t, err) require.True(t, expected.Equal(res)) }) @@ -290,7 +314,7 @@ func TestTimeRange(t *testing.T) { expected, err := time.Parse(time.RFC3339Nano, "2020-07-13T00:00:00.000-05:00") require.Nil(t, err) - res, err := tr.ParseFromWithWeekStart(location, weekstart) + res, err := tr.ParseFrom(WithLocation(location), WithWeekstart(weekstart)) require.Nil(t, err) require.True(t, expected.Equal(res)) }) @@ -299,7 +323,7 @@ func TestTimeRange(t *testing.T) { expected, err := time.Parse(time.RFC3339Nano, "2020-07-19T23:59:59.999-05:00") require.Nil(t, err) - res, err := tr.ParseToWithWeekStart(location, weekstart) + res, err := tr.ParseTo(WithLocation(location), WithWeekstart(weekstart)) require.Nil(t, err) require.True(t, expected.Equal(res)) }) @@ -319,7 +343,7 @@ func TestTimeRange(t *testing.T) { expected, err := time.Parse(time.RFC3339Nano, "2020-07-19T00:00:00.000-05:00") require.Nil(t, err) - res, err := tr.ParseFromWithWeekStart(location, weekstart) + res, err := tr.ParseFrom(WithLocation(location), WithWeekstart(weekstart)) require.Nil(t, err) require.True(t, expected.Equal(res)) }) @@ -328,7 +352,7 @@ func TestTimeRange(t *testing.T) { expected, err := time.Parse(time.RFC3339Nano, "2020-07-25T23:59:59.999-05:00") require.Nil(t, err) - res, err := tr.ParseToWithWeekStart(location, weekstart) + res, err := tr.ParseTo(WithLocation(location), WithWeekstart(weekstart)) require.Nil(t, err) require.True(t, expected.Equal(res)) }) @@ -348,7 +372,7 @@ func TestTimeRange(t *testing.T) { expected, err := time.Parse(time.RFC3339Nano, "2020-07-18T00:00:00.000-05:00") require.Nil(t, err) - res, err := tr.ParseFromWithWeekStart(location, weekstart) + res, err := tr.ParseFrom(WithLocation(location), WithWeekstart(weekstart)) require.Nil(t, err) require.True(t, expected.Equal(res)) }) @@ -357,7 +381,7 @@ func TestTimeRange(t *testing.T) { expected, err := time.Parse(time.RFC3339Nano, "2020-07-24T23:59:59.999-05:00") require.Nil(t, err) - res, err := tr.ParseToWithWeekStart(location, weekstart) + res, err := tr.ParseTo(WithLocation(location), WithWeekstart(weekstart)) require.Nil(t, err) require.True(t, expected.Equal(res)) })