From a314890f895498c2fc7a1bfe333c47c1e5c7f3fa Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Tue, 10 Apr 2018 10:32:30 +0200 Subject: [PATCH] tsdb: add support for more data types when converting sql time column to epoch (ms) --- pkg/tsdb/sql_engine.go | 30 +++++- pkg/tsdb/sql_engine_test.go | 182 +++++++++++++++++++++++++++++++----- pkg/tsdb/time_range.go | 5 + 3 files changed, 192 insertions(+), 25 deletions(-) diff --git a/pkg/tsdb/sql_engine.go b/pkg/tsdb/sql_engine.go index 16370a4ea7f..0f35cadf4d6 100644 --- a/pkg/tsdb/sql_engine.go +++ b/pkg/tsdb/sql_engine.go @@ -135,16 +135,16 @@ func (e *DefaultSqlEngine) Query( return result, nil } -// ConvertTimeColumnToEpochMs converts column named time to unix timestamp in milliseconds +// ConvertSqlTimeColumnToEpochMs converts column named time to unix timestamp in milliseconds // to make native datetime types and epoch dates work in annotation and table queries. func ConvertSqlTimeColumnToEpochMs(values RowValues, timeIndex int) { if timeIndex >= 0 { switch value := values[timeIndex].(type) { case time.Time: - values[timeIndex] = EpochPrecisionToMs(float64(value.Unix())) + values[timeIndex] = EpochPrecisionToMs(float64(value.UnixNano())) case *time.Time: if value != nil { - values[timeIndex] = EpochPrecisionToMs(float64((*value).Unix())) + values[timeIndex] = EpochPrecisionToMs(float64((*value).UnixNano())) } case int64: values[timeIndex] = int64(EpochPrecisionToMs(float64(value))) @@ -152,12 +152,36 @@ func ConvertSqlTimeColumnToEpochMs(values RowValues, timeIndex int) { if value != nil { values[timeIndex] = int64(EpochPrecisionToMs(float64(*value))) } + case uint64: + values[timeIndex] = int64(EpochPrecisionToMs(float64(value))) + case *uint64: + if value != nil { + values[timeIndex] = int64(EpochPrecisionToMs(float64(*value))) + } + case int32: + values[timeIndex] = int64(EpochPrecisionToMs(float64(value))) + case *int32: + if value != nil { + values[timeIndex] = int64(EpochPrecisionToMs(float64(*value))) + } + case uint32: + values[timeIndex] = int64(EpochPrecisionToMs(float64(value))) + case *uint32: + if value != nil { + values[timeIndex] = int64(EpochPrecisionToMs(float64(*value))) + } case float64: values[timeIndex] = EpochPrecisionToMs(value) case *float64: if value != nil { values[timeIndex] = EpochPrecisionToMs(*value) } + case float32: + values[timeIndex] = EpochPrecisionToMs(float64(value)) + case *float32: + if value != nil { + values[timeIndex] = EpochPrecisionToMs(float64(*value)) + } } } } diff --git a/pkg/tsdb/sql_engine_test.go b/pkg/tsdb/sql_engine_test.go index 48aac2c4d45..f8856489230 100644 --- a/pkg/tsdb/sql_engine_test.go +++ b/pkg/tsdb/sql_engine_test.go @@ -9,37 +9,175 @@ import ( func TestSqlEngine(t *testing.T) { Convey("SqlEngine", t, func() { - Convey("Given row values with time columns when converting them", func() { - dt := time.Date(2018, 3, 14, 21, 20, 6, 527e6, time.UTC) - fixtures := make([]interface{}, 8) - fixtures[0] = dt - fixtures[1] = dt.Unix() * 1000 - fixtures[2] = dt.Unix() - fixtures[3] = float64(dt.Unix() * 1000) - fixtures[4] = float64(dt.Unix()) + dt := time.Date(2018, 3, 14, 21, 20, 6, 527e6, time.UTC) - var nilDt *time.Time - var nilInt64 *int64 - var nilFloat64 *float64 - fixtures[5] = nilDt - fixtures[6] = nilInt64 - fixtures[7] = nilFloat64 + Convey("Given row values with time.Time as time columns", func() { + var nilPointer *time.Time + + fixtures := make([]interface{}, 3) + fixtures[0] = dt + fixtures[1] = &dt + fixtures[2] = nilPointer for i := range fixtures { ConvertSqlTimeColumnToEpochMs(fixtures, i) } - Convey("Should convert sql time columns to epoch time in ms ", func() { - expected := float64(dt.Unix() * 1000) + Convey("When converting them should return epoch time with millisecond precision ", func() { + expected := float64(dt.UnixNano() / 1e6) So(fixtures[0].(float64), ShouldEqual, expected) - So(fixtures[1].(int64), ShouldEqual, expected) - So(fixtures[2].(int64), ShouldEqual, expected) - So(fixtures[3].(float64), ShouldEqual, expected) - So(fixtures[4].(float64), ShouldEqual, expected) + So(fixtures[1].(float64), ShouldEqual, expected) + So(fixtures[2], ShouldBeNil) + }) + }) - So(fixtures[5], ShouldBeNil) + Convey("Given row values with int64 as time columns", func() { + tSeconds := dt.Unix() + tMilliseconds := dt.UnixNano() / 1e6 + tNanoSeconds := dt.UnixNano() + var nilPointer *int64 + + fixtures := make([]interface{}, 7) + fixtures[0] = tSeconds + fixtures[1] = &tSeconds + fixtures[2] = tMilliseconds + fixtures[3] = &tMilliseconds + fixtures[4] = tNanoSeconds + fixtures[5] = &tNanoSeconds + fixtures[6] = nilPointer + + for i := range fixtures { + ConvertSqlTimeColumnToEpochMs(fixtures, i) + } + + Convey("When converting them should return epoch time with millisecond precision ", func() { + So(fixtures[0].(int64), ShouldEqual, tSeconds*1e3) + So(fixtures[1].(int64), ShouldEqual, tSeconds*1e3) + So(fixtures[2].(int64), ShouldEqual, tMilliseconds) + So(fixtures[3].(int64), ShouldEqual, tMilliseconds) + So(fixtures[4].(int64), ShouldEqual, tMilliseconds) + So(fixtures[5].(int64), ShouldEqual, tMilliseconds) So(fixtures[6], ShouldBeNil) - So(fixtures[7], ShouldBeNil) + }) + }) + + Convey("Given row values with uin64 as time columns", func() { + tSeconds := uint64(dt.Unix()) + tMilliseconds := uint64(dt.UnixNano() / 1e6) + tNanoSeconds := uint64(dt.UnixNano()) + var nilPointer *uint64 + + fixtures := make([]interface{}, 7) + fixtures[0] = tSeconds + fixtures[1] = &tSeconds + fixtures[2] = tMilliseconds + fixtures[3] = &tMilliseconds + fixtures[4] = tNanoSeconds + fixtures[5] = &tNanoSeconds + fixtures[6] = nilPointer + + for i := range fixtures { + ConvertSqlTimeColumnToEpochMs(fixtures, i) + } + + Convey("When converting them should return epoch time with millisecond precision ", func() { + So(fixtures[0].(int64), ShouldEqual, tSeconds*1e3) + So(fixtures[1].(int64), ShouldEqual, tSeconds*1e3) + So(fixtures[2].(int64), ShouldEqual, tMilliseconds) + So(fixtures[3].(int64), ShouldEqual, tMilliseconds) + So(fixtures[4].(int64), ShouldEqual, tMilliseconds) + So(fixtures[5].(int64), ShouldEqual, tMilliseconds) + So(fixtures[6], ShouldBeNil) + }) + }) + + Convey("Given row values with int32 as time columns", func() { + tSeconds := int32(dt.Unix()) + var nilInt *int32 + + fixtures := make([]interface{}, 3) + fixtures[0] = tSeconds + fixtures[1] = &tSeconds + fixtures[2] = nilInt + + for i := range fixtures { + ConvertSqlTimeColumnToEpochMs(fixtures, i) + } + + Convey("When converting them should return epoch time with millisecond precision ", func() { + So(fixtures[0].(int64), ShouldEqual, dt.Unix()*1e3) + So(fixtures[1].(int64), ShouldEqual, dt.Unix()*1e3) + So(fixtures[2], ShouldBeNil) + }) + }) + + Convey("Given row values with uint32 as time columns", func() { + tSeconds := uint32(dt.Unix()) + var nilInt *uint32 + + fixtures := make([]interface{}, 3) + fixtures[0] = tSeconds + fixtures[1] = &tSeconds + fixtures[2] = nilInt + + for i := range fixtures { + ConvertSqlTimeColumnToEpochMs(fixtures, i) + } + + Convey("When converting them should return epoch time with millisecond precision ", func() { + So(fixtures[0].(int64), ShouldEqual, dt.Unix()*1e3) + So(fixtures[1].(int64), ShouldEqual, dt.Unix()*1e3) + So(fixtures[2], ShouldBeNil) + }) + }) + + Convey("Given row values with float64 as time columns", func() { + tSeconds := float64(dt.Unix()) + tMilliseconds := float64(dt.UnixNano() / 1e6) + tNanoSeconds := float64(dt.UnixNano()) + var nilPointer *float64 + + fixtures := make([]interface{}, 7) + fixtures[0] = tSeconds + fixtures[1] = &tSeconds + fixtures[2] = tMilliseconds + fixtures[3] = &tMilliseconds + fixtures[4] = tNanoSeconds + fixtures[5] = &tNanoSeconds + fixtures[6] = nilPointer + + for i := range fixtures { + ConvertSqlTimeColumnToEpochMs(fixtures, i) + } + + Convey("When converting them should return epoch time with millisecond precision ", func() { + So(fixtures[0].(float64), ShouldEqual, tSeconds*1e3) + So(fixtures[1].(float64), ShouldEqual, tSeconds*1e3) + So(fixtures[2].(float64), ShouldEqual, tMilliseconds) + So(fixtures[3].(float64), ShouldEqual, tMilliseconds) + So(fixtures[4].(float64), ShouldEqual, tMilliseconds) + So(fixtures[5].(float64), ShouldEqual, tMilliseconds) + So(fixtures[6], ShouldBeNil) + }) + }) + + Convey("Given row values with float32 as time columns", func() { + tSeconds := float32(dt.Unix()) + var nilInt *float32 + + fixtures := make([]interface{}, 3) + fixtures[0] = tSeconds + fixtures[1] = &tSeconds + fixtures[2] = nilInt + + for i := range fixtures { + ConvertSqlTimeColumnToEpochMs(fixtures, i) + } + + Convey("When converting them should return epoch time with millisecond precision ", func() { + So(fixtures[0].(float64), ShouldEqual, float32(dt.Unix()*1e3)) + So(fixtures[1].(float64), ShouldEqual, float32(dt.Unix()*1e3)) + So(fixtures[2], ShouldBeNil) }) }) }) diff --git a/pkg/tsdb/time_range.go b/pkg/tsdb/time_range.go index fd0cb3f8e82..47076eb3c6b 100644 --- a/pkg/tsdb/time_range.go +++ b/pkg/tsdb/time_range.go @@ -96,5 +96,10 @@ func EpochPrecisionToMs(value float64) float64 { return float64(value * 1e3) } + s := strconv.FormatFloat(value, 'f', -1, 64) + if len(s) == 19 { + return float64(value / 1e6) + } + return float64(value) }