diff --git a/pkg/tsdb/mysql/mysql.go b/pkg/tsdb/mysql/mysql.go index 483974c55a4..83027d4b210 100644 --- a/pkg/tsdb/mysql/mysql.go +++ b/pkg/tsdb/mysql/mysql.go @@ -8,7 +8,6 @@ import ( "math" "reflect" "strconv" - "time" "github.com/go-sql-driver/mysql" "github.com/go-xorm/core" @@ -239,15 +238,18 @@ func (e MysqlQueryEndpoint) transformToTimeSeries(query *tsdb.Query, rows *core. return err } + // converts column named time to unix timestamp in milliseconds to make + // native mysql datetime types and epoch dates work in + // annotation and table queries. + tsdb.ConvertSqlTimeColumnToEpochMs(values, timeIndex) + switch columnValue := values[timeIndex].(type) { case int64: - timestamp = float64(columnValue * 1000) + timestamp = float64(columnValue) case float64: - timestamp = columnValue * 1000 - case time.Time: - timestamp = float64(columnValue.UnixNano() / 1e6) + timestamp = columnValue default: - return fmt.Errorf("Invalid type for column time, must be of type timestamp or unix timestamp, got: %T %v", columnValue, columnValue) + return fmt.Errorf("Invalid type for column time/time_sec, must be of type timestamp or unix timestamp, got: %T %v", columnValue, columnValue) } if metricIndex >= 0 { diff --git a/pkg/tsdb/mysql/mysql_test.go b/pkg/tsdb/mysql/mysql_test.go index b8d7fc8d42b..530c8fb6c8a 100644 --- a/pkg/tsdb/mysql/mysql_test.go +++ b/pkg/tsdb/mysql/mysql_test.go @@ -147,6 +147,7 @@ func TestMySQL(t *testing.T) { So(column[23].(string), ShouldEqual, "val2") So(column[24].(string), ShouldEqual, "a,b") So(column[25].(time.Time).Format("2006-01-02T00:00:00Z"), ShouldEqual, time.Now().UTC().Format("2006-01-02T00:00:00Z")) + So(column[26].(float64), ShouldEqual, float64(1.514764861123456*1e12)) So(column[27], ShouldEqual, nil) So(column[28], ShouldEqual, nil) So(column[29], ShouldEqual, "") @@ -185,10 +186,8 @@ func TestMySQL(t *testing.T) { }) } - for _, s := range series { - _, err = sess.Insert(s) - So(err, ShouldBeNil) - } + _, err = sess.InsertMulti(series) + So(err, ShouldBeNil) Convey("When doing a metric query using timeGroup", func() { query := &tsdb.TsdbQuery{ @@ -309,10 +308,19 @@ func TestMySQL(t *testing.T) { Convey("Given a table with metrics having multiple values and measurements", func() { type metric_values struct { - Time time.Time - Measurement string - ValueOne int64 `xorm:"integer 'valueOne'"` - ValueTwo int64 `xorm:"integer 'valueTwo'"` + Time time.Time `xorm:"datetime 'time' not null"` + TimeNullable *time.Time `xorm:"datetime 'timeNullable' null"` + TimeInt64 int64 `xorm:"bigint(20) 'timeInt64' not null"` + TimeInt64Nullable *int64 `xorm:"bigint(20) 'timeInt64Nullable' null"` + TimeFloat64 float64 `xorm:"double 'timeFloat64' not null"` + TimeFloat64Nullable *float64 `xorm:"double 'timeFloat64Nullable' null"` + TimeInt32 int32 `xorm:"int(11) 'timeInt32' not null"` + TimeInt32Nullable *int32 `xorm:"int(11) 'timeInt32Nullable' null"` + TimeFloat32 float32 `xorm:"double 'timeFloat32' not null"` + TimeFloat32Nullable *float32 `xorm:"double 'timeFloat32Nullable' null"` + Measurement string + ValueOne int64 `xorm:"integer 'valueOne'"` + ValueTwo int64 `xorm:"integer 'valueTwo'"` } if exist, err := sess.IsTableExist(metric_values{}); err != nil || exist { @@ -327,26 +335,265 @@ func TestMySQL(t *testing.T) { return rand.Int63n(max-min) + min } + var tInitial time.Time + series := []*metric_values{} - for _, t := range genTimeRangeByInterval(fromStart.Add(-30*time.Minute), 90*time.Minute, 5*time.Minute) { - series = append(series, &metric_values{ - Time: t, - Measurement: "Metric A", - ValueOne: rnd(0, 100), - ValueTwo: rnd(0, 100), - }) - series = append(series, &metric_values{ - Time: t, - Measurement: "Metric B", - ValueOne: rnd(0, 100), - ValueTwo: rnd(0, 100), - }) + for i, t := range genTimeRangeByInterval(fromStart.Add(-30*time.Minute), 90*time.Minute, 5*time.Minute) { + if i == 0 { + tInitial = t + } + tSeconds := t.Unix() + tSecondsInt32 := int32(tSeconds) + tSecondsFloat32 := float32(tSeconds) + tMilliseconds := tSeconds * 1e3 + tMillisecondsFloat := float64(tMilliseconds) + t2 := t + first := metric_values{ + Time: t, + TimeNullable: &t2, + TimeInt64: tMilliseconds, + TimeInt64Nullable: &(tMilliseconds), + TimeFloat64: tMillisecondsFloat, + TimeFloat64Nullable: &tMillisecondsFloat, + TimeInt32: tSecondsInt32, + TimeInt32Nullable: &tSecondsInt32, + TimeFloat32: tSecondsFloat32, + TimeFloat32Nullable: &tSecondsFloat32, + Measurement: "Metric A", + ValueOne: rnd(0, 100), + ValueTwo: rnd(0, 100), + } + second := first + second.Measurement = "Metric B" + second.ValueOne = rnd(0, 100) + second.ValueTwo = rnd(0, 100) + + series = append(series, &first) + series = append(series, &second) } - for _, s := range series { - _, err := sess.Insert(s) + _, err = sess.InsertMulti(series) + So(err, ShouldBeNil) + + Convey("When doing a metric query using time as time column should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT time, valueOne FROM metric_values ORDER BY time LIMIT 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(nil, nil, query) So(err, ShouldBeNil) - } + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(tInitial.UnixNano()/1e6)) + }) + + Convey("When doing a metric query using time (nullable) as time column should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT timeNullable as time, valueOne FROM metric_values ORDER BY time LIMIT 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(nil, nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(tInitial.UnixNano()/1e6)) + }) + + Convey("When doing a metric query using epoch (int64) as time column should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT timeInt64 as time, valueOne FROM metric_values ORDER BY time LIMIT 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(nil, nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(tInitial.UnixNano()/1e6)) + }) + + Convey("When doing a metric query using epoch (int64 nullable) as time column should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT timeInt64Nullable as time, valueOne FROM metric_values ORDER BY time LIMIT 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(nil, nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(tInitial.UnixNano()/1e6)) + }) + + Convey("When doing a metric query using epoch (float64) as time column should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT timeFloat64 as time, valueOne FROM metric_values ORDER BY time LIMIT 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(nil, nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(tInitial.UnixNano()/1e6)) + }) + + Convey("When doing a metric query using epoch (float64 nullable) as time column should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT timeFloat64Nullable as time, valueOne FROM metric_values ORDER BY time LIMIT 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(nil, nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(tInitial.UnixNano()/1e6)) + }) + + Convey("When doing a metric query using epoch (int32) as time column should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT timeInt32 as time, valueOne FROM metric_values ORDER BY time LIMIT 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(nil, nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(tInitial.UnixNano()/1e6)) + }) + + Convey("When doing a metric query using epoch (int32 nullable) as time column should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT timeInt32Nullable as time, valueOne FROM metric_values ORDER BY time LIMIT 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(nil, nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(tInitial.UnixNano()/1e6)) + }) + + Convey("When doing a metric query using epoch (float32) as time column should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT timeFloat32 as time, valueOne FROM metric_values ORDER BY time LIMIT 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(nil, nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(float64(float32(tInitial.Unix())))*1e3) + }) + + Convey("When doing a metric query using epoch (float32 nullable) as time column should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT timeFloat32Nullable as time, valueOne FROM metric_values ORDER BY time LIMIT 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(nil, nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(float64(float32(tInitial.Unix())))*1e3) + }) Convey("When doing a metric query grouping by time and select metric column should return correct series", func() { query := &tsdb.TsdbQuery{