From 0bfb94dc6fc6419ff89f1d74f838a6130e905c8a Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 10 Oct 2016 07:12:21 +0200 Subject: [PATCH] feat(tsdb): add interval calculator --- pkg/tsdb/influxdb/query_builder.go | 12 +- pkg/tsdb/influxdb/query_builder_test.go | 4 +- pkg/tsdb/influxdb/query_part.go | 22 ++-- pkg/tsdb/influxdb/query_part_test.go | 21 ++-- pkg/tsdb/influxdb/response_parser.go | 8 +- pkg/tsdb/interval.go | 149 ++++++++++++++++++++++++ pkg/tsdb/interval_test.go | 57 +++++++++ 7 files changed, 244 insertions(+), 29 deletions(-) create mode 100644 pkg/tsdb/interval.go create mode 100644 pkg/tsdb/interval_test.go diff --git a/pkg/tsdb/influxdb/query_builder.go b/pkg/tsdb/influxdb/query_builder.go index 846c6a4ba67..5c82a41e78a 100644 --- a/pkg/tsdb/influxdb/query_builder.go +++ b/pkg/tsdb/influxdb/query_builder.go @@ -30,11 +30,11 @@ func renderTags(query *Query) []string { } func (*QueryBuilder) Build(query *Query, queryContext *tsdb.QueryContext) (string, error) { - res := renderSelectors(query) + res := renderSelectors(query, queryContext) res += renderMeasurement(query) res += renderWhereClause(query) res += renderTimeFilter(query, queryContext) - res += renderGroupBy(query) + res += renderGroupBy(query, queryContext) return res, nil } @@ -50,7 +50,7 @@ func renderTimeFilter(query *Query, queryContext *tsdb.QueryContext) string { return fmt.Sprintf("time > %s%s", from, to) } -func renderSelectors(query *Query) string { +func renderSelectors(query *Query, queryContext *tsdb.QueryContext) string { res := "SELECT " var selectors []string @@ -58,7 +58,7 @@ func renderSelectors(query *Query) string { stk := "" for _, s := range *sel { - stk = s.Render(stk) + stk = s.Render(queryContext, stk) } selectors = append(selectors, stk) } @@ -87,7 +87,7 @@ func renderWhereClause(query *Query) string { return res } -func renderGroupBy(query *Query) string { +func renderGroupBy(query *Query, queryContext *tsdb.QueryContext) string { groupBy := "" for i, group := range query.GroupBy { if i == 0 { @@ -100,7 +100,7 @@ func renderGroupBy(query *Query) string { groupBy += " " } - groupBy += group.Render("") + groupBy += group.Render(queryContext, "") } return groupBy diff --git a/pkg/tsdb/influxdb/query_builder_test.go b/pkg/tsdb/influxdb/query_builder_test.go index fcc81986d4c..083d09b360e 100644 --- a/pkg/tsdb/influxdb/query_builder_test.go +++ b/pkg/tsdb/influxdb/query_builder_test.go @@ -37,7 +37,7 @@ func TestInfluxdbQueryBuilder(t *testing.T) { rawQuery, err := builder.Build(query, queryContext) So(err, ShouldBeNil) - So(rawQuery, ShouldEqual, `SELECT mean("value") FROM "policy"."cpu" WHERE time > now() - 5m GROUP BY time(10s) fill(null)`) + So(rawQuery, ShouldEqual, `SELECT mean("value") FROM "policy"."cpu" WHERE time > now() - 5m GROUP BY time(200ms) fill(null)`) }) Convey("can build query with group bys", func() { @@ -51,7 +51,7 @@ func TestInfluxdbQueryBuilder(t *testing.T) { rawQuery, err := builder.Build(query, queryContext) So(err, ShouldBeNil) - So(rawQuery, ShouldEqual, `SELECT mean("value") FROM "cpu" WHERE "hostname" = 'server1' OR "hostname" = 'server2' AND time > now() - 5m GROUP BY time(10s), "datacenter" fill(null)`) + So(rawQuery, ShouldEqual, `SELECT mean("value") FROM "cpu" WHERE "hostname" = 'server1' OR "hostname" = 'server2' AND time > now() - 5m GROUP BY time(200ms), "datacenter" fill(null)`) }) Convey("can render time range", func() { diff --git a/pkg/tsdb/influxdb/query_part.go b/pkg/tsdb/influxdb/query_part.go index ba14bcac665..bace83246c2 100644 --- a/pkg/tsdb/influxdb/query_part.go +++ b/pkg/tsdb/influxdb/query_part.go @@ -3,6 +3,8 @@ package influxdb import ( "fmt" "strings" + + "github.com/grafana/grafana/pkg/tsdb" ) var renders map[string]QueryDefinition @@ -13,7 +15,7 @@ type DefinitionParameters struct { } type QueryDefinition struct { - Renderer func(part *QueryPart, innerExpr string) string + Renderer func(queryContext *tsdb.QueryContext, part *QueryPart, innerExpr string) string Params []DefinitionParameters } @@ -83,17 +85,17 @@ func init() { renders["alias"] = QueryDefinition{Renderer: aliasRenderer} } -func fieldRenderer(part *QueryPart, innerExpr string) string { +func fieldRenderer(queryContext *tsdb.QueryContext, part *QueryPart, innerExpr string) string { if part.Params[0] == "*" { return "*" } return fmt.Sprintf(`"%s"`, part.Params[0]) } -func functionRenderer(part *QueryPart, innerExpr string) string { +func functionRenderer(queryContext *tsdb.QueryContext, part *QueryPart, innerExpr string) string { for i, v := range part.Params { if v == "$interval" { - part.Params[i] = "10s" + part.Params[i] = tsdb.CalculateInterval(queryContext.TimeRange) } } @@ -106,16 +108,16 @@ func functionRenderer(part *QueryPart, innerExpr string) string { return fmt.Sprintf("%s(%s)", part.Type, params) } -func suffixRenderer(part *QueryPart, innerExpr string) string { +func suffixRenderer(queryContext *tsdb.QueryContext, part *QueryPart, innerExpr string) string { return fmt.Sprintf("%s %s", innerExpr, part.Params[0]) } -func aliasRenderer(part *QueryPart, innerExpr string) string { +func aliasRenderer(queryContext *tsdb.QueryContext, part *QueryPart, innerExpr string) string { return fmt.Sprintf(`%s AS "%s"`, innerExpr, part.Params[0]) } -func (r QueryDefinition) Render(part *QueryPart, innerExpr string) string { - return r.Renderer(part, innerExpr) +func (r QueryDefinition) Render(queryContext *tsdb.QueryContext, part *QueryPart, innerExpr string) string { + return r.Renderer(queryContext, part, innerExpr) } func NewQueryPart(typ string, params []string) (*QueryPart, error) { @@ -138,6 +140,6 @@ type QueryPart struct { Params []string } -func (qp *QueryPart) Render(expr string) string { - return qp.Def.Renderer(qp, expr) +func (qp *QueryPart) Render(queryContext *tsdb.QueryContext, expr string) string { + return qp.Def.Renderer(queryContext, qp, expr) } diff --git a/pkg/tsdb/influxdb/query_part_test.go b/pkg/tsdb/influxdb/query_part_test.go index c1b2a22a5a1..bd2d544d3e1 100644 --- a/pkg/tsdb/influxdb/query_part_test.go +++ b/pkg/tsdb/influxdb/query_part_test.go @@ -3,17 +3,22 @@ package influxdb import ( "testing" + "github.com/grafana/grafana/pkg/tsdb" . "github.com/smartystreets/goconvey/convey" ) func TestInfluxdbQueryPart(t *testing.T) { Convey("Influxdb query parts", t, func() { + queryContext := &tsdb.QueryContext{ + TimeRange: tsdb.NewTimeRange("5m", "now"), + } + Convey("render field ", func() { part, err := NewQueryPart("field", []string{"value"}) So(err, ShouldBeNil) - res := part.Render("value") + res := part.Render(queryContext, "value") So(res, ShouldEqual, `"value"`) }) @@ -21,7 +26,7 @@ func TestInfluxdbQueryPart(t *testing.T) { part, err := NewQueryPart("derivative", []string{"10s"}) So(err, ShouldBeNil) - res := part.Render("mean(value)") + res := part.Render(queryContext, "mean(value)") So(res, ShouldEqual, "derivative(mean(value), 10s)") }) @@ -29,7 +34,7 @@ func TestInfluxdbQueryPart(t *testing.T) { part, err := NewQueryPart("bottom", []string{"3"}) So(err, ShouldBeNil) - res := part.Render("value") + res := part.Render(queryContext, "value") So(res, ShouldEqual, "bottom(value, 3)") }) @@ -37,15 +42,15 @@ func TestInfluxdbQueryPart(t *testing.T) { part, err := NewQueryPart("time", []string{"$interval"}) So(err, ShouldBeNil) - res := part.Render("") - So(res, ShouldEqual, "time(10s)") + res := part.Render(queryContext, "") + So(res, ShouldEqual, "time(200ms)") }) Convey("render spread", func() { part, err := NewQueryPart("spread", []string{}) So(err, ShouldBeNil) - res := part.Render("value") + res := part.Render(queryContext, "value") So(res, ShouldEqual, `spread(value)`) }) @@ -53,7 +58,7 @@ func TestInfluxdbQueryPart(t *testing.T) { part, err := NewQueryPart("math", []string{"/ 100"}) So(err, ShouldBeNil) - res := part.Render("mean(value)") + res := part.Render(queryContext, "mean(value)") So(res, ShouldEqual, "mean(value) / 100") }) @@ -61,7 +66,7 @@ func TestInfluxdbQueryPart(t *testing.T) { part, err := NewQueryPart("alias", []string{"test"}) So(err, ShouldBeNil) - res := part.Render("mean(value)") + res := part.Render(queryContext, "mean(value)") So(res, ShouldEqual, `mean(value) AS "test"`) }) }) diff --git a/pkg/tsdb/influxdb/response_parser.go b/pkg/tsdb/influxdb/response_parser.go index 81c90e65419..267ebc99c63 100644 --- a/pkg/tsdb/influxdb/response_parser.go +++ b/pkg/tsdb/influxdb/response_parser.go @@ -18,9 +18,11 @@ func (rp *ResponseParser) Parse(response *Response) *tsdb.QueryResult { rp.parseResult(result.Series, queryRes) } - for _, serie := range queryRes.Series { - glog.Debug("result", "name", serie.Name, "points", serie.Points) - } + /* + for _, serie := range queryRes.Series { + glog.Debug("result", "name", serie.Name, "points", serie.Points) + } + */ return queryRes } diff --git a/pkg/tsdb/interval.go b/pkg/tsdb/interval.go new file mode 100644 index 00000000000..7bcdbfee31a --- /dev/null +++ b/pkg/tsdb/interval.go @@ -0,0 +1,149 @@ +package tsdb + +import ( + "fmt" + "time" + + "github.com/grafana/grafana/pkg/log" +) + +var ( + defaultRes int64 = 1500 + minInterval time.Duration = 1 * time.Millisecond + year time.Duration = time.Hour * 24 * 365 + day time.Duration = time.Hour * 24 * 365 +) + +func CalculateInterval(timerange *TimeRange) string { + interval := time.Duration((timerange.MustGetTo().UnixNano() - timerange.MustGetFrom().UnixNano()) / defaultRes) + + log.Info2("res", "resinMs", time.Duration(interval).String()) + + if interval < minInterval { + return formatDuration(minInterval) + } + + return formatDuration(roundInterval(interval)) +} + +func formatDuration(inter time.Duration) string { + if inter >= year { + return fmt.Sprintf("%dy", inter/year) + } + + if inter >= day { + return fmt.Sprintf("%dd", inter/day) + } + + if inter >= time.Hour { + return fmt.Sprintf("%dh", inter/time.Hour) + } + + if inter >= time.Minute { + return fmt.Sprintf("%dm", inter/time.Minute) + } + + if inter >= time.Second { + return fmt.Sprintf("%ds", inter/time.Second) + } + + if inter >= time.Millisecond { + return fmt.Sprintf("%dms", inter/time.Millisecond) + } + + return "1ms" +} + +func roundInterval(interval time.Duration) time.Duration { + switch true { + // 0.015s + case interval <= 15*time.Millisecond: + return time.Millisecond * 10 // 0.01s + // 0.035s + case interval <= 35*time.Millisecond: + return time.Millisecond * 20 // 0.02s + // 0.075s + case interval <= 75*time.Millisecond: + return time.Millisecond * 50 // 0.05s + // 0.15s + case interval <= 150*time.Millisecond: + return time.Millisecond * 100 // 0.1s + // 0.35s + case interval <= 350*time.Millisecond: + return time.Millisecond * 200 // 0.2s + // 0.75s + case interval <= 750*time.Millisecond: + return time.Millisecond * 500 // 0.5s + // 1.5s + case interval <= 1500*time.Millisecond: + return time.Millisecond * 1000 // 1s + // 3.5s + case interval <= 3500*time.Millisecond: + return time.Millisecond * 2000 // 2s + // 7.5s + case interval <= 7500*time.Millisecond: + return time.Millisecond * 5000 // 5s + // 12.5s + case interval <= 12500*time.Millisecond: + return time.Millisecond * 10000 // 10s + // 17.5s + case interval <= 17500*time.Millisecond: + return time.Millisecond * 15000 // 15s + // 25s + case interval <= 25000*time.Millisecond: + return time.Millisecond * 20000 // 20s + // 45s + case interval <= 45000*time.Millisecond: + return time.Millisecond * 30000 // 30s + // 1.5m + case interval <= 90000*time.Millisecond: + return time.Millisecond * 60000 // 1m + // 3.5m + case interval <= 210000*time.Millisecond: + return time.Millisecond * 120000 // 2m + // 7.5m + case interval <= 450000*time.Millisecond: + return time.Millisecond * 300000 // 5m + // 12.5m + case interval <= 750000*time.Millisecond: + return time.Millisecond * 600000 // 10m + // 12.5m + case interval <= 1050000*time.Millisecond: + return time.Millisecond * 900000 // 15m + // 25m + case interval <= 1500000*time.Millisecond: + return time.Millisecond * 1200000 // 20m + // 45m + case interval <= 2700000*time.Millisecond: + return time.Millisecond * 1800000 // 30m + // 1.5h + case interval <= 5400000*time.Millisecond: + return time.Millisecond * 3600000 // 1h + // 2.5h + case interval <= 9000000*time.Millisecond: + return time.Millisecond * 7200000 // 2h + // 4.5h + case interval <= 16200000*time.Millisecond: + return time.Millisecond * 10800000 // 3h + // 9h + case interval <= 32400000*time.Millisecond: + return time.Millisecond * 21600000 // 6h + // 24h + case interval <= 86400000*time.Millisecond: + return time.Millisecond * 43200000 // 12h + // 48h + case interval <= 172800000*time.Millisecond: + return time.Millisecond * 86400000 // 24h + // 1w + case interval <= 604800000*time.Millisecond: + return time.Millisecond * 86400000 // 24h + // 3w + case interval <= 1814400000*time.Millisecond: + return time.Millisecond * 604800000 // 1w + // 2y + case interval < 3628800000*time.Millisecond: + return time.Millisecond * 2592000000 // 30d + default: + return time.Millisecond * 31536000000 // 1y + } +} diff --git a/pkg/tsdb/interval_test.go b/pkg/tsdb/interval_test.go new file mode 100644 index 00000000000..c06e1879668 --- /dev/null +++ b/pkg/tsdb/interval_test.go @@ -0,0 +1,57 @@ +package tsdb + +import ( + "testing" + "time" + + "github.com/grafana/grafana/pkg/setting" + . "github.com/smartystreets/goconvey/convey" +) + +func TestInterval(t *testing.T) { + Convey("Default interval ", t, func() { + setting.NewConfigContext(&setting.CommandLineArgs{ + HomePath: "../../", + }) + + Convey("for 5min", func() { + tr := NewTimeRange("5m", "now") + + interval := CalculateInterval(tr) + So(interval, ShouldEqual, "200ms") + }) + + Convey("for 15min", func() { + tr := NewTimeRange("15m", "now") + + interval := CalculateInterval(tr) + So(interval, ShouldEqual, "500ms") + }) + + Convey("for 30min", func() { + tr := NewTimeRange("30m", "now") + + interval := CalculateInterval(tr) + So(interval, ShouldEqual, "1s") + }) + + Convey("for 1h", func() { + tr := NewTimeRange("1h", "now") + + interval := CalculateInterval(tr) + So(interval, ShouldEqual, "2s") + }) + + Convey("Round interval", func() { + So(roundInterval(time.Millisecond*30), ShouldEqual, time.Millisecond*20) + So(roundInterval(time.Millisecond*45), ShouldEqual, time.Millisecond*50) + }) + + Convey("Format value", func() { + So(formatDuration(time.Second*61), ShouldEqual, "1m") + So(formatDuration(time.Millisecond*30), ShouldEqual, "30ms") + So(formatDuration(time.Hour*23), ShouldEqual, "23h") + So(formatDuration(time.Hour*24*367), ShouldEqual, "1y") + }) + }) +}