From cc0a39f81662b28d64c9d7737995202ec71fef2c Mon Sep 17 00:00:00 2001 From: utkarshcmu Date: Sun, 16 Oct 2016 04:12:13 -0700 Subject: [PATCH 1/7] Added downsampling params to alerting --- pkg/tsdb/opentsdb/opentsdb.go | 33 ++++++++++++++++++++++++--------- pkg/tsdb/opentsdb/types.go | 11 +++-------- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/pkg/tsdb/opentsdb/opentsdb.go b/pkg/tsdb/opentsdb/opentsdb.go index 75d6e8f2711..55b89fe3390 100644 --- a/pkg/tsdb/opentsdb/opentsdb.go +++ b/pkg/tsdb/opentsdb/opentsdb.go @@ -14,7 +14,7 @@ import ( "io/ioutil" "net/http" "net/url" - //"net/http/httputil" + "net/http/httputil" "encoding/json" "gopkg.in/guregu/null.v3" @@ -58,14 +58,29 @@ func (e *OpenTsdbExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, tsdbQuery.Start = queryContext.TimeRange.GetFromAsMsEpoch() tsdbQuery.End = queryContext.TimeRange.GetToAsMsEpoch() + tsdbQuery.Queries = make([]map[string]interface{}, len(queries)) - for _, query := range queries { - tsdbQuery.Queries = []OpenTsdbMetric{ - OpenTsdbMetric{ - Metric: query.Model.Get("metric").MustString(), - Aggregator: query.Model.Get("aggregator").MustString(), - }, + for i := 0; i < len(queries); i++ { + + metric := make(map[string]interface{}) + + metric["metric"] = queries[i].Model.Get("metric").MustString() + metric["aggregator"] = queries[i].Model.Get("aggregator").MustString() + + disableDownsampling := queries[i].Model.Get("disableDownsampling").MustBool() + + if !disableDownsampling { + downsampleInterval := queries[i].Model.Get("downsampleInterval").MustString() + if downsampleInterval == "" { + downsampleInterval = "1m" //default value for blank + } + downsample := downsampleInterval + "-" + queries[i].Model.Get("downsampleAggregator").MustString() + if queries[i].Model.Get("downsampleFillPolicy").MustString() != "none" { + metric["downsample"] = downsample + "-" + queries[i].Model.Get("downsampleFillPolicy").MustString() + } } + + tsdbQuery.Queries[i] = metric } if setting.Env == setting.DEV { @@ -110,13 +125,13 @@ func (e *OpenTsdbExecutor) createRequest(data OpenTsdbQuery) (*http.Request, err req.SetBasicAuth(e.BasicAuthUser, e.BasicAuthPassword) } - /* + requestDump, err := httputil.DumpRequest(req, true) if err != nil { fmt.Println(err) } fmt.Println(string(requestDump)) - */ + return req, err } diff --git a/pkg/tsdb/opentsdb/types.go b/pkg/tsdb/opentsdb/types.go index 473c797b2d4..abd216dc47d 100644 --- a/pkg/tsdb/opentsdb/types.go +++ b/pkg/tsdb/opentsdb/types.go @@ -1,14 +1,9 @@ package opentsdb type OpenTsdbQuery struct { - Start int64 `json:"start"` - End int64 `json:"end"` - Queries []OpenTsdbMetric `json:"queries"` -} - -type OpenTsdbMetric struct { - Metric string `json:"metric"` - Aggregator string `json:"aggregator"` + Start int64 `json:"start"` + End int64 `json:"end"` + Queries []map[string]interface{} `json:"queries"` } type OpenTsdbResponse struct { From 35f93f41171de843fb9b402d56df4041c49c31fc Mon Sep 17 00:00:00 2001 From: utkarshcmu Date: Sun, 16 Oct 2016 05:20:58 -0700 Subject: [PATCH 2/7] Added rate params to alerting --- pkg/tsdb/opentsdb/opentsdb.go | 39 ++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/pkg/tsdb/opentsdb/opentsdb.go b/pkg/tsdb/opentsdb/opentsdb.go index 55b89fe3390..be5ad75aa00 100644 --- a/pkg/tsdb/opentsdb/opentsdb.go +++ b/pkg/tsdb/opentsdb/opentsdb.go @@ -63,14 +63,15 @@ func (e *OpenTsdbExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, for i := 0; i < len(queries); i++ { metric := make(map[string]interface{}) - + + // Setting metric and aggregator metric["metric"] = queries[i].Model.Get("metric").MustString() metric["aggregator"] = queries[i].Model.Get("aggregator").MustString() + // Setting downsampling options disableDownsampling := queries[i].Model.Get("disableDownsampling").MustBool() - if !disableDownsampling { - downsampleInterval := queries[i].Model.Get("downsampleInterval").MustString() + downsampleInterval := queries[i].Model.Get("downsampleInterval").MustString() if downsampleInterval == "" { downsampleInterval = "1m" //default value for blank } @@ -80,6 +81,28 @@ func (e *OpenTsdbExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, } } + // Setting rate options + if queries[i].Model.Get("shouldComputeRate").MustBool() { + metric["rate"] = true + rateOptions := make(map[string]interface{}) + rateOptions["counter"] = queries[i].Model.Get("isCounter").MustBool() + counterMax, counterMaxCheck := queries[i].Model.CheckGet("counterMax") + if counterMaxCheck { + counterMaxVal, err := strconv.ParseFloat(counterMax.MustString(),64) + if err == nil { + rateOptions["counterMax"] = counterMaxVal + } + } + resetValue, resetValueCheck := queries[i].Model.CheckGet("counterResetValue") + if resetValueCheck { + resetValueVal, err := strconv.ParseFloat(resetValue.MustString(),64) + if err == nil { + rateOptions["resetValue"] = resetValueVal + } + } + metric["rateOptions"] = rateOptions + } + tsdbQuery.Queries[i] = metric } @@ -126,11 +149,11 @@ func (e *OpenTsdbExecutor) createRequest(data OpenTsdbQuery) (*http.Request, err } - requestDump, err := httputil.DumpRequest(req, true) - if err != nil { - fmt.Println(err) - } - fmt.Println(string(requestDump)) + requestDump, err := httputil.DumpRequest(req, true) + if err != nil { + fmt.Println(err) + } + fmt.Println(string(requestDump)) return req, err } From b7744b4928c74ad77a097e9f9c4253c0ef07fa30 Mon Sep 17 00:00:00 2001 From: utkarshcmu Date: Sun, 16 Oct 2016 10:55:07 -0700 Subject: [PATCH 3/7] Added tags and filters params to alerting --- pkg/tsdb/opentsdb/opentsdb.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/pkg/tsdb/opentsdb/opentsdb.go b/pkg/tsdb/opentsdb/opentsdb.go index be5ad75aa00..349592dcde8 100644 --- a/pkg/tsdb/opentsdb/opentsdb.go +++ b/pkg/tsdb/opentsdb/opentsdb.go @@ -14,7 +14,6 @@ import ( "io/ioutil" "net/http" "net/url" - "net/http/httputil" "encoding/json" "gopkg.in/guregu/null.v3" @@ -103,6 +102,18 @@ func (e *OpenTsdbExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, metric["rateOptions"] = rateOptions } + // Setting tags + tags, tagsCheck := queries[i].Model.CheckGet("tags") + if tagsCheck && len(tags.MustMap()) > 0 { + metric["tags"] = tags.MustMap() + } + + // Setting filters + filters, filtersCheck := queries[i].Model.CheckGet("filters") + if filtersCheck && len(filters.MustArray()) > 0 { + metric["filters"] = filters.MustArray() + } + tsdbQuery.Queries[i] = metric } @@ -147,13 +158,6 @@ func (e *OpenTsdbExecutor) createRequest(data OpenTsdbQuery) (*http.Request, err if e.BasicAuth { req.SetBasicAuth(e.BasicAuthUser, e.BasicAuthPassword) } - - - requestDump, err := httputil.DumpRequest(req, true) - if err != nil { - fmt.Println(err) - } - fmt.Println(string(requestDump)) return req, err } From ace81760eaef2ca8153120802e644acad9cb3023 Mon Sep 17 00:00:00 2001 From: utkarshcmu Date: Sun, 16 Oct 2016 12:10:13 -0700 Subject: [PATCH 4/7] Added tests file for writing tests --- pkg/tsdb/opentsdb/opentsdb_test.go | 1 + 1 file changed, 1 insertion(+) create mode 100644 pkg/tsdb/opentsdb/opentsdb_test.go diff --git a/pkg/tsdb/opentsdb/opentsdb_test.go b/pkg/tsdb/opentsdb/opentsdb_test.go new file mode 100644 index 00000000000..f28e2eb52cd --- /dev/null +++ b/pkg/tsdb/opentsdb/opentsdb_test.go @@ -0,0 +1 @@ +package opentsdb From 2374d0987766d96c1cd6be78898c95bae3652f27 Mon Sep 17 00:00:00 2001 From: utkarshcmu Date: Mon, 17 Oct 2016 14:50:53 -0700 Subject: [PATCH 5/7] Fixed query parsing bugs --- pkg/tsdb/opentsdb/opentsdb.go | 116 ++++++++++++++++++---------------- 1 file changed, 61 insertions(+), 55 deletions(-) diff --git a/pkg/tsdb/opentsdb/opentsdb.go b/pkg/tsdb/opentsdb/opentsdb.go index 349592dcde8..bbd67c8658d 100644 --- a/pkg/tsdb/opentsdb/opentsdb.go +++ b/pkg/tsdb/opentsdb/opentsdb.go @@ -60,61 +60,8 @@ func (e *OpenTsdbExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, tsdbQuery.Queries = make([]map[string]interface{}, len(queries)) for i := 0; i < len(queries); i++ { - - metric := make(map[string]interface{}) - - // Setting metric and aggregator - metric["metric"] = queries[i].Model.Get("metric").MustString() - metric["aggregator"] = queries[i].Model.Get("aggregator").MustString() - - // Setting downsampling options - disableDownsampling := queries[i].Model.Get("disableDownsampling").MustBool() - if !disableDownsampling { - downsampleInterval := queries[i].Model.Get("downsampleInterval").MustString() - if downsampleInterval == "" { - downsampleInterval = "1m" //default value for blank - } - downsample := downsampleInterval + "-" + queries[i].Model.Get("downsampleAggregator").MustString() - if queries[i].Model.Get("downsampleFillPolicy").MustString() != "none" { - metric["downsample"] = downsample + "-" + queries[i].Model.Get("downsampleFillPolicy").MustString() - } - } - - // Setting rate options - if queries[i].Model.Get("shouldComputeRate").MustBool() { - metric["rate"] = true - rateOptions := make(map[string]interface{}) - rateOptions["counter"] = queries[i].Model.Get("isCounter").MustBool() - counterMax, counterMaxCheck := queries[i].Model.CheckGet("counterMax") - if counterMaxCheck { - counterMaxVal, err := strconv.ParseFloat(counterMax.MustString(),64) - if err == nil { - rateOptions["counterMax"] = counterMaxVal - } - } - resetValue, resetValueCheck := queries[i].Model.CheckGet("counterResetValue") - if resetValueCheck { - resetValueVal, err := strconv.ParseFloat(resetValue.MustString(),64) - if err == nil { - rateOptions["resetValue"] = resetValueVal - } - } - metric["rateOptions"] = rateOptions - } - - // Setting tags - tags, tagsCheck := queries[i].Model.CheckGet("tags") - if tagsCheck && len(tags.MustMap()) > 0 { - metric["tags"] = tags.MustMap() - } - - // Setting filters - filters, filtersCheck := queries[i].Model.CheckGet("filters") - if filtersCheck && len(filters.MustArray()) > 0 { - metric["filters"] = filters.MustArray() - } - - tsdbQuery.Queries[i] = metric + metric := e.buildMetric(queries[i]) + tsdbQuery.Queries[i] = metric } if setting.Env == setting.DEV { @@ -205,3 +152,62 @@ func (e *OpenTsdbExecutor) parseResponse(query OpenTsdbQuery, res *http.Response queryResults["A"] = queryRes return queryResults, nil } + +func (e *OpenTsdbExecutor) buildMetric(query *tsdb.Query) (map[string]interface{}) { + + metric := make(map[string]interface{}) + + // Setting metric and aggregator + metric["metric"] = query.Model.Get("metric").MustString() + metric["aggregator"] = query.Model.Get("aggregator").MustString() + + // Setting downsampling options + disableDownsampling := query.Model.Get("disableDownsampling").MustBool() + if !disableDownsampling { + downsampleInterval := query.Model.Get("downsampleInterval").MustString() + if downsampleInterval == "" { + downsampleInterval = "1m" //default value for blank + } + downsample := downsampleInterval + "-" + query.Model.Get("downsampleAggregator").MustString() + if query.Model.Get("downsampleFillPolicy").MustString() != "none" { + metric["downsample"] = downsample + "-" + query.Model.Get("downsampleFillPolicy").MustString() + } else { + metric["downsample"] = downsample + } + } + + // Setting rate options + if query.Model.Get("shouldComputeRate").MustBool() { + + metric["rate"] = true + rateOptions := make(map[string]interface{}) + rateOptions["counter"] = query.Model.Get("isCounter").MustBool() + + counterMax, counterMaxCheck := query.Model.CheckGet("counterMax") + if counterMaxCheck { + rateOptions["counterMax"] = counterMax.MustFloat64() + } + + resetValue, resetValueCheck := query.Model.CheckGet("counterResetValue") + if resetValueCheck { + rateOptions["resetValue"] = resetValue.MustFloat64() + } + + metric["rateOptions"] = rateOptions + } + + // Setting tags + tags, tagsCheck := query.Model.CheckGet("tags") + if tagsCheck && len(tags.MustMap()) > 0 { + metric["tags"] = tags.MustMap() + } + + // Setting filters + filters, filtersCheck := query.Model.CheckGet("filters") + if filtersCheck && len(filters.MustArray()) > 0 { + metric["filters"] = filters.MustArray() + } + + return metric + +} From 10d581dd0a290894a57fa7d204f00bcb1b6cb849 Mon Sep 17 00:00:00 2001 From: utkarshcmu Date: Mon, 17 Oct 2016 14:51:51 -0700 Subject: [PATCH 6/7] Added unit tests for query parsing --- pkg/tsdb/opentsdb/opentsdb_test.go | 175 +++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) diff --git a/pkg/tsdb/opentsdb/opentsdb_test.go b/pkg/tsdb/opentsdb/opentsdb_test.go index f28e2eb52cd..905c42b0633 100644 --- a/pkg/tsdb/opentsdb/opentsdb_test.go +++ b/pkg/tsdb/opentsdb/opentsdb_test.go @@ -1 +1,176 @@ package opentsdb + +import ( + "testing" + + "github.com/grafana/grafana/pkg/tsdb" + . "github.com/smartystreets/goconvey/convey" + "github.com/grafana/grafana/pkg/components/simplejson" +) + +func TestOpenTsdbExecutor(t *testing.T) { + Convey("OpenTsdb query testing", t, func() { + + exec := &OpenTsdbExecutor{} + + Convey("Build metric with downsampling enabled", func() { + + query := &tsdb.Query{ + Model: simplejson.New(), + } + + query.Model.Set("metric", "cpu.average.percent") + query.Model.Set("aggregator", "avg") + query.Model.Set("disableDownsampling", false) + query.Model.Set("downsampleInterval", "") + query.Model.Set("downsampleAggregator","avg") + query.Model.Set("downsampleFillPolicy","none") + + metric := exec.buildMetric(query) + + So(len(metric), ShouldEqual, 3) + So(metric["metric"], ShouldEqual, "cpu.average.percent") + So(metric["aggregator"], ShouldEqual, "avg") + So(metric["downsample"], ShouldEqual, "1m-avg") + + }) + + Convey("Build metric with downsampling diabled", func() { + + query := &tsdb.Query{ + Model: simplejson.New(), + } + + query.Model.Set("metric", "cpu.average.percent") + query.Model.Set("aggregator", "avg") + query.Model.Set("disableDownsampling", true) + query.Model.Set("downsampleInterval", "") + query.Model.Set("downsampleAggregator","avg") + query.Model.Set("downsampleFillPolicy","none") + + metric := exec.buildMetric(query) + + So(len(metric), ShouldEqual, 2) + So(metric["metric"], ShouldEqual, "cpu.average.percent") + So(metric["aggregator"], ShouldEqual, "avg") + + }) + + Convey("Build metric with downsampling enabled with params", func() { + + query := &tsdb.Query{ + Model: simplejson.New(), + } + + query.Model.Set("metric", "cpu.average.percent") + query.Model.Set("aggregator", "avg") + query.Model.Set("disableDownsampling", false) + query.Model.Set("downsampleInterval", "5m") + query.Model.Set("downsampleAggregator","sum") + query.Model.Set("downsampleFillPolicy","null") + + metric := exec.buildMetric(query) + + So(len(metric), ShouldEqual, 3) + So(metric["metric"], ShouldEqual, "cpu.average.percent") + So(metric["aggregator"], ShouldEqual, "avg") + So(metric["downsample"], ShouldEqual, "5m-sum-null") + }) + + Convey("Build metric with tags with downsampling disabled", func() { + + query := &tsdb.Query{ + Model: simplejson.New(), + } + + query.Model.Set("metric", "cpu.average.percent") + query.Model.Set("aggregator", "avg") + query.Model.Set("disableDownsampling", true) + query.Model.Set("downsampleInterval", "5m") + query.Model.Set("downsampleAggregator","sum") + query.Model.Set("downsampleFillPolicy","null") + + tags := simplejson.New() + tags.Set("env", "prod") + tags.Set("app", "grafana") + query.Model.Set("tags", tags.MustMap()) + + metric := exec.buildMetric(query) + + So(len(metric), ShouldEqual, 3) + So(metric["metric"], ShouldEqual, "cpu.average.percent") + So(metric["aggregator"], ShouldEqual, "avg") + So(metric["downsample"], ShouldEqual, nil) + So(len(metric["tags"].(map[string]interface{})), ShouldEqual, 2) + So(metric["tags"].(map[string]interface{})["env"], ShouldEqual, "prod") + So(metric["tags"].(map[string]interface{})["app"], ShouldEqual, "grafana") + So(metric["tags"].(map[string]interface{})["ip"], ShouldEqual, nil) + }) + + Convey("Build metric with rate enabled but counter disabled", func() { + + query := &tsdb.Query{ + Model: simplejson.New(), + } + + query.Model.Set("metric", "cpu.average.percent") + query.Model.Set("aggregator", "avg") + query.Model.Set("disableDownsampling", true) + query.Model.Set("shouldComputeRate", true) + query.Model.Set("isCounter",false) + + tags := simplejson.New() + tags.Set("env", "prod") + tags.Set("app", "grafana") + query.Model.Set("tags", tags.MustMap()) + + metric := exec.buildMetric(query) + + So(len(metric), ShouldEqual, 5) + So(metric["metric"], ShouldEqual, "cpu.average.percent") + So(metric["aggregator"], ShouldEqual, "avg") + So(len(metric["tags"].(map[string]interface{})), ShouldEqual, 2) + So(metric["tags"].(map[string]interface{})["env"], ShouldEqual, "prod") + So(metric["tags"].(map[string]interface{})["app"], ShouldEqual, "grafana") + So(metric["tags"].(map[string]interface{})["ip"], ShouldEqual, nil) + So(metric["rate"], ShouldEqual, true) + So(metric["rateOptions"].(map[string]interface{})["counter"], ShouldEqual, false) + }) + + Convey("Build metric with rate and counter enabled", func() { + + query := &tsdb.Query{ + Model: simplejson.New(), + } + + query.Model.Set("metric", "cpu.average.percent") + query.Model.Set("aggregator", "avg") + query.Model.Set("disableDownsampling", true) + query.Model.Set("shouldComputeRate", true) + query.Model.Set("isCounter",true) + query.Model.Set("counterMax",45) + query.Model.Set("counterResetValue",60) + + tags := simplejson.New() + tags.Set("env", "prod") + tags.Set("app", "grafana") + query.Model.Set("tags", tags.MustMap()) + + metric := exec.buildMetric(query) + + So(len(metric), ShouldEqual, 5) + So(metric["metric"], ShouldEqual, "cpu.average.percent") + So(metric["aggregator"], ShouldEqual, "avg") + So(len(metric["tags"].(map[string]interface{})), ShouldEqual, 2) + So(metric["tags"].(map[string]interface{})["env"], ShouldEqual, "prod") + So(metric["tags"].(map[string]interface{})["app"], ShouldEqual, "grafana") + So(metric["tags"].(map[string]interface{})["ip"], ShouldEqual, nil) + So(metric["rate"], ShouldEqual, true) + So(len(metric["rateOptions"].(map[string]interface{})), ShouldEqual, 3) + So(metric["rateOptions"].(map[string]interface{})["counter"], ShouldEqual, true) + So(metric["rateOptions"].(map[string]interface{})["counterMax"], ShouldEqual, 45) + So(metric["rateOptions"].(map[string]interface{})["resetValue"], ShouldEqual, 60) + }) + + }) +} \ No newline at end of file From 333a853948c051f1650f72da6ba0f64603891d3b Mon Sep 17 00:00:00 2001 From: utkarshcmu Date: Tue, 18 Oct 2016 08:47:45 -0700 Subject: [PATCH 7/7] Fixed for loop and append --- pkg/tsdb/opentsdb/opentsdb.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/tsdb/opentsdb/opentsdb.go b/pkg/tsdb/opentsdb/opentsdb.go index bbd67c8658d..3ecd52ca723 100644 --- a/pkg/tsdb/opentsdb/opentsdb.go +++ b/pkg/tsdb/opentsdb/opentsdb.go @@ -57,11 +57,10 @@ func (e *OpenTsdbExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, tsdbQuery.Start = queryContext.TimeRange.GetFromAsMsEpoch() tsdbQuery.End = queryContext.TimeRange.GetToAsMsEpoch() - tsdbQuery.Queries = make([]map[string]interface{}, len(queries)) - for i := 0; i < len(queries); i++ { - metric := e.buildMetric(queries[i]) - tsdbQuery.Queries[i] = metric + for _ , query := range queries { + metric := e.buildMetric(query) + tsdbQuery.Queries = append(tsdbQuery.Queries, metric) } if setting.Env == setting.DEV {