From bd3259d07a8bfe2150a4347b48e214a1a4452b09 Mon Sep 17 00:00:00 2001 From: bergquist Date: Wed, 9 Nov 2016 15:55:47 +0100 Subject: [PATCH 01/18] feat(mqe): init commit --- pkg/cmd/grafana-server/main.go | 1 + pkg/tsdb/mqe/mqe.go | 33 +++++++++++++++++++++++++++++++++ pkg/tsdb/mqe/types.go | 1 + 3 files changed, 35 insertions(+) create mode 100644 pkg/tsdb/mqe/mqe.go create mode 100644 pkg/tsdb/mqe/types.go diff --git a/pkg/cmd/grafana-server/main.go b/pkg/cmd/grafana-server/main.go index a0cb0abf886..61dcf023286 100644 --- a/pkg/cmd/grafana-server/main.go +++ b/pkg/cmd/grafana-server/main.go @@ -21,6 +21,7 @@ import ( _ "github.com/grafana/grafana/pkg/services/alerting/notifiers" _ "github.com/grafana/grafana/pkg/tsdb/graphite" _ "github.com/grafana/grafana/pkg/tsdb/influxdb" + _ "github.com/grafana/grafana/pkg/tsdb/mqe" _ "github.com/grafana/grafana/pkg/tsdb/opentsdb" _ "github.com/grafana/grafana/pkg/tsdb/prometheus" _ "github.com/grafana/grafana/pkg/tsdb/testdata" diff --git a/pkg/tsdb/mqe/mqe.go b/pkg/tsdb/mqe/mqe.go new file mode 100644 index 00000000000..1f77c9ab34d --- /dev/null +++ b/pkg/tsdb/mqe/mqe.go @@ -0,0 +1,33 @@ +package mqe + +import ( + "context" + "net/http" + + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/tsdb" +) + +type MQEExecutor struct { + *tsdb.DataSourceInfo +} + +func NewMQEExecutor(dsInfo *tsdb.DataSourceInfo) tsdb.Executor { + return &MQEExecutor{dsInfo} +} + +var ( + glog log.Logger + HttpClient *http.Client +) + +func init() { + glog = log.New("tsdb.mqe") + tsdb.RegisterExecutor("mqe", NewMQEExecutor) + + HttpClient = tsdb.GetDefaultClient() +} + +func (e *MQEExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, context *tsdb.QueryContext) *tsdb.BatchResult { + return &tsdb.BatchResult{} +} diff --git a/pkg/tsdb/mqe/types.go b/pkg/tsdb/mqe/types.go new file mode 100644 index 00000000000..c6a5b37aeff --- /dev/null +++ b/pkg/tsdb/mqe/types.go @@ -0,0 +1 @@ +package mqe From 118e2a63644c554a83c17f900b67039d02476d43 Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 11 Nov 2016 12:35:09 +0100 Subject: [PATCH 02/18] WIP: more boilerplate --- pkg/tsdb/mqe/model_parser.go | 12 ++++++ pkg/tsdb/mqe/model_parser_test.go | 67 +++++++++++++++++++++++++++++++ pkg/tsdb/mqe/types.go | 6 +++ 3 files changed, 85 insertions(+) create mode 100644 pkg/tsdb/mqe/model_parser.go create mode 100644 pkg/tsdb/mqe/model_parser_test.go diff --git a/pkg/tsdb/mqe/model_parser.go b/pkg/tsdb/mqe/model_parser.go new file mode 100644 index 00000000000..9c9b877f7a4 --- /dev/null +++ b/pkg/tsdb/mqe/model_parser.go @@ -0,0 +1,12 @@ +package mqe + +import ( + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/tsdb" +) + +type MQEQueryParser struct{} + +func (qp *MQEQueryParser) Parse(model *simplejson.Json, dsInfo *tsdb.DataSourceInfo) (*MQEQuery, error) { + return nil, nil +} diff --git a/pkg/tsdb/mqe/model_parser_test.go b/pkg/tsdb/mqe/model_parser_test.go new file mode 100644 index 00000000000..13329e4c662 --- /dev/null +++ b/pkg/tsdb/mqe/model_parser_test.go @@ -0,0 +1,67 @@ +package mqe + +import ( + "testing" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/tsdb" + . "github.com/smartystreets/goconvey/convey" +) + +func TestMQEQueryParser(t *testing.T) { + Convey("MQE query parser", t, func() { + parser := &MQEQueryParser{} + + dsInfo := &tsdb.DataSourceInfo{ + JsonData: simplejson.New(), + } + + Convey("can parse simple mqe model", func() { + json := ` + { + "apps": [], + "hosts": [ + "staples-lab-1" + ], + "metric": "$metric_cpu", + "metrics": [ + { + "metric": "$metric_cpu" + } + ], + "rawQuery": "", + "refId": "A" + } + ` + modelJson, err := simplejson.NewJson([]byte(json)) + So(err, ShouldBeNil) + + res, err := parser.Parse(modelJson, dsInfo) + So(err, ShouldBeNil) + So(res.Interval, ShouldEqual, ">20s") + }) + + Convey("can parse multi serie mqe model", func() { + json := ` + { + "apps": [], + "hosts": [ + "staples-lab-1" + ], + "metrics": [ + { + "metric": "os.cpu.all.active_percentage" + }, + { + "metric": "os.disk.sda.io_time" + } + ], + "rawQuery": "", + "refId": "A", + "addAppToAlias": true, + "addHostToAlias": true + } + ` + }) + }) +} diff --git a/pkg/tsdb/mqe/types.go b/pkg/tsdb/mqe/types.go index c6a5b37aeff..eaebd89993c 100644 --- a/pkg/tsdb/mqe/types.go +++ b/pkg/tsdb/mqe/types.go @@ -1 +1,7 @@ package mqe + +type MQEQuery struct { + Metrics []string + Hosts []string + Apps []string +} From c973241435880ed54d33456d6a75e820c07e9d1f Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 14 Nov 2016 14:00:35 +0100 Subject: [PATCH 03/18] feat(mqe): add token request --- pkg/tsdb/models.go | 26 +++++++++++++ pkg/tsdb/mqe/model_parser_test.go | 13 +++++-- pkg/tsdb/mqe/mqe.go | 1 + pkg/tsdb/mqe/token_client.go | 61 +++++++++++++++++++++++++++++++ pkg/tsdb/mqe/token_client_test.go | 28 ++++++++++++++ pkg/tsdb/mqe/types.go | 19 ++++++++++ 6 files changed, 144 insertions(+), 4 deletions(-) create mode 100644 pkg/tsdb/mqe/token_client.go create mode 100644 pkg/tsdb/mqe/token_client_test.go diff --git a/pkg/tsdb/models.go b/pkg/tsdb/models.go index 366709cfba7..a8ba3c1a648 100644 --- a/pkg/tsdb/models.go +++ b/pkg/tsdb/models.go @@ -1,6 +1,11 @@ package tsdb import ( + "crypto/tls" + "net" + "net/http" + "time" + "github.com/grafana/grafana/pkg/components/simplejson" "gopkg.in/guregu/null.v3" ) @@ -42,6 +47,27 @@ type DataSourceInfo struct { JsonData *simplejson.Json } +func (ds *DataSourceInfo) GetDefaultClient() *http.Client { + tr := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + + return &http.Client{ + Timeout: time.Duration(30 * time.Second), + Transport: tr, + } +} + type BatchTiming struct { TimeElapsed int64 } diff --git a/pkg/tsdb/mqe/model_parser_test.go b/pkg/tsdb/mqe/model_parser_test.go index 13329e4c662..b88698cdbb9 100644 --- a/pkg/tsdb/mqe/model_parser_test.go +++ b/pkg/tsdb/mqe/model_parser_test.go @@ -16,6 +16,8 @@ func TestMQEQueryParser(t *testing.T) { JsonData: simplejson.New(), } + queryContext := &tsdb.QueryContext{} + Convey("can parse simple mqe model", func() { json := ` { @@ -23,10 +25,9 @@ func TestMQEQueryParser(t *testing.T) { "hosts": [ "staples-lab-1" ], - "metric": "$metric_cpu", "metrics": [ { - "metric": "$metric_cpu" + "metric": "os.cpu.all*" } ], "rawQuery": "", @@ -36,9 +37,11 @@ func TestMQEQueryParser(t *testing.T) { modelJson, err := simplejson.NewJson([]byte(json)) So(err, ShouldBeNil) - res, err := parser.Parse(modelJson, dsInfo) + query, err := parser.Parse(modelJson, dsInfo) So(err, ShouldBeNil) - So(res.Interval, ShouldEqual, ">20s") + + rawQuery := query.Build(queryContext) + So(rawQuery, ShouldEqual, "") }) Convey("can parse multi serie mqe model", func() { @@ -62,6 +65,8 @@ func TestMQEQueryParser(t *testing.T) { "addHostToAlias": true } ` + + So(json, ShouldNotBeNil) }) }) } diff --git a/pkg/tsdb/mqe/mqe.go b/pkg/tsdb/mqe/mqe.go index 1f77c9ab34d..094d5414c44 100644 --- a/pkg/tsdb/mqe/mqe.go +++ b/pkg/tsdb/mqe/mqe.go @@ -29,5 +29,6 @@ func init() { } func (e *MQEExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, context *tsdb.QueryContext) *tsdb.BatchResult { + return &tsdb.BatchResult{} } diff --git a/pkg/tsdb/mqe/token_client.go b/pkg/tsdb/mqe/token_client.go new file mode 100644 index 00000000000..618ce671bb2 --- /dev/null +++ b/pkg/tsdb/mqe/token_client.go @@ -0,0 +1,61 @@ +package mqe + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "path" + + "golang.org/x/net/context/ctxhttp" + + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/tsdb" +) + +type TokenClient struct { + tlog log.Logger +} + +func NewTokenClient() *TokenClient { + return &TokenClient{ + tlog: log.New("tsdb.mqe.tokenclient"), + } +} + +func (client *TokenClient) GetTokenData(ctx context.Context, datasource *tsdb.DataSourceInfo) (*TokenResponse, error) { + u, _ := url.Parse(datasource.Url) + u.Path = path.Join(u.Path, "token") + + req, err := http.NewRequest(http.MethodGet, u.String(), nil) + if err != nil { + client.tlog.Info("Failed to create request", "error", err) + } + + res, err := ctxhttp.Do(ctx, HttpClient, req) + if err != nil { + return nil, err + } + + body, err := ioutil.ReadAll(res.Body) + defer res.Body.Close() + if err != nil { + return nil, err + } + + if res.StatusCode/100 != 2 { + client.tlog.Info("Request failed", "status", res.Status, "body", string(body)) + return nil, fmt.Errorf("Request failed status: %v", res.Status) + } + + var result *TokenResponse + err = json.Unmarshal(body, &result) + if err != nil { + client.tlog.Info("Failed to unmarshal graphite response", "error", err, "status", res.Status, "body", string(body)) + return nil, err + } + + return result, nil +} diff --git a/pkg/tsdb/mqe/token_client_test.go b/pkg/tsdb/mqe/token_client_test.go new file mode 100644 index 00000000000..5265a64e513 --- /dev/null +++ b/pkg/tsdb/mqe/token_client_test.go @@ -0,0 +1,28 @@ +package mqe + +import ( + "context" + "testing" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/tsdb" + . "github.com/smartystreets/goconvey/convey" +) + +func TestTokenClient(t *testing.T) { + SkipConvey("Token client", t, func() { + dsInfo := &tsdb.DataSourceInfo{ + JsonData: simplejson.New(), + Url: "", + } + + client := NewTokenClient() + + body, err := client.GetTokenData(context.TODO(), dsInfo) + + So(err, ShouldBeNil) + So(len(body.Body.Functions), ShouldBeGreaterThan, 1) + So(len(body.Body.Metrics), ShouldBeGreaterThan, 1) + So(body.Success, ShouldBeTrue) + }) +} diff --git a/pkg/tsdb/mqe/types.go b/pkg/tsdb/mqe/types.go index eaebd89993c..1c53ae62c5b 100644 --- a/pkg/tsdb/mqe/types.go +++ b/pkg/tsdb/mqe/types.go @@ -1,7 +1,26 @@ package mqe +import ( + "github.com/grafana/grafana/pkg/tsdb" +) + type MQEQuery struct { Metrics []string Hosts []string Apps []string } + +func (q *MQEQuery) Build(queryContext *tsdb.QueryContext) string { + return "" +} + +type TokenBody struct { + Functions []string + Metrics []string + //tagset +} + +type TokenResponse struct { + Success bool + Body TokenBody +} From 1bdda76ba90398ef13b96180123fd54a8e1f99ce Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 14 Nov 2016 14:49:58 +0100 Subject: [PATCH 04/18] tests(mqe): parse mqe query model --- pkg/tsdb/mqe/model_parser.go | 29 ++++++++++++++- pkg/tsdb/mqe/model_parser_test.go | 59 +++++++++++++++++++++++++++---- pkg/tsdb/mqe/response_handler.go | 6 ++++ pkg/tsdb/mqe/token_client.go | 25 +++++++++++-- pkg/tsdb/mqe/token_client_test.go | 7 ++-- pkg/tsdb/mqe/types.go | 12 ++++++- 6 files changed, 124 insertions(+), 14 deletions(-) create mode 100644 pkg/tsdb/mqe/response_handler.go diff --git a/pkg/tsdb/mqe/model_parser.go b/pkg/tsdb/mqe/model_parser.go index 9c9b877f7a4..26bafb6c055 100644 --- a/pkg/tsdb/mqe/model_parser.go +++ b/pkg/tsdb/mqe/model_parser.go @@ -8,5 +8,32 @@ import ( type MQEQueryParser struct{} func (qp *MQEQueryParser) Parse(model *simplejson.Json, dsInfo *tsdb.DataSourceInfo) (*MQEQuery, error) { - return nil, nil + query := &MQEQuery{} + + query.AddAppToAlias = model.Get("addAppToAlias").MustBool(false) + query.AddHostToAlias = model.Get("addHostToAlias").MustBool(false) + query.UseRawQuery = model.Get("rawQuery").MustBool(false) + query.RawQuery = model.Get("query").MustString("") + + query.Apps = model.Get("apps").MustStringArray([]string{}) + query.Hosts = model.Get("hosts").MustStringArray([]string{}) + + var metrics []MQEMetric + var err error + for _, metricsObj := range model.Get("metrics").MustArray() { + metricJson := simplejson.NewFromAny(metricsObj) + var m MQEMetric + + m.Alias = metricJson.Get("alias").MustString("") + m.Metric, err = metricJson.Get("metric").String() + if err != nil { + return nil, err + } + + metrics = append(metrics, m) + } + + query.Metrics = metrics + + return query, nil } diff --git a/pkg/tsdb/mqe/model_parser_test.go b/pkg/tsdb/mqe/model_parser_test.go index b88698cdbb9..60b83914a5c 100644 --- a/pkg/tsdb/mqe/model_parser_test.go +++ b/pkg/tsdb/mqe/model_parser_test.go @@ -16,8 +16,6 @@ func TestMQEQueryParser(t *testing.T) { JsonData: simplejson.New(), } - queryContext := &tsdb.QueryContext{} - Convey("can parse simple mqe model", func() { json := ` { @@ -39,15 +37,19 @@ func TestMQEQueryParser(t *testing.T) { query, err := parser.Parse(modelJson, dsInfo) So(err, ShouldBeNil) + So(query.UseRawQuery, ShouldBeFalse) - rawQuery := query.Build(queryContext) - So(rawQuery, ShouldEqual, "") + So(len(query.Apps), ShouldEqual, 0) + So(query.Hosts[0], ShouldEqual, "staples-lab-1") + So(query.Metrics[0].Metric, ShouldEqual, "os.cpu.all*") }) Convey("can parse multi serie mqe model", func() { json := ` { - "apps": [], + "apps": [ + "demoapp" + ], "hosts": [ "staples-lab-1" ], @@ -65,8 +67,53 @@ func TestMQEQueryParser(t *testing.T) { "addHostToAlias": true } ` + modelJson, err := simplejson.NewJson([]byte(json)) + So(err, ShouldBeNil) - So(json, ShouldNotBeNil) + query, err := parser.Parse(modelJson, dsInfo) + So(err, ShouldBeNil) + So(query.UseRawQuery, ShouldBeFalse) + So(query.Apps[0], ShouldEqual, "demoapp") + So(query.Metrics[0].Metric, ShouldEqual, "os.cpu.all.active_percentage") + So(query.Metrics[1].Metric, ShouldEqual, "os.disk.sda.io_time") + }) + + Convey("can parse raw query", func() { + json := ` + { + "addAppToAlias": true, + "addHostToAlias": true, + "apps": [], + "hosts": [ + "staples-lab-1" + ], + "metrics": [ + { + "alias": "cpu active", + "metric": "os.cpu.all.active_percentage" + }, + { + "alias": "disk sda time", + "metric": "os.disk.sda.io_time" + } + ], + "rawQuery": true, + "query": "raw-query", + "refId": "A", + "addAppToAlias": true, + "addHostToAlias": true + } + ` + modelJson, err := simplejson.NewJson([]byte(json)) + So(err, ShouldBeNil) + + query, err := parser.Parse(modelJson, dsInfo) + So(err, ShouldBeNil) + + So(query.UseRawQuery, ShouldBeTrue) + So(query.RawQuery, ShouldEqual, "raw-query") + So(query.AddAppToAlias, ShouldBeTrue) + So(query.AddHostToAlias, ShouldBeTrue) }) }) } diff --git a/pkg/tsdb/mqe/response_handler.go b/pkg/tsdb/mqe/response_handler.go new file mode 100644 index 00000000000..c8551ac7a3f --- /dev/null +++ b/pkg/tsdb/mqe/response_handler.go @@ -0,0 +1,6 @@ +package mqe + +// wildcard as alias +// add host to alias +// add app to alias +// regular alias diff --git a/pkg/tsdb/mqe/token_client.go b/pkg/tsdb/mqe/token_client.go index 618ce671bb2..19e05954dc0 100644 --- a/pkg/tsdb/mqe/token_client.go +++ b/pkg/tsdb/mqe/token_client.go @@ -25,7 +25,24 @@ func NewTokenClient() *TokenClient { } } -func (client *TokenClient) GetTokenData(ctx context.Context, datasource *tsdb.DataSourceInfo) (*TokenResponse, error) { +var cache map[int64]*TokenBody = map[int64]*TokenBody{} + +//Replace this stupid cache with internal cache from grafana master before merging +func (client *TokenClient) GetTokenData(ctx context.Context, datasource *tsdb.DataSourceInfo) (*TokenBody, error) { + _, excist := cache[datasource.Id] + if !excist { + b, err := client.RequestTokenData(ctx, datasource) + if err != nil { + return nil, err + } + + cache[datasource.Id] = b + } + + return cache[datasource.Id], nil +} + +func (client *TokenClient) RequestTokenData(ctx context.Context, datasource *tsdb.DataSourceInfo) (*TokenBody, error) { u, _ := url.Parse(datasource.Url) u.Path = path.Join(u.Path, "token") @@ -57,5 +74,9 @@ func (client *TokenClient) GetTokenData(ctx context.Context, datasource *tsdb.Da return nil, err } - return result, nil + if !result.Success { + return nil, fmt.Errorf("Request failed for unknown reason.") + } + + return &result.Body, nil } diff --git a/pkg/tsdb/mqe/token_client_test.go b/pkg/tsdb/mqe/token_client_test.go index 5265a64e513..4b68b067362 100644 --- a/pkg/tsdb/mqe/token_client_test.go +++ b/pkg/tsdb/mqe/token_client_test.go @@ -18,11 +18,10 @@ func TestTokenClient(t *testing.T) { client := NewTokenClient() - body, err := client.GetTokenData(context.TODO(), dsInfo) + body, err := client.RequestTokenData(context.TODO(), dsInfo) So(err, ShouldBeNil) - So(len(body.Body.Functions), ShouldBeGreaterThan, 1) - So(len(body.Body.Metrics), ShouldBeGreaterThan, 1) - So(body.Success, ShouldBeTrue) + So(len(body.Functions), ShouldBeGreaterThan, 1) + So(len(body.Metrics), ShouldBeGreaterThan, 1) }) } diff --git a/pkg/tsdb/mqe/types.go b/pkg/tsdb/mqe/types.go index 1c53ae62c5b..ca273a57e94 100644 --- a/pkg/tsdb/mqe/types.go +++ b/pkg/tsdb/mqe/types.go @@ -4,10 +4,20 @@ import ( "github.com/grafana/grafana/pkg/tsdb" ) +type MQEMetric struct { + Metric string + Alias string +} + type MQEQuery struct { - Metrics []string + Metrics []MQEMetric Hosts []string Apps []string + + AddAppToAlias bool + AddHostToAlias bool + UseRawQuery bool + RawQuery string } func (q *MQEQuery) Build(queryContext *tsdb.QueryContext) string { From 4739608ffb0ea3c38553250fe0aefee86b1c2ef9 Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 15 Nov 2016 16:22:42 +0100 Subject: [PATCH 05/18] tests(mqe): add query builder --- pkg/tsdb/mqe/mqe.go | 10 ++++ pkg/tsdb/mqe/token_client_test.go | 2 +- pkg/tsdb/mqe/types.go | 84 +++++++++++++++++++++++++++---- pkg/tsdb/mqe/types_test.go | 79 +++++++++++++++++++++++++++++ 4 files changed, 163 insertions(+), 12 deletions(-) create mode 100644 pkg/tsdb/mqe/types_test.go diff --git a/pkg/tsdb/mqe/mqe.go b/pkg/tsdb/mqe/mqe.go index 094d5414c44..c3f10530416 100644 --- a/pkg/tsdb/mqe/mqe.go +++ b/pkg/tsdb/mqe/mqe.go @@ -30,5 +30,15 @@ func init() { func (e *MQEExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, context *tsdb.QueryContext) *tsdb.BatchResult { + availableSeries, _ := NewTokenClient().GetTokenData(ctx, e.DataSourceInfo) + glog.Info("available series", availableSeries) + + //query, _ := &MQEQueryParser{}.Parse() + + + + //fetch all available serienames + //expaned parsed model into multiple queries + return &tsdb.BatchResult{} } diff --git a/pkg/tsdb/mqe/token_client_test.go b/pkg/tsdb/mqe/token_client_test.go index 4b68b067362..6c74dd90c89 100644 --- a/pkg/tsdb/mqe/token_client_test.go +++ b/pkg/tsdb/mqe/token_client_test.go @@ -21,7 +21,7 @@ func TestTokenClient(t *testing.T) { body, err := client.RequestTokenData(context.TODO(), dsInfo) So(err, ShouldBeNil) - So(len(body.Functions), ShouldBeGreaterThan, 1) + //So(len(body.Functions), ShouldBeGreaterThan, 1) So(len(body.Metrics), ShouldBeGreaterThan, 1) }) } diff --git a/pkg/tsdb/mqe/types.go b/pkg/tsdb/mqe/types.go index ca273a57e94..bd387cc3058 100644 --- a/pkg/tsdb/mqe/types.go +++ b/pkg/tsdb/mqe/types.go @@ -1,6 +1,10 @@ package mqe import ( + "fmt" + + "strings" + "github.com/grafana/grafana/pkg/tsdb" ) @@ -10,27 +14,85 @@ type MQEMetric struct { } type MQEQuery struct { - Metrics []MQEMetric - Hosts []string - Apps []string - + Metrics []MQEMetric + Hosts []string + Apps []string AddAppToAlias bool AddHostToAlias bool - UseRawQuery bool - RawQuery string + + TimeRange *tsdb.TimeRange + UseRawQuery bool + RawQuery string } -func (q *MQEQuery) Build(queryContext *tsdb.QueryContext) string { - return "" +//`os.disk.sda.io_time` where host in ('staples-lab-1') from 1479197578194 to 1479219178194 +func (q *MQEQuery) Build(availableSeries []string) ([]string, error) { + var queries []string + where := q.buildWhereClause() + + var metrics [] + + for _, v := range q.Metrics { + if noStar { + metrics = append(metrics, v) + + continue + } + + for _, a := range availableSeries { + if match { + metrics = append(metrics, a) + } + } + } + + for _, v := range metrics { + queries = append(queries, + fmt.Sprintf( + "`%s` %s from %v to %v", + v.Metric, + where, + q.TimeRange.GetFromAsMsEpoch(), + q.TimeRange.GetToAsMsEpoch())) + } + + return queries, nil +} + +func (q *MQEQuery) buildWhereClause() string { + hasApps := len(q.Apps) > 0 + hasHosts := len(q.Hosts) > 0 + + where := "" + if hasHosts || hasApps { + where += "where " + } + + if hasApps { + apps := strings.Join(q.Apps, "', '") + where += fmt.Sprintf(" apps in ('%s')", apps) + } + + if hasHosts && hasApps { + where += " and" + } + + if hasHosts { + hosts := strings.Join(q.Hosts, "', '") + where += fmt.Sprintf(" hosts in ('%s')", hosts) + } + + return where } type TokenBody struct { - Functions []string - Metrics []string - //tagset + Metrics []string } type TokenResponse struct { Success bool Body TokenBody } + +type MQEResponse struct { +} diff --git a/pkg/tsdb/mqe/types_test.go b/pkg/tsdb/mqe/types_test.go new file mode 100644 index 00000000000..e78cb6e54c9 --- /dev/null +++ b/pkg/tsdb/mqe/types_test.go @@ -0,0 +1,79 @@ +package mqe + +import ( + "testing" + + "time" + + "fmt" + + "github.com/grafana/grafana/pkg/tsdb" + . "github.com/smartystreets/goconvey/convey" +) + +func TestWildcardExpansion(t *testing.T) { + availableMetrics := map[string]bool{ + "os.cpu.all.idle": true, + "os.cpu.1.idle": true, + "os.cpu.2.idle": true, + "os.cpu.3.idle": true, + } + + now := time.Now() + from := now.Add((time.Minute*5)*-1).UnixNano() / int64(time.Millisecond) + to := now.UnixNano() / int64(time.Millisecond) + + Convey("Can expanding query", t, func() { + + Convey("Without wildcard series", func() { + query := &MQEQuery{ + Metrics: []MQEMetric{ + MQEMetric{ + Metric: "os.cpu.3.idle", + Alias: "cpu on core 3", + }, + MQEMetric{ + Metric: "os.cpu.2.idle", + Alias: "cpu on core 2", + }, + }, + Hosts: []string{"staples-lab-1", "staples-lab-2"}, + Apps: []string{"demoapp-1", "demoapp-2"}, + AddAppToAlias: false, + AddHostToAlias: false, + TimeRange: &tsdb.TimeRange{Now: now, From: "5m", To: "now"}, + } + + expandeQueries, err := query.Build(availableMetrics) + So(err, ShouldBeNil) + So(len(expandeQueries), ShouldEqual, 2) + So(expandeQueries[0], ShouldEqual, fmt.Sprintf("`os.cpu.3.idle` where host in ('staples-lab-1', 'staples-lab-2') from %v to %v", from, to)) + So(expandeQueries[1], ShouldEqual, fmt.Sprintf("`os.cpu.2.idle` where host in ('staples-lab-1', 'staples-lab-2') from %v to %v", from, to)) + }) + + Convey("Containg wildcard series", func() { + query := &MQEQuery{ + Metrics: []MQEMetric{ + MQEMetric{ + Metric: "os.cpu*", + Alias: "cpu on core *", + }, + }, + Hosts: []string{"staples-lab-1"}, + AddAppToAlias: false, + AddHostToAlias: false, + TimeRange: &tsdb.TimeRange{Now: now, From: "5m", To: "now"}, + } + + expandeQueries, err := query.Build(availableMetrics) + So(err, ShouldBeNil) + So(len(expandeQueries), ShouldEqual, 4) + + So(expandeQueries[0], ShouldEqual, fmt.Sprintf("`os.cpu.all.idle` where host in ('staples-lab-1') from %v to %v", from, to)) + So(expandeQueries[1], ShouldEqual, fmt.Sprintf("`os.cpu.1.idle` where host in ('staples-lab-1') from %v to %v", from, to)) + So(expandeQueries[2], ShouldEqual, fmt.Sprintf("`os.cpu.2.idle` where host in ('staples-lab-1') from %v to %v", from, to)) + So(expandeQueries[3], ShouldEqual, fmt.Sprintf("`os.cpu.3.idle` where host in ('staples-lab-1') from %v to %v", from, to)) + + }) + }) +} From fa92dfb7b23758bdafeec0f16df9f3f7bd003351 Mon Sep 17 00:00:00 2001 From: bergquist Date: Wed, 16 Nov 2016 09:54:45 +0100 Subject: [PATCH 06/18] feat(mqe): add support for apps in where clause --- pkg/tsdb/mqe/types.go | 39 ++++++++++++++++++++++---------------- pkg/tsdb/mqe/types_test.go | 14 +++++++------- 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/pkg/tsdb/mqe/types.go b/pkg/tsdb/mqe/types.go index bd387cc3058..de5ff2fc8e9 100644 --- a/pkg/tsdb/mqe/types.go +++ b/pkg/tsdb/mqe/types.go @@ -5,6 +5,8 @@ import ( "strings" + "regexp" + "github.com/grafana/grafana/pkg/tsdb" ) @@ -25,32 +27,37 @@ type MQEQuery struct { RawQuery string } +var ( + containsWildcardPattern *regexp.Regexp = regexp.MustCompile(`\*`) +) + //`os.disk.sda.io_time` where host in ('staples-lab-1') from 1479197578194 to 1479219178194 func (q *MQEQuery) Build(availableSeries []string) ([]string, error) { var queries []string where := q.buildWhereClause() - var metrics [] + var metrics []string for _, v := range q.Metrics { - if noStar { - metrics = append(metrics, v) - - continue - } - - for _, a := range availableSeries { - if match { - metrics = append(metrics, a) - } + if !containsWildcardPattern.Match([]byte(v.Metric)) { + metrics = append(metrics, v.Metric) + continue } + + /* + for _, a := range availableSeries { + if match { + metrics = append(metrics, a) + } + } + */ } - for _, v := range metrics { + for _, metric := range metrics { queries = append(queries, fmt.Sprintf( "`%s` %s from %v to %v", - v.Metric, + metric, where, q.TimeRange.GetFromAsMsEpoch(), q.TimeRange.GetToAsMsEpoch())) @@ -70,16 +77,16 @@ func (q *MQEQuery) buildWhereClause() string { if hasApps { apps := strings.Join(q.Apps, "', '") - where += fmt.Sprintf(" apps in ('%s')", apps) + where += fmt.Sprintf("app in ('%s')", apps) } if hasHosts && hasApps { - where += " and" + where += " and " } if hasHosts { hosts := strings.Join(q.Hosts, "', '") - where += fmt.Sprintf(" hosts in ('%s')", hosts) + where += fmt.Sprintf("host in ('%s')", hosts) } return where diff --git a/pkg/tsdb/mqe/types_test.go b/pkg/tsdb/mqe/types_test.go index e78cb6e54c9..c136599cb93 100644 --- a/pkg/tsdb/mqe/types_test.go +++ b/pkg/tsdb/mqe/types_test.go @@ -12,11 +12,11 @@ import ( ) func TestWildcardExpansion(t *testing.T) { - availableMetrics := map[string]bool{ - "os.cpu.all.idle": true, - "os.cpu.1.idle": true, - "os.cpu.2.idle": true, - "os.cpu.3.idle": true, + availableMetrics := []string{ + "os.cpu.all.idle", + "os.cpu.1.idle", + "os.cpu.2.idle", + "os.cpu.3.idle", } now := time.Now() @@ -47,8 +47,8 @@ func TestWildcardExpansion(t *testing.T) { expandeQueries, err := query.Build(availableMetrics) So(err, ShouldBeNil) So(len(expandeQueries), ShouldEqual, 2) - So(expandeQueries[0], ShouldEqual, fmt.Sprintf("`os.cpu.3.idle` where host in ('staples-lab-1', 'staples-lab-2') from %v to %v", from, to)) - So(expandeQueries[1], ShouldEqual, fmt.Sprintf("`os.cpu.2.idle` where host in ('staples-lab-1', 'staples-lab-2') from %v to %v", from, to)) + So(expandeQueries[0], ShouldEqual, fmt.Sprintf("`os.cpu.3.idle` where app in ('demoapp-1', 'demoapp-2') and host in ('staples-lab-1', 'staples-lab-2') from %v to %v", from, to)) + So(expandeQueries[1], ShouldEqual, fmt.Sprintf("`os.cpu.2.idle` where app in ('demoapp-1', 'demoapp-2') and host in ('staples-lab-1', 'staples-lab-2') from %v to %v", from, to)) }) Convey("Containg wildcard series", func() { From 430104aaacf584f49cadb413c25ec1c7c7c74d72 Mon Sep 17 00:00:00 2001 From: bergquist Date: Wed, 16 Nov 2016 10:03:22 +0100 Subject: [PATCH 07/18] feat(mqe): add support for wildcard series --- pkg/tsdb/mqe/types.go | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/pkg/tsdb/mqe/types.go b/pkg/tsdb/mqe/types.go index de5ff2fc8e9..34e643adc8f 100644 --- a/pkg/tsdb/mqe/types.go +++ b/pkg/tsdb/mqe/types.go @@ -7,6 +7,7 @@ import ( "regexp" + "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/tsdb" ) @@ -31,7 +32,6 @@ var ( containsWildcardPattern *regexp.Regexp = regexp.MustCompile(`\*`) ) -//`os.disk.sda.io_time` where host in ('staples-lab-1') from 1479197578194 to 1479219178194 func (q *MQEQuery) Build(availableSeries []string) ([]string, error) { var queries []string where := q.buildWhereClause() @@ -44,13 +44,20 @@ func (q *MQEQuery) Build(availableSeries []string) ([]string, error) { continue } - /* - for _, a := range availableSeries { - if match { - metrics = append(metrics, a) - } + m := strings.Replace(v.Metric, "*", ".*", -1) + mp, err := regexp.Compile(m) + + if err != nil { + log.Error2("failed to compile regex for ", "metric", m) + continue + } + + //TODO: this lookup should be cached + for _, a := range availableSeries { + if mp.Match([]byte(a)) { + metrics = append(metrics, a) } - */ + } } for _, metric := range metrics { From 113020aabeaeb77a6748c6013f4b262d1ba82ff1 Mon Sep 17 00:00:00 2001 From: bergquist Date: Wed, 16 Nov 2016 11:04:15 +0100 Subject: [PATCH 08/18] feat(mqe): support basic aliases --- pkg/tsdb/mqe/types.go | 26 +++++++++++++++++--------- pkg/tsdb/mqe/types_test.go | 11 ++++++++--- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/pkg/tsdb/mqe/types.go b/pkg/tsdb/mqe/types.go index 34e643adc8f..ba50b02ba04 100644 --- a/pkg/tsdb/mqe/types.go +++ b/pkg/tsdb/mqe/types.go @@ -33,14 +33,10 @@ var ( ) func (q *MQEQuery) Build(availableSeries []string) ([]string, error) { - var queries []string - where := q.buildWhereClause() - - var metrics []string - + var metrics []MQEMetric for _, v := range q.Metrics { if !containsWildcardPattern.Match([]byte(v.Metric)) { - metrics = append(metrics, v.Metric) + metrics = append(metrics, v) continue } @@ -55,16 +51,28 @@ func (q *MQEQuery) Build(availableSeries []string) ([]string, error) { //TODO: this lookup should be cached for _, a := range availableSeries { if mp.Match([]byte(a)) { - metrics = append(metrics, a) + metrics = append(metrics, MQEMetric{ + Metric: a, + Alias: v.Alias, + }) } } } + var queries []string + where := q.buildWhereClause() + for _, metric := range metrics { + alias := "" + if metric.Alias != "" { + alias = fmt.Sprintf(" {%s}", metric.Alias) + } + queries = append(queries, fmt.Sprintf( - "`%s` %s from %v to %v", - metric, + "`%s`%s %s from %v to %v", + metric.Metric, + alias, where, q.TimeRange.GetFromAsMsEpoch(), q.TimeRange.GetToAsMsEpoch())) diff --git a/pkg/tsdb/mqe/types_test.go b/pkg/tsdb/mqe/types_test.go index c136599cb93..c01b229e72e 100644 --- a/pkg/tsdb/mqe/types_test.go +++ b/pkg/tsdb/mqe/types_test.go @@ -30,11 +30,15 @@ func TestWildcardExpansion(t *testing.T) { Metrics: []MQEMetric{ MQEMetric{ Metric: "os.cpu.3.idle", - Alias: "cpu on core 3", + Alias: "", }, MQEMetric{ Metric: "os.cpu.2.idle", - Alias: "cpu on core 2", + Alias: "", + }, + MQEMetric{ + Metric: "os.cpu.1.idle", + Alias: "cpu", }, }, Hosts: []string{"staples-lab-1", "staples-lab-2"}, @@ -46,9 +50,10 @@ func TestWildcardExpansion(t *testing.T) { expandeQueries, err := query.Build(availableMetrics) So(err, ShouldBeNil) - So(len(expandeQueries), ShouldEqual, 2) + So(len(expandeQueries), ShouldEqual, 3) So(expandeQueries[0], ShouldEqual, fmt.Sprintf("`os.cpu.3.idle` where app in ('demoapp-1', 'demoapp-2') and host in ('staples-lab-1', 'staples-lab-2') from %v to %v", from, to)) So(expandeQueries[1], ShouldEqual, fmt.Sprintf("`os.cpu.2.idle` where app in ('demoapp-1', 'demoapp-2') and host in ('staples-lab-1', 'staples-lab-2') from %v to %v", from, to)) + So(expandeQueries[2], ShouldEqual, fmt.Sprintf("`os.cpu.1.idle` {cpu} where app in ('demoapp-1', 'demoapp-2') and host in ('staples-lab-1', 'staples-lab-2') from %v to %v", from, to)) }) Convey("Containg wildcard series", func() { From c6ad0cc5ecef05167f86470facd852b2350bb0da Mon Sep 17 00:00:00 2001 From: bergquist Date: Wed, 16 Nov 2016 13:00:25 +0100 Subject: [PATCH 09/18] test(mqe): basic test for response parser --- pkg/tsdb/mqe/response_handler.go | 6 -- pkg/tsdb/mqe/response_parser.go | 18 ++++ pkg/tsdb/mqe/response_parser_test.go | 140 +++++++++++++++++++++++++++ pkg/tsdb/mqe/types.go | 3 - pkg/tsdb/mqe/types_test.go | 22 +---- 5 files changed, 162 insertions(+), 27 deletions(-) delete mode 100644 pkg/tsdb/mqe/response_handler.go create mode 100644 pkg/tsdb/mqe/response_parser.go create mode 100644 pkg/tsdb/mqe/response_parser_test.go diff --git a/pkg/tsdb/mqe/response_handler.go b/pkg/tsdb/mqe/response_handler.go deleted file mode 100644 index c8551ac7a3f..00000000000 --- a/pkg/tsdb/mqe/response_handler.go +++ /dev/null @@ -1,6 +0,0 @@ -package mqe - -// wildcard as alias -// add host to alias -// add app to alias -// regular alias diff --git a/pkg/tsdb/mqe/response_parser.go b/pkg/tsdb/mqe/response_parser.go new file mode 100644 index 00000000000..69b620a5790 --- /dev/null +++ b/pkg/tsdb/mqe/response_parser.go @@ -0,0 +1,18 @@ +package mqe + +import ( + "net/http" + + "github.com/grafana/grafana/pkg/tsdb" +) + +// wildcard as alias +// add host to alias +// add app to alias +// regular alias + +type MQEResponseParser struct{} + +func (parser *MQEResponseParser) Parse(res *http.Response) (*tsdb.QueryResult, error) { + return nil, nil +} diff --git a/pkg/tsdb/mqe/response_parser_test.go b/pkg/tsdb/mqe/response_parser_test.go new file mode 100644 index 00000000000..eabc437c8a1 --- /dev/null +++ b/pkg/tsdb/mqe/response_parser_test.go @@ -0,0 +1,140 @@ +package mqe + +import ( + "testing" + + "time" + + "net/http" + "strings" + + "io/ioutil" + + "github.com/grafana/grafana/pkg/components/simplejson" + . "github.com/smartystreets/goconvey/convey" +) + +var ( + dummieJson string +) + +func TestMQEResponseParser(t *testing.T) { + Convey("MQE response parser", t, func() { + parser := &MQEResponseParser{} + + Convey("Can parse response", func() { + response := &http.Response{ + StatusCode: 200, + Body: ioutil.NopCloser(strings.NewReader(dummieJson)), + } + _, err := parser.Parse(response) + So(err, ShouldBeNil) + }) + }) +} + +type MQEResponse struct { + Success bool `json:"success"` + Name string `json:"name"` + Body []MQEResponseSerie `json:"body"` +} + +type ResponseTimeRange struct { + Start time.Time `json:"start"` + End time.Time `json:"end"` + Resolution time.Duration `json:"Resolution"` +} + +type MQEResponseSerie struct { + Query string `json:"query"` + Name string `json:"name"` + Type string `json:"type"` + Series []simplejson.Json `json:"series"` +} + +func init() { + dummieJson = `{ + "success": true, + "name": "select", + "body": [ + { + "query": "os.disk.sda3.weighted_io_time", + "name": "os.disk.sda3.weighted_io_time", + "type": "series", + "series": [ + { + "tagset": { + "app": "demoapp", + "host": "staples-lab-1" + }, + "values": [1,2,3,4,5,6,7,8,9,10,11] + }, + { + "tagset": { + "app": "demoapp", + "host": "staples-lab-2" + }, + "values": [11,10,9,8,7,6,5,4,3,2,1] + } + ], + "timerange": { + "start": 1479287280000, + "end": 1479287580000, + "resolution": 30000 + } + } + ], + "metadata": { + "description": { + "app": [ + "demoapp" + ], + "host": [ + "staples-lab-1", + "staples-lab-2" + ] + }, + "notes": null, + "profile": [ + { + "name": "Parsing Query", + "start": "2016-11-16T04:16:21.874354721-05:00", + "finish": "2016-11-16T04:16:21.874762291-05:00" + }, + { + "name": "Cassandra GetAllTags", + "start": "2016-11-16T04:16:21.874907171-05:00", + "finish": "2016-11-16T04:16:21.876401922-05:00" + }, + { + "name": "CachedMetricMetadataAPI_GetAllTags_Expired", + "start": "2016-11-16T04:16:21.874904751-05:00", + "finish": "2016-11-16T04:16:21.876407852-05:00" + }, + { + "name": "CachedMetricMetadataAPI_GetAllTags", + "start": "2016-11-16T04:16:21.874899491-05:00", + "finish": "2016-11-16T04:16:21.876410382-05:00" + }, + { + "name": "Blueflood FetchSingleTimeseries Resolution", + "description": "os.disk.sda3.weighted_io_time [app=demoapp,host=staples-lab-1]\n at 30s", + "start": "2016-11-16T04:16:21.876623312-05:00", + "finish": "2016-11-16T04:16:21.881763444-05:00" + }, + { + "name": "Blueflood FetchSingleTimeseries Resolution", + "description": "os.disk.sda3.weighted_io_time [app=demoapp,host=staples-lab-2]\n at 30s", + "start": "2016-11-16T04:16:21.876642682-05:00", + "finish": "2016-11-16T04:16:21.881895914-05:00" + }, + { + "name": "Blueflood FetchMultipleTimeseries", + "start": "2016-11-16T04:16:21.876418022-05:00", + "finish": "2016-11-16T04:16:21.881921474-05:00" + } + ] + } + } + ` +} diff --git a/pkg/tsdb/mqe/types.go b/pkg/tsdb/mqe/types.go index ba50b02ba04..8e077efc0cb 100644 --- a/pkg/tsdb/mqe/types.go +++ b/pkg/tsdb/mqe/types.go @@ -115,6 +115,3 @@ type TokenResponse struct { Success bool Body TokenBody } - -type MQEResponse struct { -} diff --git a/pkg/tsdb/mqe/types_test.go b/pkg/tsdb/mqe/types_test.go index c01b229e72e..2550791c86d 100644 --- a/pkg/tsdb/mqe/types_test.go +++ b/pkg/tsdb/mqe/types_test.go @@ -24,22 +24,12 @@ func TestWildcardExpansion(t *testing.T) { to := now.UnixNano() / int64(time.Millisecond) Convey("Can expanding query", t, func() { - Convey("Without wildcard series", func() { query := &MQEQuery{ Metrics: []MQEMetric{ - MQEMetric{ - Metric: "os.cpu.3.idle", - Alias: "", - }, - MQEMetric{ - Metric: "os.cpu.2.idle", - Alias: "", - }, - MQEMetric{ - Metric: "os.cpu.1.idle", - Alias: "cpu", - }, + MQEMetric{Metric: "os.cpu.3.idle", Alias: ""}, + MQEMetric{Metric: "os.cpu.2.idle", Alias: ""}, + MQEMetric{Metric: "os.cpu.1.idle", Alias: "cpu"}, }, Hosts: []string{"staples-lab-1", "staples-lab-2"}, Apps: []string{"demoapp-1", "demoapp-2"}, @@ -59,10 +49,7 @@ func TestWildcardExpansion(t *testing.T) { Convey("Containg wildcard series", func() { query := &MQEQuery{ Metrics: []MQEMetric{ - MQEMetric{ - Metric: "os.cpu*", - Alias: "cpu on core *", - }, + MQEMetric{Metric: "os.cpu*", Alias: ""}, }, Hosts: []string{"staples-lab-1"}, AddAppToAlias: false, @@ -78,7 +65,6 @@ func TestWildcardExpansion(t *testing.T) { So(expandeQueries[1], ShouldEqual, fmt.Sprintf("`os.cpu.1.idle` where host in ('staples-lab-1') from %v to %v", from, to)) So(expandeQueries[2], ShouldEqual, fmt.Sprintf("`os.cpu.2.idle` where host in ('staples-lab-1') from %v to %v", from, to)) So(expandeQueries[3], ShouldEqual, fmt.Sprintf("`os.cpu.3.idle` where host in ('staples-lab-1') from %v to %v", from, to)) - }) }) } From f1897b7e969b9c75eeda56c8361b9bf9df0f83d9 Mon Sep 17 00:00:00 2001 From: bergquist Date: Wed, 16 Nov 2016 14:42:43 +0100 Subject: [PATCH 10/18] feat(mqe): add response parser --- pkg/tsdb/mqe/mqe.go | 40 +++++++++++-- pkg/tsdb/mqe/response_parser.go | 85 +++++++++++++++++++++++++++- pkg/tsdb/mqe/response_parser_test.go | 28 ++------- 3 files changed, 121 insertions(+), 32 deletions(-) diff --git a/pkg/tsdb/mqe/mqe.go b/pkg/tsdb/mqe/mqe.go index c3f10530416..aeee8c6f5b5 100644 --- a/pkg/tsdb/mqe/mqe.go +++ b/pkg/tsdb/mqe/mqe.go @@ -10,10 +10,14 @@ import ( type MQEExecutor struct { *tsdb.DataSourceInfo + QueryParser *MQEQueryParser } func NewMQEExecutor(dsInfo *tsdb.DataSourceInfo) tsdb.Executor { - return &MQEExecutor{dsInfo} + return &MQEExecutor{ + DataSourceInfo: dsInfo, + QueryParser: &MQEQueryParser{}, + } } var ( @@ -29,16 +33,40 @@ func init() { } func (e *MQEExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, context *tsdb.QueryContext) *tsdb.BatchResult { + result := &tsdb.BatchResult{} + + availableSeries, err := NewTokenClient().GetTokenData(ctx, e.DataSourceInfo) + if err != nil { + return result.WithError(err) + } - availableSeries, _ := NewTokenClient().GetTokenData(ctx, e.DataSourceInfo) glog.Info("available series", availableSeries) - //query, _ := &MQEQueryParser{}.Parse() + var mqeQueries []*MQEQuery + for _, v := range queries { + q, err := e.QueryParser.Parse(v.Model, e.DataSourceInfo) + if err != nil { + return result.WithError(err) + } + mqeQueries = append(mqeQueries, q) + } + var rawQueries []string + for _, v := range mqeQueries { + queries, err := v.Build(availableSeries.Metrics) + if err != nil { + return result.WithError(err) + } + rawQueries = append(rawQueries, queries...) + } - //fetch all available serienames - //expaned parsed model into multiple queries + for _, v := range rawQueries { + glog.Info("Mqe executor", "query", v) + //create request from v + //send request + //parse request + } - return &tsdb.BatchResult{} + return result } diff --git a/pkg/tsdb/mqe/response_parser.go b/pkg/tsdb/mqe/response_parser.go index 69b620a5790..06e9266c714 100644 --- a/pkg/tsdb/mqe/response_parser.go +++ b/pkg/tsdb/mqe/response_parser.go @@ -1,8 +1,16 @@ package mqe import ( + "encoding/json" + "io/ioutil" "net/http" + "time" + null "gopkg.in/guregu/null.v3" + + "fmt" + + "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/tsdb" ) @@ -11,8 +19,81 @@ import ( // add app to alias // regular alias -type MQEResponseParser struct{} +func NewResponseParser() *MQEResponseParser { + return &MQEResponseParser{ + log: log.New("tsdb.mqe"), + } +} + +type MQEResponse struct { + Success bool `json:"success"` + Name string `json:"name"` + Body []MQEResponseSerie `json:"body"` +} + +type ResponseTimeRange struct { + Start int64 `json:"start"` + End int64 `json:"end"` + Resolution time.Duration `json:"Resolution"` +} + +type MQEResponseSerie struct { + Query string `json:"query"` + Name string `json:"name"` + Type string `json:"type"` + Series []MQESerie `json:"series"` + TimeRange ResponseTimeRange `json:"timerange"` +} + +type MQESerie struct { + Values []null.Float `json:"values"` + Tagset map[string]string `json:"tagset"` +} + +type MQEResponseParser struct { + log log.Logger +} func (parser *MQEResponseParser) Parse(res *http.Response) (*tsdb.QueryResult, error) { - return nil, nil + body, err := ioutil.ReadAll(res.Body) + defer res.Body.Close() + if err != nil { + return nil, err + } + + if res.StatusCode/100 != 2 { + parser.log.Error("Request failed", "status code", res.StatusCode, "body", string(body)) + return nil, fmt.Errorf("Returned invalid statuscode") + } + + var data *MQEResponse = &MQEResponse{} + err = json.Unmarshal(body, data) + if err != nil { + parser.log.Info("Failed to unmarshal graphite response", "error", err, "status", res.Status, "body", string(body)) + return nil, err + } + + if !data.Success { + return nil, fmt.Errorf("MQE request failed.") + } + + var series tsdb.TimeSeriesSlice + for _, v := range data.Body { + for _, k := range v.Series { + serie := &tsdb.TimeSeries{ + Name: v.Name, + } + + startTime := time.Unix(v.TimeRange.Start*1000, 0) + for i, l := range k.Values { + timestamp := startTime.Add(time.Duration(int64(v.TimeRange.Resolution) * int64(i))) + serie.Points = append(serie.Points, tsdb.NewTimePoint(l, float64(timestamp.UnixNano()/int64(time.Millisecond)))) + } + + series = append(series, serie) + } + + } + + return &tsdb.QueryResult{Series: series}, nil } diff --git a/pkg/tsdb/mqe/response_parser_test.go b/pkg/tsdb/mqe/response_parser_test.go index eabc437c8a1..85626a07df5 100644 --- a/pkg/tsdb/mqe/response_parser_test.go +++ b/pkg/tsdb/mqe/response_parser_test.go @@ -3,14 +3,11 @@ package mqe import ( "testing" - "time" - "net/http" "strings" "io/ioutil" - "github.com/grafana/grafana/pkg/components/simplejson" . "github.com/smartystreets/goconvey/convey" ) @@ -20,38 +17,21 @@ var ( func TestMQEResponseParser(t *testing.T) { Convey("MQE response parser", t, func() { - parser := &MQEResponseParser{} + parser := NewResponseParser() Convey("Can parse response", func() { response := &http.Response{ StatusCode: 200, Body: ioutil.NopCloser(strings.NewReader(dummieJson)), } - _, err := parser.Parse(response) + res, err := parser.Parse(response) So(err, ShouldBeNil) + So(len(res.Series), ShouldEqual, 2) + So(len(res.Series[0].Points), ShouldEqual, 11) }) }) } -type MQEResponse struct { - Success bool `json:"success"` - Name string `json:"name"` - Body []MQEResponseSerie `json:"body"` -} - -type ResponseTimeRange struct { - Start time.Time `json:"start"` - End time.Time `json:"end"` - Resolution time.Duration `json:"Resolution"` -} - -type MQEResponseSerie struct { - Query string `json:"query"` - Name string `json:"name"` - Type string `json:"type"` - Series []simplejson.Json `json:"series"` -} - func init() { dummieJson = `{ "success": true, From fa8213d8d50b0e14c90ad00c7009b567dbdf21c2 Mon Sep 17 00:00:00 2001 From: bergquist Date: Wed, 16 Nov 2016 15:28:31 +0100 Subject: [PATCH 11/18] feat(mqe): send http requests --- pkg/tsdb/mqe/mqe.go | 58 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 54 insertions(+), 4 deletions(-) diff --git a/pkg/tsdb/mqe/mqe.go b/pkg/tsdb/mqe/mqe.go index aeee8c6f5b5..fac167eaa67 100644 --- a/pkg/tsdb/mqe/mqe.go +++ b/pkg/tsdb/mqe/mqe.go @@ -3,20 +3,28 @@ package mqe import ( "context" "net/http" + "net/url" + "path" + "strings" + "golang.org/x/net/context/ctxhttp" + + "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/tsdb" ) type MQEExecutor struct { *tsdb.DataSourceInfo - QueryParser *MQEQueryParser + QueryParser *MQEQueryParser + ResponseParser *MQEResponseParser } func NewMQEExecutor(dsInfo *tsdb.DataSourceInfo) tsdb.Executor { return &MQEExecutor{ DataSourceInfo: dsInfo, QueryParser: &MQEQueryParser{}, + ResponseParser: &MQEResponseParser{}, } } @@ -61,12 +69,54 @@ func (e *MQEExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, cont rawQueries = append(rawQueries, queries...) } + asdf := &tsdb.QueryResult{} for _, v := range rawQueries { glog.Info("Mqe executor", "query", v) - //create request from v - //send request - //parse request + + req, err := e.createRequest(v) + + resp, err := ctxhttp.Do(ctx, HttpClient, req) + if err != nil { + return result.WithError(err) + } + + series, err := e.ResponseParser.Parse(resp) + if err != nil { + return result.WithError(err) + } + + asdf.Series = append(asdf.Series, series.Series...) } + result.QueryResults["A"] = asdf + return result } + +func (e *MQEExecutor) createRequest(query string) (*http.Request, error) { + u, _ := url.Parse(e.Url) + u.Path = path.Join(u.Path, "query") + + payload := simplejson.New() + payload.Set("query", query) + + jsonPayload, err := payload.MarshalJSON() + if err != nil { + return nil, err + } + + req, err := http.NewRequest(http.MethodPost, u.String(), strings.NewReader(string(jsonPayload))) + if err != nil { + return nil, err + } + + req.Header.Set("User-Agent", "Grafana") + req.Header.Set("Content-Type", "application/json") + + if e.BasicAuth { + req.SetBasicAuth(e.BasicAuthUser, e.BasicAuthPassword) + } + + glog.Debug("Mqe request", "url", req.URL.String()) + return req, nil +} From 9572f086ae7d5a0ac5bc35efd4daeb87788e74bf Mon Sep 17 00:00:00 2001 From: bergquist Date: Wed, 16 Nov 2016 17:03:21 +0100 Subject: [PATCH 12/18] feat(mqe): add timerange to query --- pkg/tsdb/mqe/model_parser.go | 5 ++--- pkg/tsdb/mqe/model_parser_test.go | 11 +++++------ pkg/tsdb/mqe/mqe.go | 9 ++++----- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/pkg/tsdb/mqe/model_parser.go b/pkg/tsdb/mqe/model_parser.go index 26bafb6c055..679ffcfd713 100644 --- a/pkg/tsdb/mqe/model_parser.go +++ b/pkg/tsdb/mqe/model_parser.go @@ -7,9 +7,8 @@ import ( type MQEQueryParser struct{} -func (qp *MQEQueryParser) Parse(model *simplejson.Json, dsInfo *tsdb.DataSourceInfo) (*MQEQuery, error) { - query := &MQEQuery{} - +func (qp *MQEQueryParser) Parse(model *simplejson.Json, dsInfo *tsdb.DataSourceInfo, queryContext *tsdb.QueryContext) (*MQEQuery, error) { + query := &MQEQuery{TimeRange: queryContext.TimeRange} query.AddAppToAlias = model.Get("addAppToAlias").MustBool(false) query.AddHostToAlias = model.Get("addHostToAlias").MustBool(false) query.UseRawQuery = model.Get("rawQuery").MustBool(false) diff --git a/pkg/tsdb/mqe/model_parser_test.go b/pkg/tsdb/mqe/model_parser_test.go index 60b83914a5c..960705978d7 100644 --- a/pkg/tsdb/mqe/model_parser_test.go +++ b/pkg/tsdb/mqe/model_parser_test.go @@ -12,9 +12,8 @@ func TestMQEQueryParser(t *testing.T) { Convey("MQE query parser", t, func() { parser := &MQEQueryParser{} - dsInfo := &tsdb.DataSourceInfo{ - JsonData: simplejson.New(), - } + dsInfo := &tsdb.DataSourceInfo{JsonData: simplejson.New()} + queryContext := &tsdb.QueryContext{} Convey("can parse simple mqe model", func() { json := ` @@ -35,7 +34,7 @@ func TestMQEQueryParser(t *testing.T) { modelJson, err := simplejson.NewJson([]byte(json)) So(err, ShouldBeNil) - query, err := parser.Parse(modelJson, dsInfo) + query, err := parser.Parse(modelJson, dsInfo, queryContext) So(err, ShouldBeNil) So(query.UseRawQuery, ShouldBeFalse) @@ -70,7 +69,7 @@ func TestMQEQueryParser(t *testing.T) { modelJson, err := simplejson.NewJson([]byte(json)) So(err, ShouldBeNil) - query, err := parser.Parse(modelJson, dsInfo) + query, err := parser.Parse(modelJson, dsInfo, queryContext) So(err, ShouldBeNil) So(query.UseRawQuery, ShouldBeFalse) So(query.Apps[0], ShouldEqual, "demoapp") @@ -107,7 +106,7 @@ func TestMQEQueryParser(t *testing.T) { modelJson, err := simplejson.NewJson([]byte(json)) So(err, ShouldBeNil) - query, err := parser.Parse(modelJson, dsInfo) + query, err := parser.Parse(modelJson, dsInfo, queryContext) So(err, ShouldBeNil) So(query.UseRawQuery, ShouldBeTrue) diff --git a/pkg/tsdb/mqe/mqe.go b/pkg/tsdb/mqe/mqe.go index fac167eaa67..75a291bac0b 100644 --- a/pkg/tsdb/mqe/mqe.go +++ b/pkg/tsdb/mqe/mqe.go @@ -35,12 +35,12 @@ var ( func init() { glog = log.New("tsdb.mqe") - tsdb.RegisterExecutor("mqe", NewMQEExecutor) + tsdb.RegisterExecutor("mqe-datasource", NewMQEExecutor) HttpClient = tsdb.GetDefaultClient() } -func (e *MQEExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, context *tsdb.QueryContext) *tsdb.BatchResult { +func (e *MQEExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, queryContext *tsdb.QueryContext) *tsdb.BatchResult { result := &tsdb.BatchResult{} availableSeries, err := NewTokenClient().GetTokenData(ctx, e.DataSourceInfo) @@ -48,11 +48,9 @@ func (e *MQEExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, cont return result.WithError(err) } - glog.Info("available series", availableSeries) - var mqeQueries []*MQEQuery for _, v := range queries { - q, err := e.QueryParser.Parse(v.Model, e.DataSourceInfo) + q, err := e.QueryParser.Parse(v.Model, e.DataSourceInfo, queryContext) if err != nil { return result.WithError(err) } @@ -88,6 +86,7 @@ func (e *MQEExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, cont asdf.Series = append(asdf.Series, series.Series...) } + result.QueryResults = make(map[string]*tsdb.QueryResult) result.QueryResults["A"] = asdf return result From 23387bd39bde627c4b4499597eca2084df8310db Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 17 Nov 2016 08:42:09 +0100 Subject: [PATCH 13/18] feat(mqe): simplify timestamp parsing --- pkg/tsdb/mqe/mqe.go | 6 +++--- pkg/tsdb/mqe/response_parser.go | 14 ++++++-------- pkg/tsdb/mqe/response_parser_test.go | 10 ++++++++-- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/pkg/tsdb/mqe/mqe.go b/pkg/tsdb/mqe/mqe.go index 75a291bac0b..a47f78ca041 100644 --- a/pkg/tsdb/mqe/mqe.go +++ b/pkg/tsdb/mqe/mqe.go @@ -67,7 +67,7 @@ func (e *MQEExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, quer rawQueries = append(rawQueries, queries...) } - asdf := &tsdb.QueryResult{} + queryResult := &tsdb.QueryResult{} for _, v := range rawQueries { glog.Info("Mqe executor", "query", v) @@ -83,11 +83,11 @@ func (e *MQEExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, quer return result.WithError(err) } - asdf.Series = append(asdf.Series, series.Series...) + queryResult.Series = append(queryResult.Series, series.Series...) } result.QueryResults = make(map[string]*tsdb.QueryResult) - result.QueryResults["A"] = asdf + result.QueryResults["A"] = queryResult return result } diff --git a/pkg/tsdb/mqe/response_parser.go b/pkg/tsdb/mqe/response_parser.go index 06e9266c714..e91c2a589e2 100644 --- a/pkg/tsdb/mqe/response_parser.go +++ b/pkg/tsdb/mqe/response_parser.go @@ -4,7 +4,6 @@ import ( "encoding/json" "io/ioutil" "net/http" - "time" null "gopkg.in/guregu/null.v3" @@ -32,9 +31,9 @@ type MQEResponse struct { } type ResponseTimeRange struct { - Start int64 `json:"start"` - End int64 `json:"end"` - Resolution time.Duration `json:"Resolution"` + Start int64 `json:"start"` + End int64 `json:"end"` + Resolution int64 `json:"Resolution"` } type MQEResponseSerie struct { @@ -84,10 +83,9 @@ func (parser *MQEResponseParser) Parse(res *http.Response) (*tsdb.QueryResult, e Name: v.Name, } - startTime := time.Unix(v.TimeRange.Start*1000, 0) - for i, l := range k.Values { - timestamp := startTime.Add(time.Duration(int64(v.TimeRange.Resolution) * int64(i))) - serie.Points = append(serie.Points, tsdb.NewTimePoint(l, float64(timestamp.UnixNano()/int64(time.Millisecond)))) + for i, value := range k.Values { + timestamp := v.TimeRange.Start + int64(i)*v.TimeRange.Resolution + serie.Points = append(serie.Points, tsdb.NewTimePoint(value, float64(timestamp))) } series = append(series, serie) diff --git a/pkg/tsdb/mqe/response_parser_test.go b/pkg/tsdb/mqe/response_parser_test.go index 85626a07df5..39b49f67bc0 100644 --- a/pkg/tsdb/mqe/response_parser_test.go +++ b/pkg/tsdb/mqe/response_parser_test.go @@ -27,7 +27,13 @@ func TestMQEResponseParser(t *testing.T) { res, err := parser.Parse(response) So(err, ShouldBeNil) So(len(res.Series), ShouldEqual, 2) - So(len(res.Series[0].Points), ShouldEqual, 11) + So(len(res.Series[0].Points), ShouldEqual, 14) + + startTime := 1479287280000 + for i := 0; i < 11; i++ { + So(res.Series[0].Points[i][0].Float64, ShouldEqual, i+1) + So(res.Series[0].Points[i][1].Float64, ShouldEqual, startTime+(i*30000)) + } }) }) } @@ -47,7 +53,7 @@ func init() { "app": "demoapp", "host": "staples-lab-1" }, - "values": [1,2,3,4,5,6,7,8,9,10,11] + "values": [1,2,3,4,5,6,7,8,9,10,11, null, null, null] }, { "tagset": { From 0a0f558c4838df32e23400af91186d6e14759a66 Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 17 Nov 2016 08:48:13 +0100 Subject: [PATCH 14/18] style(mqe): improve code --- pkg/tsdb/mqe/mqe.go | 10 ++++++++++ pkg/tsdb/mqe/response_parser.go | 20 ++++++-------------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/pkg/tsdb/mqe/mqe.go b/pkg/tsdb/mqe/mqe.go index a47f78ca041..2a16477c297 100644 --- a/pkg/tsdb/mqe/mqe.go +++ b/pkg/tsdb/mqe/mqe.go @@ -14,6 +14,16 @@ import ( "github.com/grafana/grafana/pkg/tsdb" ) +/* + + TODO: + * response serie names + * response serie names with tags + * response serie names with wildcards + * real caching + +*/ + type MQEExecutor struct { *tsdb.DataSourceInfo QueryParser *MQEQueryParser diff --git a/pkg/tsdb/mqe/response_parser.go b/pkg/tsdb/mqe/response_parser.go index e91c2a589e2..70ae4b94e9c 100644 --- a/pkg/tsdb/mqe/response_parser.go +++ b/pkg/tsdb/mqe/response_parser.go @@ -13,11 +13,6 @@ import ( "github.com/grafana/grafana/pkg/tsdb" ) -// wildcard as alias -// add host to alias -// add app to alias -// regular alias - func NewResponseParser() *MQEResponseParser { return &MQEResponseParser{ log: log.New("tsdb.mqe"), @@ -68,7 +63,7 @@ func (parser *MQEResponseParser) Parse(res *http.Response) (*tsdb.QueryResult, e var data *MQEResponse = &MQEResponse{} err = json.Unmarshal(body, data) if err != nil { - parser.log.Info("Failed to unmarshal graphite response", "error", err, "status", res.Status, "body", string(body)) + parser.log.Info("Failed to unmarshal mqe response", "error", err, "status", res.Status, "body", string(body)) return nil, err } @@ -77,20 +72,17 @@ func (parser *MQEResponseParser) Parse(res *http.Response) (*tsdb.QueryResult, e } var series tsdb.TimeSeriesSlice - for _, v := range data.Body { - for _, k := range v.Series { - serie := &tsdb.TimeSeries{ - Name: v.Name, - } + for _, body := range data.Body { + for _, mqeSerie := range body.Series { + serie := &tsdb.TimeSeries{Name: body.Name} - for i, value := range k.Values { - timestamp := v.TimeRange.Start + int64(i)*v.TimeRange.Resolution + for i, value := range mqeSerie.Values { + timestamp := body.TimeRange.Start + int64(i)*body.TimeRange.Resolution serie.Points = append(serie.Points, tsdb.NewTimePoint(value, float64(timestamp))) } series = append(series, serie) } - } return &tsdb.QueryResult{Series: series}, nil From 1b0005a9e551aa71388385869ca2ab08ac762759 Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 18 Nov 2016 11:40:38 +0100 Subject: [PATCH 15/18] feat(mqe): add support for app and host alias --- pkg/tsdb/mqe/mqe.go | 11 +++-- pkg/tsdb/mqe/response_parser.go | 15 ++++++- pkg/tsdb/mqe/response_parser_test.go | 9 +++- pkg/tsdb/mqe/types.go | 63 ++++++++++++++++------------ pkg/tsdb/mqe/types_test.go | 14 +++---- 5 files changed, 71 insertions(+), 41 deletions(-) diff --git a/pkg/tsdb/mqe/mqe.go b/pkg/tsdb/mqe/mqe.go index 2a16477c297..8c1dd89e2fa 100644 --- a/pkg/tsdb/mqe/mqe.go +++ b/pkg/tsdb/mqe/mqe.go @@ -50,6 +50,11 @@ func init() { HttpClient = tsdb.GetDefaultClient() } +type QueryToSend struct { + RawQuery string + QueryRef *MQEQuery +} + func (e *MQEExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, queryContext *tsdb.QueryContext) *tsdb.BatchResult { result := &tsdb.BatchResult{} @@ -67,7 +72,7 @@ func (e *MQEExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, quer mqeQueries = append(mqeQueries, q) } - var rawQueries []string + var rawQueries []QueryToSend for _, v := range mqeQueries { queries, err := v.Build(availableSeries.Metrics) if err != nil { @@ -81,14 +86,14 @@ func (e *MQEExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, quer for _, v := range rawQueries { glog.Info("Mqe executor", "query", v) - req, err := e.createRequest(v) + req, err := e.createRequest(v.RawQuery) resp, err := ctxhttp.Do(ctx, HttpClient, req) if err != nil { return result.WithError(err) } - series, err := e.ResponseParser.Parse(resp) + series, err := e.ResponseParser.Parse(resp, v.QueryRef) if err != nil { return result.WithError(err) } diff --git a/pkg/tsdb/mqe/response_parser.go b/pkg/tsdb/mqe/response_parser.go index 70ae4b94e9c..a3f96f6c2dc 100644 --- a/pkg/tsdb/mqe/response_parser.go +++ b/pkg/tsdb/mqe/response_parser.go @@ -48,7 +48,7 @@ type MQEResponseParser struct { log log.Logger } -func (parser *MQEResponseParser) Parse(res *http.Response) (*tsdb.QueryResult, error) { +func (parser *MQEResponseParser) Parse(res *http.Response, queryRef *MQEQuery) (*tsdb.QueryResult, error) { body, err := ioutil.ReadAll(res.Body) defer res.Body.Close() if err != nil { @@ -74,7 +74,18 @@ func (parser *MQEResponseParser) Parse(res *http.Response) (*tsdb.QueryResult, e var series tsdb.TimeSeriesSlice for _, body := range data.Body { for _, mqeSerie := range body.Series { - serie := &tsdb.TimeSeries{Name: body.Name} + namePrefix := "" + + for key, value := range mqeSerie.Tagset { + if key == "app" && queryRef.AddAppToAlias { + namePrefix += value + " " + } + if key == "host" && queryRef.AddHostToAlias { + namePrefix += value + " " + } + } + + serie := &tsdb.TimeSeries{Name: namePrefix + body.Name} for i, value := range mqeSerie.Values { timestamp := body.TimeRange.Start + int64(i)*body.TimeRange.Resolution diff --git a/pkg/tsdb/mqe/response_parser_test.go b/pkg/tsdb/mqe/response_parser_test.go index 39b49f67bc0..240f1719e95 100644 --- a/pkg/tsdb/mqe/response_parser_test.go +++ b/pkg/tsdb/mqe/response_parser_test.go @@ -20,15 +20,20 @@ func TestMQEResponseParser(t *testing.T) { parser := NewResponseParser() Convey("Can parse response", func() { + queryRef := &MQEQuery{ + AddAppToAlias: true, + AddHostToAlias: true, + } + response := &http.Response{ StatusCode: 200, Body: ioutil.NopCloser(strings.NewReader(dummieJson)), } - res, err := parser.Parse(response) + res, err := parser.Parse(response, queryRef) So(err, ShouldBeNil) So(len(res.Series), ShouldEqual, 2) So(len(res.Series[0].Points), ShouldEqual, 14) - + So(res.Series[0].Name, ShouldEqual, "demoapp staples-lab-1 os.disk.sda3.weighted_io_time") startTime := 1479287280000 for i := 0; i < 11; i++ { So(res.Series[0].Points[i][0].Float64, ShouldEqual, i+1) diff --git a/pkg/tsdb/mqe/types.go b/pkg/tsdb/mqe/types.go index 8e077efc0cb..d28bc8dd3fb 100644 --- a/pkg/tsdb/mqe/types.go +++ b/pkg/tsdb/mqe/types.go @@ -32,11 +32,27 @@ var ( containsWildcardPattern *regexp.Regexp = regexp.MustCompile(`\*`) ) -func (q *MQEQuery) Build(availableSeries []string) ([]string, error) { - var metrics []MQEMetric +func (q *MQEQuery) Build(availableSeries []string) ([]QueryToSend, error) { + var queriesToSend []QueryToSend + where := q.buildWhereClause() + for _, v := range q.Metrics { if !containsWildcardPattern.Match([]byte(v.Metric)) { - metrics = append(metrics, v) + alias := "" + if v.Alias != "" { + alias = fmt.Sprintf(" {%s}", v.Alias) + } + rawQuery := fmt.Sprintf( + "`%s`%s %s from %v to %v", + v.Metric, + alias, + where, + q.TimeRange.GetFromAsMsEpoch(), + q.TimeRange.GetToAsMsEpoch()) + queriesToSend = append(queriesToSend, QueryToSend{ + RawQuery: rawQuery, + QueryRef: q, + }) continue } @@ -51,34 +67,27 @@ func (q *MQEQuery) Build(availableSeries []string) ([]string, error) { //TODO: this lookup should be cached for _, a := range availableSeries { if mp.Match([]byte(a)) { - metrics = append(metrics, MQEMetric{ - Metric: a, - Alias: v.Alias, + alias := "" + if v.Alias != "" { + alias = fmt.Sprintf(" {%s}", v.Alias) + } + + rawQuery := fmt.Sprintf( + "`%s`%s %s from %v to %v", + a, + alias, + where, + q.TimeRange.GetFromAsMsEpoch(), + q.TimeRange.GetToAsMsEpoch()) + + queriesToSend = append(queriesToSend, QueryToSend{ + RawQuery: rawQuery, + QueryRef: q, }) } } } - - var queries []string - where := q.buildWhereClause() - - for _, metric := range metrics { - alias := "" - if metric.Alias != "" { - alias = fmt.Sprintf(" {%s}", metric.Alias) - } - - queries = append(queries, - fmt.Sprintf( - "`%s`%s %s from %v to %v", - metric.Metric, - alias, - where, - q.TimeRange.GetFromAsMsEpoch(), - q.TimeRange.GetToAsMsEpoch())) - } - - return queries, nil + return queriesToSend, nil } func (q *MQEQuery) buildWhereClause() string { diff --git a/pkg/tsdb/mqe/types_test.go b/pkg/tsdb/mqe/types_test.go index 2550791c86d..37f6c6e5279 100644 --- a/pkg/tsdb/mqe/types_test.go +++ b/pkg/tsdb/mqe/types_test.go @@ -41,9 +41,9 @@ func TestWildcardExpansion(t *testing.T) { expandeQueries, err := query.Build(availableMetrics) So(err, ShouldBeNil) So(len(expandeQueries), ShouldEqual, 3) - So(expandeQueries[0], ShouldEqual, fmt.Sprintf("`os.cpu.3.idle` where app in ('demoapp-1', 'demoapp-2') and host in ('staples-lab-1', 'staples-lab-2') from %v to %v", from, to)) - So(expandeQueries[1], ShouldEqual, fmt.Sprintf("`os.cpu.2.idle` where app in ('demoapp-1', 'demoapp-2') and host in ('staples-lab-1', 'staples-lab-2') from %v to %v", from, to)) - So(expandeQueries[2], ShouldEqual, fmt.Sprintf("`os.cpu.1.idle` {cpu} where app in ('demoapp-1', 'demoapp-2') and host in ('staples-lab-1', 'staples-lab-2') from %v to %v", from, to)) + So(expandeQueries[0].RawQuery, ShouldEqual, fmt.Sprintf("`os.cpu.3.idle` where app in ('demoapp-1', 'demoapp-2') and host in ('staples-lab-1', 'staples-lab-2') from %v to %v", from, to)) + So(expandeQueries[1].RawQuery, ShouldEqual, fmt.Sprintf("`os.cpu.2.idle` where app in ('demoapp-1', 'demoapp-2') and host in ('staples-lab-1', 'staples-lab-2') from %v to %v", from, to)) + So(expandeQueries[2].RawQuery, ShouldEqual, fmt.Sprintf("`os.cpu.1.idle` {cpu} where app in ('demoapp-1', 'demoapp-2') and host in ('staples-lab-1', 'staples-lab-2') from %v to %v", from, to)) }) Convey("Containg wildcard series", func() { @@ -61,10 +61,10 @@ func TestWildcardExpansion(t *testing.T) { So(err, ShouldBeNil) So(len(expandeQueries), ShouldEqual, 4) - So(expandeQueries[0], ShouldEqual, fmt.Sprintf("`os.cpu.all.idle` where host in ('staples-lab-1') from %v to %v", from, to)) - So(expandeQueries[1], ShouldEqual, fmt.Sprintf("`os.cpu.1.idle` where host in ('staples-lab-1') from %v to %v", from, to)) - So(expandeQueries[2], ShouldEqual, fmt.Sprintf("`os.cpu.2.idle` where host in ('staples-lab-1') from %v to %v", from, to)) - So(expandeQueries[3], ShouldEqual, fmt.Sprintf("`os.cpu.3.idle` where host in ('staples-lab-1') from %v to %v", from, to)) + So(expandeQueries[0].RawQuery, ShouldEqual, fmt.Sprintf("`os.cpu.all.idle` where host in ('staples-lab-1') from %v to %v", from, to)) + So(expandeQueries[1].RawQuery, ShouldEqual, fmt.Sprintf("`os.cpu.1.idle` where host in ('staples-lab-1') from %v to %v", from, to)) + So(expandeQueries[2].RawQuery, ShouldEqual, fmt.Sprintf("`os.cpu.2.idle` where host in ('staples-lab-1') from %v to %v", from, to)) + So(expandeQueries[3].RawQuery, ShouldEqual, fmt.Sprintf("`os.cpu.3.idle` where host in ('staples-lab-1') from %v to %v", from, to)) }) }) } From 549c3ab78b11f095f3c8ac5311385a649d81b521 Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 13 Dec 2016 11:42:06 +0100 Subject: [PATCH 16/18] tech(mqe): update todo --- pkg/tsdb/mqe/mqe.go | 4 ---- pkg/tsdb/mqe/response_parser.go | 1 + 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/tsdb/mqe/mqe.go b/pkg/tsdb/mqe/mqe.go index af842b4a19c..3813f84a35a 100644 --- a/pkg/tsdb/mqe/mqe.go +++ b/pkg/tsdb/mqe/mqe.go @@ -16,13 +16,9 @@ import ( ) /* - TODO: - * response serie names - * response serie names with tags * response serie names with wildcards * real caching - */ type MQEExecutor struct { diff --git a/pkg/tsdb/mqe/response_parser.go b/pkg/tsdb/mqe/response_parser.go index a3f96f6c2dc..66013523d2d 100644 --- a/pkg/tsdb/mqe/response_parser.go +++ b/pkg/tsdb/mqe/response_parser.go @@ -76,6 +76,7 @@ func (parser *MQEResponseParser) Parse(res *http.Response, queryRef *MQEQuery) ( for _, mqeSerie := range body.Series { namePrefix := "" + //append predefined tags to seriename for key, value := range mqeSerie.Tagset { if key == "app" && queryRef.AddAppToAlias { namePrefix += value + " " From 69d6316512a022d20b2e0264cea7a3e007bc7747 Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 13 Dec 2016 12:47:57 +0100 Subject: [PATCH 17/18] style(mqe): instance dependencies in ctor func --- pkg/tsdb/mqe/model_parser.go | 4 ++++ pkg/tsdb/mqe/mqe.go | 33 +++++++++++++++---------------- pkg/tsdb/mqe/token_client.go | 20 +++++++++++-------- pkg/tsdb/mqe/token_client_test.go | 6 ++---- 4 files changed, 34 insertions(+), 29 deletions(-) diff --git a/pkg/tsdb/mqe/model_parser.go b/pkg/tsdb/mqe/model_parser.go index e68cb012711..ebd75e9e380 100644 --- a/pkg/tsdb/mqe/model_parser.go +++ b/pkg/tsdb/mqe/model_parser.go @@ -6,6 +6,10 @@ import ( "github.com/grafana/grafana/pkg/tsdb" ) +func NewQueryParser() *MQEQueryParser { + return &MQEQueryParser{} +} + type MQEQueryParser struct{} func (qp *MQEQueryParser) Parse(model *simplejson.Json, dsInfo *models.DataSource, queryContext *tsdb.QueryContext) (*MQEQuery, error) { diff --git a/pkg/tsdb/mqe/mqe.go b/pkg/tsdb/mqe/mqe.go index 3813f84a35a..8806e14f978 100644 --- a/pkg/tsdb/mqe/mqe.go +++ b/pkg/tsdb/mqe/mqe.go @@ -23,9 +23,11 @@ import ( type MQEExecutor struct { *models.DataSource - QueryParser *MQEQueryParser - ResponseParser *MQEResponseParser - HttpClient *http.Client + queryParser *MQEQueryParser + responseParser *MQEResponseParser + httpClient *http.Client + log log.Logger + tokenClient *TokenClient } func NewMQEExecutor(dsInfo *models.DataSource) (tsdb.Executor, error) { @@ -36,18 +38,15 @@ func NewMQEExecutor(dsInfo *models.DataSource) (tsdb.Executor, error) { return &MQEExecutor{ DataSource: dsInfo, - QueryParser: &MQEQueryParser{}, - ResponseParser: &MQEResponseParser{}, - HttpClient: httpclient, + httpClient: httpclient, + log: log.New("tsdb.mqe"), + queryParser: NewQueryParser(), + responseParser: NewResponseParser(), + tokenClient: NewTokenClient(dsInfo), }, nil } -var ( - glog log.Logger -) - func init() { - glog = log.New("tsdb.mqe") tsdb.RegisterExecutor("mqe-datasource", NewMQEExecutor) } @@ -59,14 +58,14 @@ type QueryToSend struct { func (e *MQEExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, queryContext *tsdb.QueryContext) *tsdb.BatchResult { result := &tsdb.BatchResult{} - availableSeries, err := NewTokenClient(e.HttpClient).GetTokenData(ctx, e.DataSource) + availableSeries, err := e.tokenClient.GetTokenData(ctx) if err != nil { return result.WithError(err) } var mqeQueries []*MQEQuery for _, v := range queries { - q, err := e.QueryParser.Parse(v.Model, e.DataSource, queryContext) + q, err := e.queryParser.Parse(v.Model, e.DataSource, queryContext) if err != nil { return result.WithError(err) } @@ -85,16 +84,16 @@ func (e *MQEExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, quer queryResult := &tsdb.QueryResult{} for _, v := range rawQueries { - glog.Info("Mqe executor", "query", v) + e.log.Info("Mqe executor", "query", v) req, err := e.createRequest(v.RawQuery) - resp, err := ctxhttp.Do(ctx, e.HttpClient, req) + resp, err := ctxhttp.Do(ctx, e.httpClient, req) if err != nil { return result.WithError(err) } - series, err := e.ResponseParser.Parse(resp, v.QueryRef) + series, err := e.responseParser.Parse(resp, v.QueryRef) if err != nil { return result.WithError(err) } @@ -132,6 +131,6 @@ func (e *MQEExecutor) createRequest(query string) (*http.Request, error) { req.SetBasicAuth(e.BasicAuthUser, e.BasicAuthPassword) } - glog.Debug("Mqe request", "url", req.URL.String()) + e.log.Debug("Mqe request", "url", req.URL.String()) return req, nil } diff --git a/pkg/tsdb/mqe/token_client.go b/pkg/tsdb/mqe/token_client.go index 488de00bbbf..6832dfc8922 100644 --- a/pkg/tsdb/mqe/token_client.go +++ b/pkg/tsdb/mqe/token_client.go @@ -17,12 +17,16 @@ import ( type TokenClient struct { tlog log.Logger + Datasource *models.DataSource HttpClient *http.Client } -func NewTokenClient(httpClient *http.Client) *TokenClient { +func NewTokenClient(datasource *models.DataSource) *TokenClient { + httpClient, _ := datasource.GetHttpClient() + return &TokenClient{ tlog: log.New("tsdb.mqe.tokenclient"), + Datasource: datasource, HttpClient: httpClient, } } @@ -30,22 +34,22 @@ func NewTokenClient(httpClient *http.Client) *TokenClient { var cache map[int64]*TokenBody = map[int64]*TokenBody{} //Replace this stupid cache with internal cache from grafana master before merging -func (client *TokenClient) GetTokenData(ctx context.Context, datasource *models.DataSource) (*TokenBody, error) { - _, excist := cache[datasource.Id] +func (client *TokenClient) GetTokenData(ctx context.Context) (*TokenBody, error) { + _, excist := cache[client.Datasource.Id] if !excist { - b, err := client.RequestTokenData(ctx, datasource) + b, err := client.RequestTokenData(ctx) if err != nil { return nil, err } - cache[datasource.Id] = b + cache[client.Datasource.Id] = b } - return cache[datasource.Id], nil + return cache[client.Datasource.Id], nil } -func (client *TokenClient) RequestTokenData(ctx context.Context, datasource *models.DataSource) (*TokenBody, error) { - u, _ := url.Parse(datasource.Url) +func (client *TokenClient) RequestTokenData(ctx context.Context) (*TokenBody, error) { + u, _ := url.Parse(client.Datasource.Url) u.Path = path.Join(u.Path, "token") req, err := http.NewRequest(http.MethodGet, u.String(), nil) diff --git a/pkg/tsdb/mqe/token_client_test.go b/pkg/tsdb/mqe/token_client_test.go index a3012e46a1a..f940f798b36 100644 --- a/pkg/tsdb/mqe/token_client_test.go +++ b/pkg/tsdb/mqe/token_client_test.go @@ -4,8 +4,6 @@ import ( "context" "testing" - "net/http" - "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/models" . "github.com/smartystreets/goconvey/convey" @@ -18,9 +16,9 @@ func TestTokenClient(t *testing.T) { Url: "", } - client := NewTokenClient(http.DefaultClient) + client := NewTokenClient(dsInfo) - body, err := client.RequestTokenData(context.TODO(), dsInfo) + body, err := client.RequestTokenData(context.TODO()) So(err, ShouldBeNil) //So(len(body.Functions), ShouldBeGreaterThan, 1) From 362162d6fa0234a8b17d3ff901806f3aabc954ce Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 13 Dec 2016 13:47:37 +0100 Subject: [PATCH 18/18] feat(mqe): add token cache --- pkg/tsdb/mqe/model_parser.go | 14 +- pkg/tsdb/mqe/model_parser_test.go | 2 +- pkg/tsdb/mqe/mqe.go | 26 +- pkg/tsdb/mqe/response_parser.go | 12 +- pkg/tsdb/mqe/response_parser_test.go | 2 +- pkg/tsdb/mqe/token_client.go | 45 +- pkg/tsdb/mqe/types.go | 10 +- pkg/tsdb/mqe/types_test.go | 16 +- .../patrickmn/go-cache/CONTRIBUTORS | 8 + vendor/github.com/patrickmn/go-cache/LICENSE | 19 + .../github.com/patrickmn/go-cache/README.md | 107 ++ vendor/github.com/patrickmn/go-cache/cache.go | 1131 +++++++++++++++++ .../github.com/patrickmn/go-cache/sharded.go | 192 +++ vendor/vendor.json | 6 + 14 files changed, 1537 insertions(+), 53 deletions(-) create mode 100644 vendor/github.com/patrickmn/go-cache/CONTRIBUTORS create mode 100644 vendor/github.com/patrickmn/go-cache/LICENSE create mode 100644 vendor/github.com/patrickmn/go-cache/README.md create mode 100644 vendor/github.com/patrickmn/go-cache/cache.go create mode 100644 vendor/github.com/patrickmn/go-cache/sharded.go diff --git a/pkg/tsdb/mqe/model_parser.go b/pkg/tsdb/mqe/model_parser.go index ebd75e9e380..f69177ab12c 100644 --- a/pkg/tsdb/mqe/model_parser.go +++ b/pkg/tsdb/mqe/model_parser.go @@ -6,14 +6,14 @@ import ( "github.com/grafana/grafana/pkg/tsdb" ) -func NewQueryParser() *MQEQueryParser { - return &MQEQueryParser{} +func NewQueryParser() *QueryParser { + return &QueryParser{} } -type MQEQueryParser struct{} +type QueryParser struct{} -func (qp *MQEQueryParser) Parse(model *simplejson.Json, dsInfo *models.DataSource, queryContext *tsdb.QueryContext) (*MQEQuery, error) { - query := &MQEQuery{TimeRange: queryContext.TimeRange} +func (qp *QueryParser) Parse(model *simplejson.Json, dsInfo *models.DataSource, queryContext *tsdb.QueryContext) (*Query, error) { + query := &Query{TimeRange: queryContext.TimeRange} query.AddAppToAlias = model.Get("addAppToAlias").MustBool(false) query.AddHostToAlias = model.Get("addHostToAlias").MustBool(false) query.UseRawQuery = model.Get("rawQuery").MustBool(false) @@ -22,11 +22,11 @@ func (qp *MQEQueryParser) Parse(model *simplejson.Json, dsInfo *models.DataSourc query.Apps = model.Get("apps").MustStringArray([]string{}) query.Hosts = model.Get("hosts").MustStringArray([]string{}) - var metrics []MQEMetric + var metrics []Metric var err error for _, metricsObj := range model.Get("metrics").MustArray() { metricJson := simplejson.NewFromAny(metricsObj) - var m MQEMetric + var m Metric m.Alias = metricJson.Get("alias").MustString("") m.Metric, err = metricJson.Get("metric").String() diff --git a/pkg/tsdb/mqe/model_parser_test.go b/pkg/tsdb/mqe/model_parser_test.go index 20a60e1e90e..432df9e6041 100644 --- a/pkg/tsdb/mqe/model_parser_test.go +++ b/pkg/tsdb/mqe/model_parser_test.go @@ -11,7 +11,7 @@ import ( func TestMQEQueryParser(t *testing.T) { Convey("MQE query parser", t, func() { - parser := &MQEQueryParser{} + parser := &QueryParser{} dsInfo := &models.DataSource{JsonData: simplejson.New()} queryContext := &tsdb.QueryContext{} diff --git a/pkg/tsdb/mqe/mqe.go b/pkg/tsdb/mqe/mqe.go index 8806e14f978..a78bd24a707 100644 --- a/pkg/tsdb/mqe/mqe.go +++ b/pkg/tsdb/mqe/mqe.go @@ -12,19 +12,20 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/tsdb" ) /* TODO: - * response serie names with wildcards - * real caching + * performance. outgoing requests in pararell. + * frontend plugin. targetContainsTemplates */ type MQEExecutor struct { *models.DataSource - queryParser *MQEQueryParser - responseParser *MQEResponseParser + queryParser *QueryParser + responseParser *ResponseParser httpClient *http.Client log log.Logger tokenClient *TokenClient @@ -52,7 +53,7 @@ func init() { type QueryToSend struct { RawQuery string - QueryRef *MQEQuery + QueryRef *Query } func (e *MQEExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, queryContext *tsdb.QueryContext) *tsdb.BatchResult { @@ -63,7 +64,7 @@ func (e *MQEExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, quer return result.WithError(err) } - var mqeQueries []*MQEQuery + var mqeQueries []*Query for _, v := range queries { q, err := e.queryParser.Parse(v.Model, e.DataSource, queryContext) if err != nil { @@ -82,9 +83,13 @@ func (e *MQEExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, quer rawQueries = append(rawQueries, queries...) } + e.log.Debug("Sending request", "url", e.DataSource.Url) + queryResult := &tsdb.QueryResult{} for _, v := range rawQueries { - e.log.Info("Mqe executor", "query", v) + if setting.Env == setting.DEV { + e.log.Debug("Executing", "query", v) + } req, err := e.createRequest(v.RawQuery) @@ -108,7 +113,11 @@ func (e *MQEExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, quer } func (e *MQEExecutor) createRequest(query string) (*http.Request, error) { - u, _ := url.Parse(e.Url) + u, err := url.Parse(e.Url) + if err != nil { + return nil, err + } + u.Path = path.Join(u.Path, "query") payload := simplejson.New() @@ -131,6 +140,5 @@ func (e *MQEExecutor) createRequest(query string) (*http.Request, error) { req.SetBasicAuth(e.BasicAuthUser, e.BasicAuthPassword) } - e.log.Debug("Mqe request", "url", req.URL.String()) return req, nil } diff --git a/pkg/tsdb/mqe/response_parser.go b/pkg/tsdb/mqe/response_parser.go index 66013523d2d..88b5c6ce343 100644 --- a/pkg/tsdb/mqe/response_parser.go +++ b/pkg/tsdb/mqe/response_parser.go @@ -13,8 +13,8 @@ import ( "github.com/grafana/grafana/pkg/tsdb" ) -func NewResponseParser() *MQEResponseParser { - return &MQEResponseParser{ +func NewResponseParser() *ResponseParser { + return &ResponseParser{ log: log.New("tsdb.mqe"), } } @@ -44,11 +44,11 @@ type MQESerie struct { Tagset map[string]string `json:"tagset"` } -type MQEResponseParser struct { +type ResponseParser struct { log log.Logger } -func (parser *MQEResponseParser) Parse(res *http.Response, queryRef *MQEQuery) (*tsdb.QueryResult, error) { +func (parser *ResponseParser) Parse(res *http.Response, queryRef *Query) (*tsdb.QueryResult, error) { body, err := ioutil.ReadAll(res.Body) defer res.Body.Close() if err != nil { @@ -63,12 +63,12 @@ func (parser *MQEResponseParser) Parse(res *http.Response, queryRef *MQEQuery) ( var data *MQEResponse = &MQEResponse{} err = json.Unmarshal(body, data) if err != nil { - parser.log.Info("Failed to unmarshal mqe response", "error", err, "status", res.Status, "body", string(body)) + parser.log.Info("Failed to unmarshal response", "error", err, "status", res.Status, "body", string(body)) return nil, err } if !data.Success { - return nil, fmt.Errorf("MQE request failed.") + return nil, fmt.Errorf("Request failed.") } var series tsdb.TimeSeriesSlice diff --git a/pkg/tsdb/mqe/response_parser_test.go b/pkg/tsdb/mqe/response_parser_test.go index 240f1719e95..cda7a00bd3c 100644 --- a/pkg/tsdb/mqe/response_parser_test.go +++ b/pkg/tsdb/mqe/response_parser_test.go @@ -20,7 +20,7 @@ func TestMQEResponseParser(t *testing.T) { parser := NewResponseParser() Convey("Can parse response", func() { - queryRef := &MQEQuery{ + queryRef := &Query{ AddAppToAlias: true, AddHostToAlias: true, } diff --git a/pkg/tsdb/mqe/token_client.go b/pkg/tsdb/mqe/token_client.go index 6832dfc8922..df136738ab6 100644 --- a/pkg/tsdb/mqe/token_client.go +++ b/pkg/tsdb/mqe/token_client.go @@ -8,15 +8,25 @@ import ( "net/http" "net/url" "path" + "time" "golang.org/x/net/context/ctxhttp" + "strconv" + "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/models" + "github.com/patrickmn/go-cache" ) +var tokenCache *cache.Cache + +func init() { + tokenCache = cache.New(5*time.Minute, 30*time.Second) +} + type TokenClient struct { - tlog log.Logger + log log.Logger Datasource *models.DataSource HttpClient *http.Client } @@ -25,27 +35,30 @@ func NewTokenClient(datasource *models.DataSource) *TokenClient { httpClient, _ := datasource.GetHttpClient() return &TokenClient{ - tlog: log.New("tsdb.mqe.tokenclient"), + log: log.New("tsdb.mqe.tokenclient"), Datasource: datasource, HttpClient: httpClient, } } -var cache map[int64]*TokenBody = map[int64]*TokenBody{} - -//Replace this stupid cache with internal cache from grafana master before merging func (client *TokenClient) GetTokenData(ctx context.Context) (*TokenBody, error) { - _, excist := cache[client.Datasource.Id] - if !excist { - b, err := client.RequestTokenData(ctx) - if err != nil { - return nil, err - } + key := strconv.FormatInt(client.Datasource.Id, 10) - cache[client.Datasource.Id] = b + item, found := tokenCache.Get(key) + if found { + if result, ok := item.(*TokenBody); ok { + return result, nil + } } - return cache[client.Datasource.Id], nil + b, err := client.RequestTokenData(ctx) + if err != nil { + return nil, err + } + + tokenCache.Set(key, b, cache.DefaultExpiration) + + return b, nil } func (client *TokenClient) RequestTokenData(ctx context.Context) (*TokenBody, error) { @@ -54,7 +67,7 @@ func (client *TokenClient) RequestTokenData(ctx context.Context) (*TokenBody, er req, err := http.NewRequest(http.MethodGet, u.String(), nil) if err != nil { - client.tlog.Info("Failed to create request", "error", err) + client.log.Info("Failed to create request", "error", err) } res, err := ctxhttp.Do(ctx, client.HttpClient, req) @@ -69,14 +82,14 @@ func (client *TokenClient) RequestTokenData(ctx context.Context) (*TokenBody, er } if res.StatusCode/100 != 2 { - client.tlog.Info("Request failed", "status", res.Status, "body", string(body)) + client.log.Info("Request failed", "status", res.Status, "body", string(body)) return nil, fmt.Errorf("Request failed status: %v", res.Status) } var result *TokenResponse err = json.Unmarshal(body, &result) if err != nil { - client.tlog.Info("Failed to unmarshal graphite response", "error", err, "status", res.Status, "body", string(body)) + client.log.Info("Failed to unmarshal response", "error", err, "status", res.Status, "body", string(body)) return nil, err } diff --git a/pkg/tsdb/mqe/types.go b/pkg/tsdb/mqe/types.go index d28bc8dd3fb..558ee207b34 100644 --- a/pkg/tsdb/mqe/types.go +++ b/pkg/tsdb/mqe/types.go @@ -11,13 +11,13 @@ import ( "github.com/grafana/grafana/pkg/tsdb" ) -type MQEMetric struct { +type Metric struct { Metric string Alias string } -type MQEQuery struct { - Metrics []MQEMetric +type Query struct { + Metrics []Metric Hosts []string Apps []string AddAppToAlias bool @@ -32,7 +32,7 @@ var ( containsWildcardPattern *regexp.Regexp = regexp.MustCompile(`\*`) ) -func (q *MQEQuery) Build(availableSeries []string) ([]QueryToSend, error) { +func (q *Query) Build(availableSeries []string) ([]QueryToSend, error) { var queriesToSend []QueryToSend where := q.buildWhereClause() @@ -90,7 +90,7 @@ func (q *MQEQuery) Build(availableSeries []string) ([]QueryToSend, error) { return queriesToSend, nil } -func (q *MQEQuery) buildWhereClause() string { +func (q *Query) buildWhereClause() string { hasApps := len(q.Apps) > 0 hasHosts := len(q.Hosts) > 0 diff --git a/pkg/tsdb/mqe/types_test.go b/pkg/tsdb/mqe/types_test.go index 37f6c6e5279..0243e7d84ff 100644 --- a/pkg/tsdb/mqe/types_test.go +++ b/pkg/tsdb/mqe/types_test.go @@ -25,11 +25,11 @@ func TestWildcardExpansion(t *testing.T) { Convey("Can expanding query", t, func() { Convey("Without wildcard series", func() { - query := &MQEQuery{ - Metrics: []MQEMetric{ - MQEMetric{Metric: "os.cpu.3.idle", Alias: ""}, - MQEMetric{Metric: "os.cpu.2.idle", Alias: ""}, - MQEMetric{Metric: "os.cpu.1.idle", Alias: "cpu"}, + query := &Query{ + Metrics: []Metric{ + Metric{Metric: "os.cpu.3.idle", Alias: ""}, + Metric{Metric: "os.cpu.2.idle", Alias: ""}, + Metric{Metric: "os.cpu.1.idle", Alias: "cpu"}, }, Hosts: []string{"staples-lab-1", "staples-lab-2"}, Apps: []string{"demoapp-1", "demoapp-2"}, @@ -47,9 +47,9 @@ func TestWildcardExpansion(t *testing.T) { }) Convey("Containg wildcard series", func() { - query := &MQEQuery{ - Metrics: []MQEMetric{ - MQEMetric{Metric: "os.cpu*", Alias: ""}, + query := &Query{ + Metrics: []Metric{ + Metric{Metric: "os.cpu*", Alias: ""}, }, Hosts: []string{"staples-lab-1"}, AddAppToAlias: false, diff --git a/vendor/github.com/patrickmn/go-cache/CONTRIBUTORS b/vendor/github.com/patrickmn/go-cache/CONTRIBUTORS new file mode 100644 index 00000000000..8a4da4ed294 --- /dev/null +++ b/vendor/github.com/patrickmn/go-cache/CONTRIBUTORS @@ -0,0 +1,8 @@ +This is a list of people who have contributed code to go-cache. They, or their +employers, are the copyright holders of the contributed code. Contributed code +is subject to the license restrictions listed in LICENSE (as they were when the +code was contributed.) + +Dustin Sallings +Jason Mooberry +Sergey Shepelev diff --git a/vendor/github.com/patrickmn/go-cache/LICENSE b/vendor/github.com/patrickmn/go-cache/LICENSE new file mode 100644 index 00000000000..f9fe27156a5 --- /dev/null +++ b/vendor/github.com/patrickmn/go-cache/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2012-2016 Patrick Mylund Nielsen and the go-cache contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/patrickmn/go-cache/README.md b/vendor/github.com/patrickmn/go-cache/README.md new file mode 100644 index 00000000000..168ff7ba5fc --- /dev/null +++ b/vendor/github.com/patrickmn/go-cache/README.md @@ -0,0 +1,107 @@ +# go-cache + +go-cache is an in-memory key:value store/cache similar to memcached that is +suitable for applications running on a single machine. Its major advantage is +that, being essentially a thread-safe `map[string]interface{}` with expiration +times, it doesn't need to serialize or transmit its contents over the network. + +Any object can be stored, for a given duration or forever, and the cache can be +safely used by multiple goroutines. + +Although go-cache isn't meant to be used as a persistent datastore, the entire +cache can be saved to and loaded from a file (using `c.Items()` to retrieve the +items map to serialize, and `NewFrom()` to create a cache from a deserialized +one) to recover from downtime quickly. (See the docs for `NewFrom()` for caveats.) + +### Installation + +`go get github.com/patrickmn/go-cache` + +### Usage + +```go + import ( + "fmt" + "github.com/patrickmn/go-cache" + "time" + ) + + func main() { + + // Create a cache with a default expiration time of 5 minutes, and which + // purges expired items every 30 seconds + c := cache.New(5*time.Minute, 30*time.Second) + + // Set the value of the key "foo" to "bar", with the default expiration time + c.Set("foo", "bar", cache.DefaultExpiration) + + // Set the value of the key "baz" to 42, with no expiration time + // (the item won't be removed until it is re-set, or removed using + // c.Delete("baz") + c.Set("baz", 42, cache.NoExpiration) + + // Get the string associated with the key "foo" from the cache + foo, found := c.Get("foo") + if found { + fmt.Println(foo) + } + + // Since Go is statically typed, and cache values can be anything, type + // assertion is needed when values are being passed to functions that don't + // take arbitrary types, (i.e. interface{}). The simplest way to do this for + // values which will only be used once--e.g. for passing to another + // function--is: + foo, found := c.Get("foo") + if found { + MyFunction(foo.(string)) + } + + // This gets tedious if the value is used several times in the same function. + // You might do either of the following instead: + if x, found := c.Get("foo"); found { + foo := x.(string) + // ... + } + // or + var foo string + if x, found := c.Get("foo"); found { + foo = x.(string) + } + // ... + // foo can then be passed around freely as a string + + // Want performance? Store pointers! + c.Set("foo", &MyStruct, cache.DefaultExpiration) + if x, found := c.Get("foo"); found { + foo := x.(*MyStruct) + // ... + } + + // If you store a reference type like a pointer, slice, map or channel, you + // do not need to run Set if you modify the underlying data. The cached + // reference points to the same memory, so if you modify a struct whose + // pointer you've stored in the cache, retrieving that pointer with Get will + // point you to the same data: + foo := &MyStruct{Num: 1} + c.Set("foo", foo, cache.DefaultExpiration) + // ... + x, _ := c.Get("foo") + foo := x.(*MyStruct) + fmt.Println(foo.Num) + // ... + foo.Num++ + // ... + x, _ := c.Get("foo") + foo := x.(*MyStruct) + foo.Println(foo.Num) + + // will print: + // 1 + // 2 + + } +``` + +### Reference + +`godoc` or [http://godoc.org/github.com/patrickmn/go-cache](http://godoc.org/github.com/patrickmn/go-cache) diff --git a/vendor/github.com/patrickmn/go-cache/cache.go b/vendor/github.com/patrickmn/go-cache/cache.go new file mode 100644 index 00000000000..70e4dad337a --- /dev/null +++ b/vendor/github.com/patrickmn/go-cache/cache.go @@ -0,0 +1,1131 @@ +package cache + +import ( + "encoding/gob" + "fmt" + "io" + "os" + "runtime" + "sync" + "time" +) + +type Item struct { + Object interface{} + Expiration int64 +} + +// Returns true if the item has expired. +func (item Item) Expired() bool { + if item.Expiration == 0 { + return false + } + return time.Now().UnixNano() > item.Expiration +} + +const ( + // For use with functions that take an expiration time. + NoExpiration time.Duration = -1 + // For use with functions that take an expiration time. Equivalent to + // passing in the same expiration duration as was given to New() or + // NewFrom() when the cache was created (e.g. 5 minutes.) + DefaultExpiration time.Duration = 0 +) + +type Cache struct { + *cache + // If this is confusing, see the comment at the bottom of New() +} + +type cache struct { + defaultExpiration time.Duration + items map[string]Item + mu sync.RWMutex + onEvicted func(string, interface{}) + janitor *janitor +} + +// Add an item to the cache, replacing any existing item. If the duration is 0 +// (DefaultExpiration), the cache's default expiration time is used. If it is -1 +// (NoExpiration), the item never expires. +func (c *cache) Set(k string, x interface{}, d time.Duration) { + // "Inlining" of set + var e int64 + if d == DefaultExpiration { + d = c.defaultExpiration + } + if d > 0 { + e = time.Now().Add(d).UnixNano() + } + c.mu.Lock() + c.items[k] = Item{ + Object: x, + Expiration: e, + } + // TODO: Calls to mu.Unlock are currently not deferred because defer + // adds ~200 ns (as of go1.) + c.mu.Unlock() +} + +func (c *cache) set(k string, x interface{}, d time.Duration) { + var e int64 + if d == DefaultExpiration { + d = c.defaultExpiration + } + if d > 0 { + e = time.Now().Add(d).UnixNano() + } + c.items[k] = Item{ + Object: x, + Expiration: e, + } +} + +// Add an item to the cache, replacing any existing item, using the default +// expiration. +func (c *cache) SetDefault(k string, x interface{}) { + c.Set(k, x, DefaultExpiration) +} + +// Add an item to the cache only if an item doesn't already exist for the given +// key, or if the existing item has expired. Returns an error otherwise. +func (c *cache) Add(k string, x interface{}, d time.Duration) error { + c.mu.Lock() + _, found := c.get(k) + if found { + c.mu.Unlock() + return fmt.Errorf("Item %s already exists", k) + } + c.set(k, x, d) + c.mu.Unlock() + return nil +} + +// Set a new value for the cache key only if it already exists, and the existing +// item hasn't expired. Returns an error otherwise. +func (c *cache) Replace(k string, x interface{}, d time.Duration) error { + c.mu.Lock() + _, found := c.get(k) + if !found { + c.mu.Unlock() + return fmt.Errorf("Item %s doesn't exist", k) + } + c.set(k, x, d) + c.mu.Unlock() + return nil +} + +// Get an item from the cache. Returns the item or nil, and a bool indicating +// whether the key was found. +func (c *cache) Get(k string) (interface{}, bool) { + c.mu.RLock() + // "Inlining" of get and Expired + item, found := c.items[k] + if !found { + c.mu.RUnlock() + return nil, false + } + if item.Expiration > 0 { + if time.Now().UnixNano() > item.Expiration { + c.mu.RUnlock() + return nil, false + } + } + c.mu.RUnlock() + return item.Object, true +} + +func (c *cache) get(k string) (interface{}, bool) { + item, found := c.items[k] + if !found { + return nil, false + } + // "Inlining" of Expired + if item.Expiration > 0 { + if time.Now().UnixNano() > item.Expiration { + return nil, false + } + } + return item.Object, true +} + +// Increment an item of type int, int8, int16, int32, int64, uintptr, uint, +// uint8, uint32, or uint64, float32 or float64 by n. Returns an error if the +// item's value is not an integer, if it was not found, or if it is not +// possible to increment it by n. To retrieve the incremented value, use one +// of the specialized methods, e.g. IncrementInt64. +func (c *cache) Increment(k string, n int64) error { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return fmt.Errorf("Item %s not found", k) + } + switch v.Object.(type) { + case int: + v.Object = v.Object.(int) + int(n) + case int8: + v.Object = v.Object.(int8) + int8(n) + case int16: + v.Object = v.Object.(int16) + int16(n) + case int32: + v.Object = v.Object.(int32) + int32(n) + case int64: + v.Object = v.Object.(int64) + n + case uint: + v.Object = v.Object.(uint) + uint(n) + case uintptr: + v.Object = v.Object.(uintptr) + uintptr(n) + case uint8: + v.Object = v.Object.(uint8) + uint8(n) + case uint16: + v.Object = v.Object.(uint16) + uint16(n) + case uint32: + v.Object = v.Object.(uint32) + uint32(n) + case uint64: + v.Object = v.Object.(uint64) + uint64(n) + case float32: + v.Object = v.Object.(float32) + float32(n) + case float64: + v.Object = v.Object.(float64) + float64(n) + default: + c.mu.Unlock() + return fmt.Errorf("The value for %s is not an integer", k) + } + c.items[k] = v + c.mu.Unlock() + return nil +} + +// Increment an item of type float32 or float64 by n. Returns an error if the +// item's value is not floating point, if it was not found, or if it is not +// possible to increment it by n. Pass a negative number to decrement the +// value. To retrieve the incremented value, use one of the specialized methods, +// e.g. IncrementFloat64. +func (c *cache) IncrementFloat(k string, n float64) error { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return fmt.Errorf("Item %s not found", k) + } + switch v.Object.(type) { + case float32: + v.Object = v.Object.(float32) + float32(n) + case float64: + v.Object = v.Object.(float64) + n + default: + c.mu.Unlock() + return fmt.Errorf("The value for %s does not have type float32 or float64", k) + } + c.items[k] = v + c.mu.Unlock() + return nil +} + +// Increment an item of type int by n. Returns an error if the item's value is +// not an int, or if it was not found. If there is no error, the incremented +// value is returned. +func (c *cache) IncrementInt(k string, n int) (int, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(int) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an int", k) + } + nv := rv + n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Increment an item of type int8 by n. Returns an error if the item's value is +// not an int8, or if it was not found. If there is no error, the incremented +// value is returned. +func (c *cache) IncrementInt8(k string, n int8) (int8, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(int8) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an int8", k) + } + nv := rv + n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Increment an item of type int16 by n. Returns an error if the item's value is +// not an int16, or if it was not found. If there is no error, the incremented +// value is returned. +func (c *cache) IncrementInt16(k string, n int16) (int16, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(int16) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an int16", k) + } + nv := rv + n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Increment an item of type int32 by n. Returns an error if the item's value is +// not an int32, or if it was not found. If there is no error, the incremented +// value is returned. +func (c *cache) IncrementInt32(k string, n int32) (int32, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(int32) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an int32", k) + } + nv := rv + n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Increment an item of type int64 by n. Returns an error if the item's value is +// not an int64, or if it was not found. If there is no error, the incremented +// value is returned. +func (c *cache) IncrementInt64(k string, n int64) (int64, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(int64) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an int64", k) + } + nv := rv + n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Increment an item of type uint by n. Returns an error if the item's value is +// not an uint, or if it was not found. If there is no error, the incremented +// value is returned. +func (c *cache) IncrementUint(k string, n uint) (uint, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(uint) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an uint", k) + } + nv := rv + n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Increment an item of type uintptr by n. Returns an error if the item's value +// is not an uintptr, or if it was not found. If there is no error, the +// incremented value is returned. +func (c *cache) IncrementUintptr(k string, n uintptr) (uintptr, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(uintptr) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an uintptr", k) + } + nv := rv + n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Increment an item of type uint8 by n. Returns an error if the item's value +// is not an uint8, or if it was not found. If there is no error, the +// incremented value is returned. +func (c *cache) IncrementUint8(k string, n uint8) (uint8, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(uint8) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an uint8", k) + } + nv := rv + n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Increment an item of type uint16 by n. Returns an error if the item's value +// is not an uint16, or if it was not found. If there is no error, the +// incremented value is returned. +func (c *cache) IncrementUint16(k string, n uint16) (uint16, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(uint16) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an uint16", k) + } + nv := rv + n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Increment an item of type uint32 by n. Returns an error if the item's value +// is not an uint32, or if it was not found. If there is no error, the +// incremented value is returned. +func (c *cache) IncrementUint32(k string, n uint32) (uint32, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(uint32) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an uint32", k) + } + nv := rv + n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Increment an item of type uint64 by n. Returns an error if the item's value +// is not an uint64, or if it was not found. If there is no error, the +// incremented value is returned. +func (c *cache) IncrementUint64(k string, n uint64) (uint64, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(uint64) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an uint64", k) + } + nv := rv + n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Increment an item of type float32 by n. Returns an error if the item's value +// is not an float32, or if it was not found. If there is no error, the +// incremented value is returned. +func (c *cache) IncrementFloat32(k string, n float32) (float32, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(float32) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an float32", k) + } + nv := rv + n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Increment an item of type float64 by n. Returns an error if the item's value +// is not an float64, or if it was not found. If there is no error, the +// incremented value is returned. +func (c *cache) IncrementFloat64(k string, n float64) (float64, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(float64) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an float64", k) + } + nv := rv + n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Decrement an item of type int, int8, int16, int32, int64, uintptr, uint, +// uint8, uint32, or uint64, float32 or float64 by n. Returns an error if the +// item's value is not an integer, if it was not found, or if it is not +// possible to decrement it by n. To retrieve the decremented value, use one +// of the specialized methods, e.g. DecrementInt64. +func (c *cache) Decrement(k string, n int64) error { + // TODO: Implement Increment and Decrement more cleanly. + // (Cannot do Increment(k, n*-1) for uints.) + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return fmt.Errorf("Item not found") + } + switch v.Object.(type) { + case int: + v.Object = v.Object.(int) - int(n) + case int8: + v.Object = v.Object.(int8) - int8(n) + case int16: + v.Object = v.Object.(int16) - int16(n) + case int32: + v.Object = v.Object.(int32) - int32(n) + case int64: + v.Object = v.Object.(int64) - n + case uint: + v.Object = v.Object.(uint) - uint(n) + case uintptr: + v.Object = v.Object.(uintptr) - uintptr(n) + case uint8: + v.Object = v.Object.(uint8) - uint8(n) + case uint16: + v.Object = v.Object.(uint16) - uint16(n) + case uint32: + v.Object = v.Object.(uint32) - uint32(n) + case uint64: + v.Object = v.Object.(uint64) - uint64(n) + case float32: + v.Object = v.Object.(float32) - float32(n) + case float64: + v.Object = v.Object.(float64) - float64(n) + default: + c.mu.Unlock() + return fmt.Errorf("The value for %s is not an integer", k) + } + c.items[k] = v + c.mu.Unlock() + return nil +} + +// Decrement an item of type float32 or float64 by n. Returns an error if the +// item's value is not floating point, if it was not found, or if it is not +// possible to decrement it by n. Pass a negative number to decrement the +// value. To retrieve the decremented value, use one of the specialized methods, +// e.g. DecrementFloat64. +func (c *cache) DecrementFloat(k string, n float64) error { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return fmt.Errorf("Item %s not found", k) + } + switch v.Object.(type) { + case float32: + v.Object = v.Object.(float32) - float32(n) + case float64: + v.Object = v.Object.(float64) - n + default: + c.mu.Unlock() + return fmt.Errorf("The value for %s does not have type float32 or float64", k) + } + c.items[k] = v + c.mu.Unlock() + return nil +} + +// Decrement an item of type int by n. Returns an error if the item's value is +// not an int, or if it was not found. If there is no error, the decremented +// value is returned. +func (c *cache) DecrementInt(k string, n int) (int, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(int) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an int", k) + } + nv := rv - n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Decrement an item of type int8 by n. Returns an error if the item's value is +// not an int8, or if it was not found. If there is no error, the decremented +// value is returned. +func (c *cache) DecrementInt8(k string, n int8) (int8, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(int8) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an int8", k) + } + nv := rv - n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Decrement an item of type int16 by n. Returns an error if the item's value is +// not an int16, or if it was not found. If there is no error, the decremented +// value is returned. +func (c *cache) DecrementInt16(k string, n int16) (int16, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(int16) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an int16", k) + } + nv := rv - n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Decrement an item of type int32 by n. Returns an error if the item's value is +// not an int32, or if it was not found. If there is no error, the decremented +// value is returned. +func (c *cache) DecrementInt32(k string, n int32) (int32, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(int32) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an int32", k) + } + nv := rv - n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Decrement an item of type int64 by n. Returns an error if the item's value is +// not an int64, or if it was not found. If there is no error, the decremented +// value is returned. +func (c *cache) DecrementInt64(k string, n int64) (int64, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(int64) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an int64", k) + } + nv := rv - n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Decrement an item of type uint by n. Returns an error if the item's value is +// not an uint, or if it was not found. If there is no error, the decremented +// value is returned. +func (c *cache) DecrementUint(k string, n uint) (uint, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(uint) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an uint", k) + } + nv := rv - n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Decrement an item of type uintptr by n. Returns an error if the item's value +// is not an uintptr, or if it was not found. If there is no error, the +// decremented value is returned. +func (c *cache) DecrementUintptr(k string, n uintptr) (uintptr, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(uintptr) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an uintptr", k) + } + nv := rv - n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Decrement an item of type uint8 by n. Returns an error if the item's value is +// not an uint8, or if it was not found. If there is no error, the decremented +// value is returned. +func (c *cache) DecrementUint8(k string, n uint8) (uint8, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(uint8) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an uint8", k) + } + nv := rv - n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Decrement an item of type uint16 by n. Returns an error if the item's value +// is not an uint16, or if it was not found. If there is no error, the +// decremented value is returned. +func (c *cache) DecrementUint16(k string, n uint16) (uint16, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(uint16) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an uint16", k) + } + nv := rv - n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Decrement an item of type uint32 by n. Returns an error if the item's value +// is not an uint32, or if it was not found. If there is no error, the +// decremented value is returned. +func (c *cache) DecrementUint32(k string, n uint32) (uint32, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(uint32) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an uint32", k) + } + nv := rv - n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Decrement an item of type uint64 by n. Returns an error if the item's value +// is not an uint64, or if it was not found. If there is no error, the +// decremented value is returned. +func (c *cache) DecrementUint64(k string, n uint64) (uint64, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(uint64) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an uint64", k) + } + nv := rv - n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Decrement an item of type float32 by n. Returns an error if the item's value +// is not an float32, or if it was not found. If there is no error, the +// decremented value is returned. +func (c *cache) DecrementFloat32(k string, n float32) (float32, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(float32) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an float32", k) + } + nv := rv - n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Decrement an item of type float64 by n. Returns an error if the item's value +// is not an float64, or if it was not found. If there is no error, the +// decremented value is returned. +func (c *cache) DecrementFloat64(k string, n float64) (float64, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(float64) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an float64", k) + } + nv := rv - n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Delete an item from the cache. Does nothing if the key is not in the cache. +func (c *cache) Delete(k string) { + c.mu.Lock() + v, evicted := c.delete(k) + c.mu.Unlock() + if evicted { + c.onEvicted(k, v) + } +} + +func (c *cache) delete(k string) (interface{}, bool) { + if c.onEvicted != nil { + if v, found := c.items[k]; found { + delete(c.items, k) + return v.Object, true + } + } + delete(c.items, k) + return nil, false +} + +type keyAndValue struct { + key string + value interface{} +} + +// Delete all expired items from the cache. +func (c *cache) DeleteExpired() { + var evictedItems []keyAndValue + now := time.Now().UnixNano() + c.mu.Lock() + for k, v := range c.items { + // "Inlining" of expired + if v.Expiration > 0 && now > v.Expiration { + ov, evicted := c.delete(k) + if evicted { + evictedItems = append(evictedItems, keyAndValue{k, ov}) + } + } + } + c.mu.Unlock() + for _, v := range evictedItems { + c.onEvicted(v.key, v.value) + } +} + +// Sets an (optional) function that is called with the key and value when an +// item is evicted from the cache. (Including when it is deleted manually, but +// not when it is overwritten.) Set to nil to disable. +func (c *cache) OnEvicted(f func(string, interface{})) { + c.mu.Lock() + c.onEvicted = f + c.mu.Unlock() +} + +// Write the cache's items (using Gob) to an io.Writer. +// +// NOTE: This method is deprecated in favor of c.Items() and NewFrom() (see the +// documentation for NewFrom().) +func (c *cache) Save(w io.Writer) (err error) { + enc := gob.NewEncoder(w) + defer func() { + if x := recover(); x != nil { + err = fmt.Errorf("Error registering item types with Gob library") + } + }() + c.mu.RLock() + defer c.mu.RUnlock() + for _, v := range c.items { + gob.Register(v.Object) + } + err = enc.Encode(&c.items) + return +} + +// Save the cache's items to the given filename, creating the file if it +// doesn't exist, and overwriting it if it does. +// +// NOTE: This method is deprecated in favor of c.Items() and NewFrom() (see the +// documentation for NewFrom().) +func (c *cache) SaveFile(fname string) error { + fp, err := os.Create(fname) + if err != nil { + return err + } + err = c.Save(fp) + if err != nil { + fp.Close() + return err + } + return fp.Close() +} + +// Add (Gob-serialized) cache items from an io.Reader, excluding any items with +// keys that already exist (and haven't expired) in the current cache. +// +// NOTE: This method is deprecated in favor of c.Items() and NewFrom() (see the +// documentation for NewFrom().) +func (c *cache) Load(r io.Reader) error { + dec := gob.NewDecoder(r) + items := map[string]Item{} + err := dec.Decode(&items) + if err == nil { + c.mu.Lock() + defer c.mu.Unlock() + for k, v := range items { + ov, found := c.items[k] + if !found || ov.Expired() { + c.items[k] = v + } + } + } + return err +} + +// Load and add cache items from the given filename, excluding any items with +// keys that already exist in the current cache. +// +// NOTE: This method is deprecated in favor of c.Items() and NewFrom() (see the +// documentation for NewFrom().) +func (c *cache) LoadFile(fname string) error { + fp, err := os.Open(fname) + if err != nil { + return err + } + err = c.Load(fp) + if err != nil { + fp.Close() + return err + } + return fp.Close() +} + +// Copies all unexpired items in the cache into a new map and returns it. +func (c *cache) Items() map[string]Item { + c.mu.RLock() + defer c.mu.RUnlock() + m := make(map[string]Item, len(c.items)) + now := time.Now().UnixNano() + for k, v := range c.items { + // "Inlining" of Expired + if v.Expiration > 0 { + if now > v.Expiration { + continue + } + } + m[k] = v + } + return m +} + +// Returns the number of items in the cache. This may include items that have +// expired, but have not yet been cleaned up. +func (c *cache) ItemCount() int { + c.mu.RLock() + n := len(c.items) + c.mu.RUnlock() + return n +} + +// Delete all items from the cache. +func (c *cache) Flush() { + c.mu.Lock() + c.items = map[string]Item{} + c.mu.Unlock() +} + +type janitor struct { + Interval time.Duration + stop chan bool +} + +func (j *janitor) Run(c *cache) { + j.stop = make(chan bool) + ticker := time.NewTicker(j.Interval) + for { + select { + case <-ticker.C: + c.DeleteExpired() + case <-j.stop: + ticker.Stop() + return + } + } +} + +func stopJanitor(c *Cache) { + c.janitor.stop <- true +} + +func runJanitor(c *cache, ci time.Duration) { + j := &janitor{ + Interval: ci, + } + c.janitor = j + go j.Run(c) +} + +func newCache(de time.Duration, m map[string]Item) *cache { + if de == 0 { + de = -1 + } + c := &cache{ + defaultExpiration: de, + items: m, + } + return c +} + +func newCacheWithJanitor(de time.Duration, ci time.Duration, m map[string]Item) *Cache { + c := newCache(de, m) + // This trick ensures that the janitor goroutine (which--granted it + // was enabled--is running DeleteExpired on c forever) does not keep + // the returned C object from being garbage collected. When it is + // garbage collected, the finalizer stops the janitor goroutine, after + // which c can be collected. + C := &Cache{c} + if ci > 0 { + runJanitor(c, ci) + runtime.SetFinalizer(C, stopJanitor) + } + return C +} + +// Return a new cache with a given default expiration duration and cleanup +// interval. If the expiration duration is less than one (or NoExpiration), +// the items in the cache never expire (by default), and must be deleted +// manually. If the cleanup interval is less than one, expired items are not +// deleted from the cache before calling c.DeleteExpired(). +func New(defaultExpiration, cleanupInterval time.Duration) *Cache { + items := make(map[string]Item) + return newCacheWithJanitor(defaultExpiration, cleanupInterval, items) +} + +// Return a new cache with a given default expiration duration and cleanup +// interval. If the expiration duration is less than one (or NoExpiration), +// the items in the cache never expire (by default), and must be deleted +// manually. If the cleanup interval is less than one, expired items are not +// deleted from the cache before calling c.DeleteExpired(). +// +// NewFrom() also accepts an items map which will serve as the underlying map +// for the cache. This is useful for starting from a deserialized cache +// (serialized using e.g. gob.Encode() on c.Items()), or passing in e.g. +// make(map[string]Item, 500) to improve startup performance when the cache +// is expected to reach a certain minimum size. +// +// Only the cache's methods synchronize access to this map, so it is not +// recommended to keep any references to the map around after creating a cache. +// If need be, the map can be accessed at a later point using c.Items() (subject +// to the same caveat.) +// +// Note regarding serialization: When using e.g. gob, make sure to +// gob.Register() the individual types stored in the cache before encoding a +// map retrieved with c.Items(), and to register those same types before +// decoding a blob containing an items map. +func NewFrom(defaultExpiration, cleanupInterval time.Duration, items map[string]Item) *Cache { + return newCacheWithJanitor(defaultExpiration, cleanupInterval, items) +} diff --git a/vendor/github.com/patrickmn/go-cache/sharded.go b/vendor/github.com/patrickmn/go-cache/sharded.go new file mode 100644 index 00000000000..bcc0538bcc7 --- /dev/null +++ b/vendor/github.com/patrickmn/go-cache/sharded.go @@ -0,0 +1,192 @@ +package cache + +import ( + "crypto/rand" + "math" + "math/big" + insecurerand "math/rand" + "os" + "runtime" + "time" +) + +// This is an experimental and unexported (for now) attempt at making a cache +// with better algorithmic complexity than the standard one, namely by +// preventing write locks of the entire cache when an item is added. As of the +// time of writing, the overhead of selecting buckets results in cache +// operations being about twice as slow as for the standard cache with small +// total cache sizes, and faster for larger ones. +// +// See cache_test.go for a few benchmarks. + +type unexportedShardedCache struct { + *shardedCache +} + +type shardedCache struct { + seed uint32 + m uint32 + cs []*cache + janitor *shardedJanitor +} + +// djb2 with better shuffling. 5x faster than FNV with the hash.Hash overhead. +func djb33(seed uint32, k string) uint32 { + var ( + l = uint32(len(k)) + d = 5381 + seed + l + i = uint32(0) + ) + // Why is all this 5x faster than a for loop? + if l >= 4 { + for i < l-4 { + d = (d * 33) ^ uint32(k[i]) + d = (d * 33) ^ uint32(k[i+1]) + d = (d * 33) ^ uint32(k[i+2]) + d = (d * 33) ^ uint32(k[i+3]) + i += 4 + } + } + switch l - i { + case 1: + case 2: + d = (d * 33) ^ uint32(k[i]) + case 3: + d = (d * 33) ^ uint32(k[i]) + d = (d * 33) ^ uint32(k[i+1]) + case 4: + d = (d * 33) ^ uint32(k[i]) + d = (d * 33) ^ uint32(k[i+1]) + d = (d * 33) ^ uint32(k[i+2]) + } + return d ^ (d >> 16) +} + +func (sc *shardedCache) bucket(k string) *cache { + return sc.cs[djb33(sc.seed, k)%sc.m] +} + +func (sc *shardedCache) Set(k string, x interface{}, d time.Duration) { + sc.bucket(k).Set(k, x, d) +} + +func (sc *shardedCache) Add(k string, x interface{}, d time.Duration) error { + return sc.bucket(k).Add(k, x, d) +} + +func (sc *shardedCache) Replace(k string, x interface{}, d time.Duration) error { + return sc.bucket(k).Replace(k, x, d) +} + +func (sc *shardedCache) Get(k string) (interface{}, bool) { + return sc.bucket(k).Get(k) +} + +func (sc *shardedCache) Increment(k string, n int64) error { + return sc.bucket(k).Increment(k, n) +} + +func (sc *shardedCache) IncrementFloat(k string, n float64) error { + return sc.bucket(k).IncrementFloat(k, n) +} + +func (sc *shardedCache) Decrement(k string, n int64) error { + return sc.bucket(k).Decrement(k, n) +} + +func (sc *shardedCache) Delete(k string) { + sc.bucket(k).Delete(k) +} + +func (sc *shardedCache) DeleteExpired() { + for _, v := range sc.cs { + v.DeleteExpired() + } +} + +// Returns the items in the cache. This may include items that have expired, +// but have not yet been cleaned up. If this is significant, the Expiration +// fields of the items should be checked. Note that explicit synchronization +// is needed to use a cache and its corresponding Items() return values at +// the same time, as the maps are shared. +func (sc *shardedCache) Items() []map[string]Item { + res := make([]map[string]Item, len(sc.cs)) + for i, v := range sc.cs { + res[i] = v.Items() + } + return res +} + +func (sc *shardedCache) Flush() { + for _, v := range sc.cs { + v.Flush() + } +} + +type shardedJanitor struct { + Interval time.Duration + stop chan bool +} + +func (j *shardedJanitor) Run(sc *shardedCache) { + j.stop = make(chan bool) + tick := time.Tick(j.Interval) + for { + select { + case <-tick: + sc.DeleteExpired() + case <-j.stop: + return + } + } +} + +func stopShardedJanitor(sc *unexportedShardedCache) { + sc.janitor.stop <- true +} + +func runShardedJanitor(sc *shardedCache, ci time.Duration) { + j := &shardedJanitor{ + Interval: ci, + } + sc.janitor = j + go j.Run(sc) +} + +func newShardedCache(n int, de time.Duration) *shardedCache { + max := big.NewInt(0).SetUint64(uint64(math.MaxUint32)) + rnd, err := rand.Int(rand.Reader, max) + var seed uint32 + if err != nil { + os.Stderr.Write([]byte("WARNING: go-cache's newShardedCache failed to read from the system CSPRNG (/dev/urandom or equivalent.) Your system's security may be compromised. Continuing with an insecure seed.\n")) + seed = insecurerand.Uint32() + } else { + seed = uint32(rnd.Uint64()) + } + sc := &shardedCache{ + seed: seed, + m: uint32(n), + cs: make([]*cache, n), + } + for i := 0; i < n; i++ { + c := &cache{ + defaultExpiration: de, + items: map[string]Item{}, + } + sc.cs[i] = c + } + return sc +} + +func unexportedNewSharded(defaultExpiration, cleanupInterval time.Duration, shards int) *unexportedShardedCache { + if defaultExpiration == 0 { + defaultExpiration = -1 + } + sc := newShardedCache(shards, defaultExpiration) + SC := &unexportedShardedCache{sc} + if cleanupInterval > 0 { + runShardedJanitor(sc, cleanupInterval) + runtime.SetFinalizer(SC, stopShardedJanitor) + } + return SC +} diff --git a/vendor/vendor.json b/vendor/vendor.json index 941af06bd47..388a4e69c5e 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -314,6 +314,12 @@ "version": "v1.21.1", "versionExact": "v1.21.1" }, + { + "checksumSHA1": "8z32QKTSDusa4QQyunKE4kyYXZ8=", + "path": "github.com/patrickmn/go-cache", + "revision": "e7a9def80f35fe1b170b7b8b68871d59dea117e1", + "revisionTime": "2016-11-25T23:48:19Z" + }, { "checksumSHA1": "SMUvX2B8eoFd9wnPofwBKlN6btE=", "path": "github.com/prometheus/client_golang/api/prometheus",