From 8e7d23cdebc3df236d519777e3e4485d5ad32d12 Mon Sep 17 00:00:00 2001 From: wph95 Date: Fri, 23 Mar 2018 23:50:16 +0800 Subject: [PATCH 01/22] wip Signed-off-by: wph95 --- pkg/cmd/grafana-server/main.go | 1 + pkg/tsdb/elasticsearch/elasticsearch.go | 131 +++++++++++ pkg/tsdb/elasticsearch/model_parser.go | 97 +++++++++ pkg/tsdb/elasticsearch/models.go | 131 +++++++++++ pkg/tsdb/elasticsearch/query.go | 204 ++++++++++++++++++ pkg/tsdb/elasticsearch/response_parser.go | 111 ++++++++++ .../datasource/elasticsearch/plugin.json | 1 + 7 files changed, 676 insertions(+) create mode 100644 pkg/tsdb/elasticsearch/elasticsearch.go create mode 100644 pkg/tsdb/elasticsearch/model_parser.go create mode 100644 pkg/tsdb/elasticsearch/models.go create mode 100644 pkg/tsdb/elasticsearch/query.go create mode 100644 pkg/tsdb/elasticsearch/response_parser.go diff --git a/pkg/cmd/grafana-server/main.go b/pkg/cmd/grafana-server/main.go index ab0e12f2d9f..21090153bc0 100644 --- a/pkg/cmd/grafana-server/main.go +++ b/pkg/cmd/grafana-server/main.go @@ -21,6 +21,7 @@ import ( _ "github.com/grafana/grafana/pkg/services/alerting/conditions" _ "github.com/grafana/grafana/pkg/services/alerting/notifiers" _ "github.com/grafana/grafana/pkg/tsdb/cloudwatch" + _ "github.com/grafana/grafana/pkg/tsdb/elasticsearch" _ "github.com/grafana/grafana/pkg/tsdb/graphite" _ "github.com/grafana/grafana/pkg/tsdb/influxdb" _ "github.com/grafana/grafana/pkg/tsdb/mysql" diff --git a/pkg/tsdb/elasticsearch/elasticsearch.go b/pkg/tsdb/elasticsearch/elasticsearch.go new file mode 100644 index 00000000000..d67b4ad902d --- /dev/null +++ b/pkg/tsdb/elasticsearch/elasticsearch.go @@ -0,0 +1,131 @@ +package elasticsearch + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "github.com/davecgh/go-spew/spew" + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/tsdb" + "golang.org/x/net/context/ctxhttp" + "net/http" + "net/url" + "path" + "strings" + "time" +) + +type ElasticsearchExecutor struct { + Transport *http.Transport +} + +var ( + glog log.Logger + intervalCalculator tsdb.IntervalCalculator +) + +func NewElasticsearchExecutor(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) { + transport, err := dsInfo.GetHttpTransport() + if err != nil { + return nil, err + } + + return &ElasticsearchExecutor{ + Transport: transport, + }, nil +} + +func init() { + glog = log.New("tsdb.elasticsearch") + tsdb.RegisterTsdbQueryEndpoint("elasticsearch", NewElasticsearchExecutor) + intervalCalculator = tsdb.NewIntervalCalculator(&tsdb.IntervalOptions{MinInterval: time.Millisecond * 1}) +} + +func (e *ElasticsearchExecutor) Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) { + result := &tsdb.Response{} + result.Results = make(map[string]*tsdb.QueryResult) + + queryParser := ElasticSearchQueryParser{ + dsInfo, + tsdbQuery.TimeRange, + tsdbQuery.Queries, + glog, + } + + glog.Warn(spew.Sdump(dsInfo)) + glog.Warn(spew.Sdump(tsdbQuery)) + + payload, err := queryParser.Parse() + if err != nil { + return nil, err + } + + if setting.Env == setting.DEV { + glog.Debug("Elasticsearch playload", "raw playload", payload) + } + glog.Info("Elasticsearch playload", "raw playload", payload) + + req, err := e.createRequest(dsInfo, payload) + if err != nil { + return nil, err + } + + httpClient, err := dsInfo.GetHttpClient() + if err != nil { + return nil, err + } + + resp, err := ctxhttp.Do(ctx, httpClient, req) + if err != nil { + return nil, err + } + + if resp.StatusCode/100 != 2 { + return nil, fmt.Errorf("elasticsearch returned statuscode invalid status code: %v", resp.Status) + } + + var responses Responses + dec := json.NewDecoder(resp.Body) + defer resp.Body.Close() + dec.UseNumber() + err = dec.Decode(&responses) + if err != nil { + return nil, err + } + + glog.Warn(spew.Sdump(responses)) + for _, res := range responses.Responses { + if res.Err != nil { + return nil, errors.New(res.getErrMsg()) + } + + } + + return result, nil +} + +func (e *ElasticsearchExecutor) createRequest(dsInfo *models.DataSource, query string) (*http.Request, error) { + u, _ := url.Parse(dsInfo.Url) + u.Path = path.Join(u.Path, "_msearch") + req, err := http.NewRequest(http.MethodPost, u.String(), strings.NewReader(query)) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", "Grafana") + req.Header.Set("Content-Type", "application/json") + + if dsInfo.BasicAuth { + req.SetBasicAuth(dsInfo.BasicAuthUser, dsInfo.BasicAuthPassword) + } + + if !dsInfo.BasicAuth && dsInfo.User != "" { + req.SetBasicAuth(dsInfo.User, dsInfo.Password) + } + + glog.Debug("Elasticsearch request", "url", req.URL.String()) + glog.Debug("Elasticsearch request", "body", query) + return req, nil +} diff --git a/pkg/tsdb/elasticsearch/model_parser.go b/pkg/tsdb/elasticsearch/model_parser.go new file mode 100644 index 00000000000..136db6baed7 --- /dev/null +++ b/pkg/tsdb/elasticsearch/model_parser.go @@ -0,0 +1,97 @@ +package elasticsearch + +import ( + "bytes" + "encoding/json" + "fmt" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/tsdb" + "src/github.com/davecgh/go-spew/spew" + "strconv" + "strings" + "time" +) + +type ElasticSearchQueryParser struct { + DsInfo *models.DataSource + TimeRange *tsdb.TimeRange + Queries []*tsdb.Query + glog log.Logger +} + +func (qp *ElasticSearchQueryParser) Parse() (string, error) { + payload := bytes.Buffer{} + queryHeader := qp.getQueryHeader() + + for _, q := range qp.Queries { + timeField, err := q.Model.Get("timeField").String() + if err != nil { + return "", err + } + rawQuery := q.Model.Get("query").MustString("") + bucketAggs := q.Model.Get("bucketAggs").MustArray() + metrics := q.Model.Get("metrics").MustArray() + alias := q.Model.Get("alias").MustString("") + builder := QueryBuilder{timeField, rawQuery, bucketAggs, metrics, alias} + + query, err := builder.Build() + if err != nil { + return "", err + } + queryBytes, err := json.Marshal(query) + if err != nil { + return "", err + } + + payload.WriteString(queryHeader.String() + "\n") + payload.WriteString(string(queryBytes) + "\n") + } + + return qp.payloadReplace(payload.String(), qp.DsInfo.JsonData) + +} + +func (qp *ElasticSearchQueryParser) getQueryHeader() *QueryHeader { + var header QueryHeader + esVersion := qp.DsInfo.JsonData.Get("esVersion").MustInt() + + searchType := "query_then_fetch" + if esVersion < 5 { + searchType = "count" + } + header.SearchType = searchType + header.IgnoreUnavailable = true + header.Index = qp.getIndexList() + + if esVersion >= 56 { + header.MaxConcurrentShardRequests = qp.DsInfo.JsonData.Get("maxConcurrentShardRequests").MustInt() + } + return &header +} +func (qp *ElasticSearchQueryParser) payloadReplace(payload string, model *simplejson.Json) (string, error) { + parsedInterval, err := tsdb.GetIntervalFrom(qp.DsInfo, model, time.Millisecond) + if err != nil { + return "", nil + } + + interval := intervalCalculator.Calculate(qp.TimeRange, parsedInterval) + glog.Warn(spew.Sdump(interval)) + payload = strings.Replace(payload, "$timeFrom", fmt.Sprintf("%d", qp.TimeRange.GetFromAsMsEpoch()), -1) + payload = strings.Replace(payload, "$timeTo", fmt.Sprintf("%d", qp.TimeRange.GetToAsMsEpoch()), -1) + payload = strings.Replace(payload, "$interval", interval.Text, -1) + payload = strings.Replace(payload, "$__interval_ms", strconv.FormatInt(interval.Value.Nanoseconds()/int64(time.Millisecond), 10), -1) + payload = strings.Replace(payload, "$__interval", interval.Text, -1) + + return payload, nil +} + +func (qp *ElasticSearchQueryParser) getIndexList() string { + _, err := qp.DsInfo.JsonData.Get("interval").String() + if err != nil { + return qp.DsInfo.Database + } + // todo: support interval + return qp.DsInfo.Database +} diff --git a/pkg/tsdb/elasticsearch/models.go b/pkg/tsdb/elasticsearch/models.go new file mode 100644 index 00000000000..8662f6efbd3 --- /dev/null +++ b/pkg/tsdb/elasticsearch/models.go @@ -0,0 +1,131 @@ +package elasticsearch + +import ( + "github.com/grafana/grafana/pkg/components/simplejson" + "bytes" + "fmt" + "encoding/json" +) + +type QueryHeader struct { + SearchType string `json:"search_type"` + IgnoreUnavailable bool `json:"ignore_unavailable"` + Index interface{} `json:"index"` + MaxConcurrentShardRequests int `json:"max_concurrent_shard_requests"` +} + +func (q *QueryHeader) String() (string) { + r, _ := json.Marshal(q) + return string(r) +} + +type Query struct { + Query map[string]interface{} `json:"query"` + Aggs Aggs `json:"aggs"` + Size int `json:"size"` +} + +type Aggs map[string]interface{} + +type HistogramAgg struct { + Interval string `json:"interval,omitempty"` + Field string `json:"field"` + MinDocCount int `json:"min_doc_count"` + Missing string `json:"missing,omitempty"` +} + +type DateHistogramAgg struct { + HistogramAgg + ExtendedBounds ExtendedBounds `json:"extended_bounds"` + Format string `json:"format"` +} + +type FiltersAgg struct { + Filter map[string]interface{} `json:"filter"` +} + +type TermsAggSetting struct { + Field string `json:"field"` + Size int `json:"size"` + Order map[string]interface{} `json:"order"` + MinDocCount int `json:"min_doc_count"` + Missing string `json:"missing"` +} + +type TermsAgg struct { + Terms TermsAggSetting `json:"terms"` + Aggs Aggs `json:"aggs"` +} + +type ExtendedBounds struct { + Min string `json:"min"` + Max string `json:"max"` +} + +type RangeFilter struct { + Range map[string]RangeFilterSetting `json:"range"` +} +type RangeFilterSetting struct { + Gte string `json:"gte"` + Lte string `json:"lte"` + Format string `json:"format"` +} + +func newRangeFilter(field string, rangeFilterSetting RangeFilterSetting) *RangeFilter { + return &RangeFilter{ + map[string]RangeFilterSetting{field: rangeFilterSetting}} +} + +type QueryStringFilter struct { + QueryString QueryStringFilterSetting `json:"query_string"` +} +type QueryStringFilterSetting struct { + AnalyzeWildcard bool `json:"analyze_wildcard"` + Query string `json:"query"` +} + +func newQueryStringFilter(analyzeWildcard bool, query string) *QueryStringFilter { + return &QueryStringFilter{QueryStringFilterSetting{AnalyzeWildcard: analyzeWildcard, Query: query}} +} + +type BoolQuery struct { + Filter []interface{} `json:"filter"` +} + +type Metric map[string]interface{} + +type Responses struct { + Responses []Response `json:"responses"` +} + +type Response struct { + Status int `json:"status"` + Err map[string]interface{} `json:"error"` + Aggregations map[string]interface{} `json:"aggregations"` +} + +func (r *Response) getErrMsg() (string) { + var msg bytes.Buffer + errJson := simplejson.NewFromAny(r.Err) + errType, err := errJson.Get("type").String() + if err == nil { + msg.WriteString(fmt.Sprintf("type:%s", errType)) + } + + reason, err := errJson.Get("type").String() + if err == nil { + msg.WriteString(fmt.Sprintf("reason:%s", reason)) + } + return msg.String() +} + +type PercentilesResult struct { + Buckets struct { + map[string]struct { + Values map[string]string `json:"values"` + } + KeyAsString string `json:"key_as_string"` + Key int64 `json:"key"` + DocCount int `json:"doc_count"` + } `json:"buckets"` +} diff --git a/pkg/tsdb/elasticsearch/query.go b/pkg/tsdb/elasticsearch/query.go new file mode 100644 index 00000000000..69dd5caa3b4 --- /dev/null +++ b/pkg/tsdb/elasticsearch/query.go @@ -0,0 +1,204 @@ +package elasticsearch + +import ( + "errors" + "github.com/grafana/grafana/pkg/components/simplejson" +) + +var rangeFilterSetting = RangeFilterSetting{Gte: "$timeFrom", + Lte: "$timeTo", + Format: "epoch_millis"} + +type QueryBuilder struct { + TimeField string + RawQuery string + BucketAggs []interface{} + Metrics []interface{} + Alias string +} + +func (b *QueryBuilder) Build() (Query, error) { + var err error + var res Query + res.Query = make(map[string]interface{}) + res.Size = 0 + + if err != nil { + return res, err + } + + boolQuery := BoolQuery{} + boolQuery.Filter = append(boolQuery.Filter, newRangeFilter(b.TimeField, rangeFilterSetting)) + boolQuery.Filter = append(boolQuery.Filter, newQueryStringFilter(true, b.RawQuery)) + res.Query["bool"] = boolQuery + + // handle document query + if len(b.BucketAggs) == 0 { + if len(b.Metrics) > 0 { + metric := simplejson.NewFromAny(b.Metrics[0]) + if metric.Get("type").MustString("") == "raw_document" { + return res, errors.New("alert not support Raw_Document") + } + } + } + aggs, err := b.parseAggs(b.BucketAggs, b.Metrics) + res.Aggs = aggs["aggs"].(Aggs) + + return res, err +} + +func (b *QueryBuilder) parseAggs(bucketAggs []interface{}, metrics []interface{}) (Aggs, error) { + query := make(Aggs) + nestedAggs := query + for _, aggRaw := range bucketAggs { + esAggs := make(Aggs) + aggJson := simplejson.NewFromAny(aggRaw) + aggType, err := aggJson.Get("type").String() + if err != nil { + return nil, err + } + id, err := aggJson.Get("id").String() + if err != nil { + return nil, err + } + + switch aggType { + case "date_histogram": + esAggs["date_histogram"] = b.getDateHistogramAgg(aggJson) + case "histogram": + esAggs["histogram"] = b.getHistogramAgg(aggJson) + case "filters": + esAggs["filters"] = b.getFilters(aggJson) + case "terms": + terms := b.getTerms(aggJson) + esAggs["terms"] = terms.Terms + esAggs["aggs"] = terms.Aggs + case "geohash_grid": + return nil, errors.New("alert not support Geo_Hash_Grid") + } + + if _, ok := nestedAggs["aggs"]; !ok { + nestedAggs["aggs"] = make(Aggs) + } + + if aggs, ok := (nestedAggs["aggs"]).(Aggs); ok { + aggs[id] = esAggs + } + nestedAggs = esAggs + + } + nestedAggs["aggs"] = make(Aggs) + + for _, metricRaw := range metrics { + metric := make(Metric) + metricJson := simplejson.NewFromAny(metricRaw) + + id, err := metricJson.Get("id").String() + if err != nil { + return nil, err + } + metricType, err := metricJson.Get("type").String() + if err != nil { + return nil, err + } + if metricType == "count" { + continue + } + + // todo support pipeline Agg + + settings := metricJson.Get("settings").MustMap() + settings["field"] = metricJson.Get("field").MustString() + metric[metricType] = settings + nestedAggs["aggs"].(Aggs)[id] = metric + } + return query, nil +} + +func (b *QueryBuilder) getDateHistogramAgg(model *simplejson.Json) DateHistogramAgg { + agg := &DateHistogramAgg{} + settings := simplejson.NewFromAny(model.Get("settings").Interface()) + interval, err := settings.Get("interval").String() + if err == nil { + agg.Interval = interval + } + agg.Field = b.TimeField + agg.MinDocCount = settings.Get("min_doc_count").MustInt(0) + agg.ExtendedBounds = ExtendedBounds{"$timeFrom", "$timeTo"} + agg.Format = "epoch_millis" + + if agg.Interval == "auto" { + agg.Interval = "$__interval" + } + + missing, err := settings.Get("missing").String() + if err == nil { + agg.Missing = missing + } + return *agg +} + +func (b *QueryBuilder) getHistogramAgg(model *simplejson.Json) HistogramAgg { + agg := &HistogramAgg{} + settings := simplejson.NewFromAny(model.Get("settings").Interface()) + interval, err := settings.Get("interval").String() + if err == nil { + agg.Interval = interval + } + field, err := model.Get("field").String() + if err == nil { + agg.Field = field + } + agg.MinDocCount = settings.Get("min_doc_count").MustInt(0) + missing, err := settings.Get("missing").String() + if err == nil { + agg.Missing = missing + } + return *agg +} + +func (b *QueryBuilder) getFilters(model *simplejson.Json) FiltersAgg { + agg := &FiltersAgg{} + settings := simplejson.NewFromAny(model.Get("settings").Interface()) + for filter := range settings.Get("filters").MustArray() { + filterJson := simplejson.NewFromAny(filter) + query := filterJson.Get("query").MustString("") + label := filterJson.Get("label").MustString("") + if label == "" { + label = query + } + agg.Filter[label] = newQueryStringFilter(true, query) + } + return *agg +} + +func (b *QueryBuilder) getTerms(model *simplejson.Json) TermsAgg { + agg := &TermsAgg{} + settings := simplejson.NewFromAny(model.Get("settings").Interface()) + agg.Terms.Field = model.Get("field").MustString() + if settings == nil { + return *agg + } + agg.Terms.Size = settings.Get("size").MustInt(0) + if agg.Terms.Size == 0 { + agg.Terms.Size = 500 + } + orderBy := settings.Get("orderBy").MustString("") + if orderBy != "" { + agg.Terms.Order[orderBy] = settings.Get("order").MustString("") + // if orderBy is a int, means this fields is metric result value + // TODO set subAggs + } + + minDocCount, err := settings.Get("min_doc_count").Int() + if err == nil { + agg.Terms.MinDocCount = minDocCount + } + + missing, err := settings.Get("missing").String() + if err == nil { + agg.Terms.Missing = missing + } + + return *agg +} diff --git a/pkg/tsdb/elasticsearch/response_parser.go b/pkg/tsdb/elasticsearch/response_parser.go new file mode 100644 index 00000000000..bc47a3f935e --- /dev/null +++ b/pkg/tsdb/elasticsearch/response_parser.go @@ -0,0 +1,111 @@ +package elasticsearch + +import ( + "errors" + "fmt" + "github.com/grafana/grafana/pkg/components/null" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/tsdb" + "strconv" +) + +type ElasticsearchResponseParser struct { + Responses []Response + Targets []QueryBuilder +} + +func (rp *ElasticsearchResponseParser) getTimeSeries() []interface{} { + for i, res := range rp.Responses { + var series []interface{} + target := rp.Targets[i] + props := make(map[string]interface{}) + rp.processBuckets(res.Aggregations, target, &series, props, 0) + } +} + +func findAgg(target QueryBuilder, aggId string) (*simplejson.Json, error) { + for _, v := range target.BucketAggs { + aggDef := simplejson.NewFromAny(v) + if aggId == aggDef.Get("id").MustString() { + return aggDef, nil + } + } + return nil, errors.New("can't found aggDef, aggID:" + aggId) +} + +func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{}, target QueryBuilder, series *[]interface{}, props map[string]interface{}, depth int) error { + maxDepth := len(target.BucketAggs) - 1 + for aggId, v := range aggs { + aggDef, _ := findAgg(target, aggId) + esAgg := simplejson.NewFromAny(v) + if aggDef == nil { + continue + } + + if depth == maxDepth { + if aggDef.Get("type").MustString() == "date_histogram" { + rp.processMetrics(esAgg, target, series, props) + } + } + + } + +} + +func mapCopy(originalMap, newMap *map[string]string) { + for k, v := range originalMap { + newMap[k] = v + } + +} + +func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, target QueryBuilder, props map[string]string) ([]*tsdb.TimeSeries, error) { + var series []*tsdb.TimeSeries + for _, v := range target.Metrics { + metric := simplejson.NewFromAny(v) + if metric.Get("hide").MustBool(false) { + continue + } + metricId := fmt.Sprintf("%d", metric.Get("id").MustInt()) + metricField := metric.Get("field").MustString() + + switch metric.Get("type").MustString() { + case "count": + newSeries := tsdb.TimeSeries{} + for _, v := range esAgg.Get("buckets").MustMap() { + bucket := simplejson.NewFromAny(v) + value := bucket.Get("doc_count").MustFloat64() + key := bucket.Get("key").MustFloat64() + newSeries.Points = append(newSeries.Points, tsdb.TimePoint{null.FloatFromPtr(&value), null.FloatFromPtr(&key)}) + } + newSeries.Tags = props + newSeries.Tags["metric"] = "count" + series = append(series, &newSeries) + + case "percentiles": + buckets := esAgg.Get("buckets").MustArray() + if len(buckets) == 0 { + break + } + + firstBucket := simplejson.NewFromAny(buckets[0]) + percentiles := firstBucket.GetPath(metricId, "values").MustMap() + + for percentileName := range percentiles { + newSeries := tsdb.TimeSeries{} + newSeries.Tags = props + newSeries.Tags["metric"] = "p" + percentileName + newSeries.Tags["field"] = metricField + for _, v := range buckets { + bucket := simplejson.NewFromAny(v) + valueStr := bucket.GetPath(metricId, "values", percentileName).MustString() + value, _ := strconv.ParseFloat(valueStr, 64) + key := bucket.Get("key").MustFloat64() + newSeries.Points = append(newSeries.Points, tsdb.TimePoint{null.FloatFromPtr(&value), null.FloatFromPtr(&key)}) + } + series = append(series, &newSeries) + } + } + } + return series +} diff --git a/public/app/plugins/datasource/elasticsearch/plugin.json b/public/app/plugins/datasource/elasticsearch/plugin.json index 59d26b785ac..89cca1251d5 100644 --- a/public/app/plugins/datasource/elasticsearch/plugin.json +++ b/public/app/plugins/datasource/elasticsearch/plugin.json @@ -20,6 +20,7 @@ "version": "5.0.0" }, + "alerting": true, "annotations": true, "metrics": true, From bc5b59737c2f6f99b64b395de2e20b888d043c97 Mon Sep 17 00:00:00 2001 From: wph95 Date: Sat, 24 Mar 2018 13:06:21 +0800 Subject: [PATCH 02/22] finished CODING PHASE 1 Signed-off-by: wph95 --- pkg/tsdb/elasticsearch/elasticsearch.go | 13 ++++--------- pkg/tsdb/elasticsearch/model_parser.go | 16 ++++++++-------- pkg/tsdb/elasticsearch/models.go | 11 ----------- 3 files changed, 12 insertions(+), 28 deletions(-) diff --git a/pkg/tsdb/elasticsearch/elasticsearch.go b/pkg/tsdb/elasticsearch/elasticsearch.go index d67b4ad902d..8fd82a179e8 100644 --- a/pkg/tsdb/elasticsearch/elasticsearch.go +++ b/pkg/tsdb/elasticsearch/elasticsearch.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/davecgh/go-spew/spew" "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/setting" @@ -52,13 +51,9 @@ func (e *ElasticsearchExecutor) Query(ctx context.Context, dsInfo *models.DataSo dsInfo, tsdbQuery.TimeRange, tsdbQuery.Queries, - glog, } - glog.Warn(spew.Sdump(dsInfo)) - glog.Warn(spew.Sdump(tsdbQuery)) - - payload, err := queryParser.Parse() + payload, targets, err := queryParser.Parse() if err != nil { return nil, err } @@ -96,14 +91,14 @@ func (e *ElasticsearchExecutor) Query(ctx context.Context, dsInfo *models.DataSo return nil, err } - glog.Warn(spew.Sdump(responses)) for _, res := range responses.Responses { if res.Err != nil { return nil, errors.New(res.getErrMsg()) } - } - + responseParser := ElasticsearchResponseParser{responses.Responses, targets} + queryRes := responseParser.getTimeSeries() + result.Results["A"] = queryRes return result, nil } diff --git a/pkg/tsdb/elasticsearch/model_parser.go b/pkg/tsdb/elasticsearch/model_parser.go index 136db6baed7..233a35efdc6 100644 --- a/pkg/tsdb/elasticsearch/model_parser.go +++ b/pkg/tsdb/elasticsearch/model_parser.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/tsdb" "src/github.com/davecgh/go-spew/spew" @@ -18,38 +17,39 @@ type ElasticSearchQueryParser struct { DsInfo *models.DataSource TimeRange *tsdb.TimeRange Queries []*tsdb.Query - glog log.Logger } -func (qp *ElasticSearchQueryParser) Parse() (string, error) { +func (qp *ElasticSearchQueryParser) Parse() (string, []*QueryBuilder, error) { payload := bytes.Buffer{} queryHeader := qp.getQueryHeader() - + targets := make([]*QueryBuilder, 0) for _, q := range qp.Queries { timeField, err := q.Model.Get("timeField").String() if err != nil { - return "", err + return "", nil, err } rawQuery := q.Model.Get("query").MustString("") bucketAggs := q.Model.Get("bucketAggs").MustArray() metrics := q.Model.Get("metrics").MustArray() alias := q.Model.Get("alias").MustString("") builder := QueryBuilder{timeField, rawQuery, bucketAggs, metrics, alias} + targets = append(targets, &builder) query, err := builder.Build() if err != nil { - return "", err + return "", nil, err } queryBytes, err := json.Marshal(query) if err != nil { - return "", err + return "", nil, err } payload.WriteString(queryHeader.String() + "\n") payload.WriteString(string(queryBytes) + "\n") } + p, err := qp.payloadReplace(payload.String(), qp.DsInfo.JsonData) - return qp.payloadReplace(payload.String(), qp.DsInfo.JsonData) + return p, targets, err } diff --git a/pkg/tsdb/elasticsearch/models.go b/pkg/tsdb/elasticsearch/models.go index 8662f6efbd3..d758e2159de 100644 --- a/pkg/tsdb/elasticsearch/models.go +++ b/pkg/tsdb/elasticsearch/models.go @@ -118,14 +118,3 @@ func (r *Response) getErrMsg() (string) { } return msg.String() } - -type PercentilesResult struct { - Buckets struct { - map[string]struct { - Values map[string]string `json:"values"` - } - KeyAsString string `json:"key_as_string"` - Key int64 `json:"key"` - DocCount int `json:"doc_count"` - } `json:"buckets"` -} From 1e275d0cd1ff976f44dfce6affe8661160cdd873 Mon Sep 17 00:00:00 2001 From: wph95 Date: Sun, 25 Mar 2018 02:18:28 +0800 Subject: [PATCH 03/22] set right series name Signed-off-by: wph95 --- pkg/tsdb/elasticsearch/query.go | 14 +- pkg/tsdb/elasticsearch/query_def.go | 26 +++ pkg/tsdb/elasticsearch/response_parser.go | 219 ++++++++++++++++++---- 3 files changed, 215 insertions(+), 44 deletions(-) create mode 100644 pkg/tsdb/elasticsearch/query_def.go diff --git a/pkg/tsdb/elasticsearch/query.go b/pkg/tsdb/elasticsearch/query.go index 69dd5caa3b4..d6d70e79a2a 100644 --- a/pkg/tsdb/elasticsearch/query.go +++ b/pkg/tsdb/elasticsearch/query.go @@ -3,10 +3,11 @@ package elasticsearch import ( "errors" "github.com/grafana/grafana/pkg/components/simplejson" + "strconv" ) var rangeFilterSetting = RangeFilterSetting{Gte: "$timeFrom", - Lte: "$timeTo", + Lte: "$timeTo", Format: "epoch_millis"} type QueryBuilder struct { @@ -173,18 +174,21 @@ func (b *QueryBuilder) getFilters(model *simplejson.Json) FiltersAgg { } func (b *QueryBuilder) getTerms(model *simplejson.Json) TermsAgg { - agg := &TermsAgg{} + agg := &TermsAgg{Aggs: make(Aggs)} settings := simplejson.NewFromAny(model.Get("settings").Interface()) agg.Terms.Field = model.Get("field").MustString() if settings == nil { return *agg } - agg.Terms.Size = settings.Get("size").MustInt(0) - if agg.Terms.Size == 0 { - agg.Terms.Size = 500 + sizeStr := settings.Get("size").MustString("") + size, err := strconv.Atoi(sizeStr) + if err != nil { + size = 500 } + agg.Terms.Size = size orderBy := settings.Get("orderBy").MustString("") if orderBy != "" { + agg.Terms.Order = make(map[string]interface{}) agg.Terms.Order[orderBy] = settings.Get("order").MustString("") // if orderBy is a int, means this fields is metric result value // TODO set subAggs diff --git a/pkg/tsdb/elasticsearch/query_def.go b/pkg/tsdb/elasticsearch/query_def.go new file mode 100644 index 00000000000..5dc02aa359e --- /dev/null +++ b/pkg/tsdb/elasticsearch/query_def.go @@ -0,0 +1,26 @@ +package elasticsearch + +var metricAggType = map[string]string{ + "count": "Count", + "avg": "Average", + "sum": "Sum", + "max": "Max", + "min": "Min", + "extended_stats": "Extended Stats", + "percentiles": "Percentiles", + "cardinality": "Unique Count", + "moving_avg": "Moving Average", + "derivative": "Derivative", + "raw_document": "Raw Document", +} + +var extendedStats = map[string]string{ + "avg": "Avg", + "min": "Min", + "max": "Max", + "sum": "Sum", + "count": "Count", + "std_deviation": "Std Dev", + "std_deviation_bounds_upper": "Std Dev Upper", + "std_deviation_bounds_lower": "Std Dev Lower", +} diff --git a/pkg/tsdb/elasticsearch/response_parser.go b/pkg/tsdb/elasticsearch/response_parser.go index bc47a3f935e..a2a8565641f 100644 --- a/pkg/tsdb/elasticsearch/response_parser.go +++ b/pkg/tsdb/elasticsearch/response_parser.go @@ -7,33 +7,30 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/tsdb" "strconv" + "regexp" + "strings" ) type ElasticsearchResponseParser struct { Responses []Response - Targets []QueryBuilder + Targets []*QueryBuilder } -func (rp *ElasticsearchResponseParser) getTimeSeries() []interface{} { +func (rp *ElasticsearchResponseParser) getTimeSeries() *tsdb.QueryResult { + queryRes := tsdb.NewQueryResult() for i, res := range rp.Responses { - var series []interface{} target := rp.Targets[i] - props := make(map[string]interface{}) + props := make(map[string]string) + series := make([]*tsdb.TimeSeries, 0) rp.processBuckets(res.Aggregations, target, &series, props, 0) + rp.nameSeries(&series, target) + queryRes.Series = append(queryRes.Series, series...) } + return queryRes } -func findAgg(target QueryBuilder, aggId string) (*simplejson.Json, error) { - for _, v := range target.BucketAggs { - aggDef := simplejson.NewFromAny(v) - if aggId == aggDef.Get("id").MustString() { - return aggDef, nil - } - } - return nil, errors.New("can't found aggDef, aggID:" + aggId) -} - -func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{}, target QueryBuilder, series *[]interface{}, props map[string]interface{}, depth int) error { +func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{}, target *QueryBuilder, series *[]*tsdb.TimeSeries, props map[string]string, depth int) (error) { + var err error maxDepth := len(target.BucketAggs) - 1 for aggId, v := range aggs { aggDef, _ := findAgg(target, aggId) @@ -44,43 +41,59 @@ func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{ if depth == maxDepth { if aggDef.Get("type").MustString() == "date_histogram" { - rp.processMetrics(esAgg, target, series, props) + err = rp.processMetrics(esAgg, target, series, props) + if err != nil { + return err + } + } else { + return fmt.Errorf("not support type:%s", aggDef.Get("type").MustString()) + } + } else { + for i, b := range esAgg.Get("buckets").MustArray() { + field := aggDef.Get("field").MustString() + bucket := simplejson.NewFromAny(b) + newProps := props + if key, err := bucket.Get("key").String(); err == nil { + newProps[field] = key + } else { + props["filter"] = strconv.Itoa(i) + } + + if key, err := bucket.Get("key_as_string").String(); err == nil { + props[field] = key + } + rp.processBuckets(bucket.MustMap(), target, series, newProps, depth+1) } } } + return nil } -func mapCopy(originalMap, newMap *map[string]string) { - for k, v := range originalMap { - newMap[k] = v - } - -} - -func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, target QueryBuilder, props map[string]string) ([]*tsdb.TimeSeries, error) { - var series []*tsdb.TimeSeries +func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, target *QueryBuilder, series *[]*tsdb.TimeSeries, props map[string]string) (error) { for _, v := range target.Metrics { metric := simplejson.NewFromAny(v) if metric.Get("hide").MustBool(false) { continue } - metricId := fmt.Sprintf("%d", metric.Get("id").MustInt()) - metricField := metric.Get("field").MustString() - switch metric.Get("type").MustString() { + metricId := metric.Get("id").MustString() + metricField := metric.Get("field").MustString() + metricType := metric.Get("type").MustString() + + switch metricType { case "count": newSeries := tsdb.TimeSeries{} - for _, v := range esAgg.Get("buckets").MustMap() { + for _, v := range esAgg.Get("buckets").MustArray() { bucket := simplejson.NewFromAny(v) - value := bucket.Get("doc_count").MustFloat64() - key := bucket.Get("key").MustFloat64() - newSeries.Points = append(newSeries.Points, tsdb.TimePoint{null.FloatFromPtr(&value), null.FloatFromPtr(&key)}) + value := castToNullFloat(bucket.Get("doc_count")) + key := castToNullFloat(bucket.Get("key")) + newSeries.Points = append(newSeries.Points, tsdb.TimePoint{value, key}) } newSeries.Tags = props newSeries.Tags["metric"] = "count" - series = append(series, &newSeries) + *series = append(*series, &newSeries) case "percentiles": buckets := esAgg.Get("buckets").MustArray() @@ -98,14 +111,142 @@ func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, ta newSeries.Tags["field"] = metricField for _, v := range buckets { bucket := simplejson.NewFromAny(v) - valueStr := bucket.GetPath(metricId, "values", percentileName).MustString() - value, _ := strconv.ParseFloat(valueStr, 64) - key := bucket.Get("key").MustFloat64() - newSeries.Points = append(newSeries.Points, tsdb.TimePoint{null.FloatFromPtr(&value), null.FloatFromPtr(&key)}) + value := castToNullFloat(bucket.GetPath(metricId, "values", percentileName)) + key := castToNullFloat(bucket.Get("key")) + newSeries.Points = append(newSeries.Points, tsdb.TimePoint{value, key}) } - series = append(series, &newSeries) + *series = append(*series, &newSeries) + } + default: + newSeries := tsdb.TimeSeries{} + newSeries.Tags = props + newSeries.Tags["metric"] = metricType + newSeries.Tags["field"] = metricField + for _, v := range esAgg.Get("buckets").MustArray() { + bucket := simplejson.NewFromAny(v) + key := castToNullFloat(bucket.Get("key")) + valueObj, err := bucket.Get(metricId).Map() + if err != nil { + break + } + var value null.Float + if _, ok := valueObj["normalized_value"]; ok { + value = castToNullFloat(bucket.GetPath(metricId, "normalized_value")) + } else { + value = castToNullFloat(bucket.GetPath(metricId, "value")) + } + newSeries.Points = append(newSeries.Points, tsdb.TimePoint{value, key}) + } + *series = append(*series, &newSeries) + } + } + return nil +} + +func (rp *ElasticsearchResponseParser) nameSeries(seriesList *[]*tsdb.TimeSeries, target *QueryBuilder) { + set := make(map[string]string) + for _, v := range *seriesList { + if metricType, exists := v.Tags["metric"]; exists { + if _, ok := set[metricType]; !ok { + set[metricType] = "" } } } - return series + metricTypeCount := len(set) + for _, series := range *seriesList { + series.Name = rp.getSeriesName(series, target, metricTypeCount) + } + +} + +func (rp *ElasticsearchResponseParser) getSeriesName(series *tsdb.TimeSeries, target *QueryBuilder, metricTypeCount int) (string) { + metricName := rp.getMetricName(series.Tags["metric"]) + delete(series.Tags, "metric") + + field := "" + if v, ok := series.Tags["field"]; ok { + field = v + delete(series.Tags, "field") + } + + if target.Alias != "" { + var re = regexp.MustCompile(`{{([\s\S]+?)}}`) + for _, match := range re.FindAllString(target.Alias, -1) { + group := match[2:len(match)-2] + + if strings.HasPrefix(group, "term ") { + if term, ok := series.Tags["term "]; ok { + strings.Replace(target.Alias, match, term, 1) + } + } + if v, ok := series.Tags[group]; ok { + strings.Replace(target.Alias, match, v, 1) + } + + switch group { + case "metric": + strings.Replace(target.Alias, match, metricName, 1) + case "field": + strings.Replace(target.Alias, match, field, 1) + } + + } + } + // todo, if field and pipelineAgg + if field != "" { + metricName += " " + field + } + + if len(series.Tags) == 0 { + return metricName + } + + name := "" + for _, v := range series.Tags { + name += v + " " + } + + if metricTypeCount == 1 { + return strings.TrimSpace(name) + } + + return strings.TrimSpace(name) + " " + metricName + +} + +func (rp *ElasticsearchResponseParser) getMetricName(metric string) string { + if text, ok := metricAggType[metric]; ok { + return text + } + + if text, ok := extendedStats[metric]; ok { + return text + } + + return metric +} + +func castToNullFloat(j *simplejson.Json) null.Float { + f, err := j.Float64() + if err == nil { + return null.FloatFrom(f) + } + + s, err := j.String() + if err == nil { + v, _ := strconv.ParseFloat(s, 64) + return null.FloatFromPtr(&v) + } + + return null.NewFloat(0, false) +} + +func findAgg(target *QueryBuilder, aggId string) (*simplejson.Json, error) { + for _, v := range target.BucketAggs { + aggDef := simplejson.NewFromAny(v) + if aggId == aggDef.Get("id").MustString() { + return aggDef, nil + } + } + return nil, errors.New("can't found aggDef, aggID:" + aggId) } From d6cdc2497c929039f93830dd8b7a61661046ae57 Mon Sep 17 00:00:00 2001 From: wph95 Date: Mon, 26 Mar 2018 16:13:14 +0800 Subject: [PATCH 04/22] Handle Interval Date Format similar to the JS variant https://github.com/grafana/grafana/pull/10343/commits/7e14e272fa37df5b4d412c16845d1e525711f726 --- Gopkg.lock | 8 +- Gopkg.toml | 4 + pkg/tsdb/elasticsearch/model_parser.go | 46 +- pkg/tsdb/elasticsearch/model_parser_test.go | 49 + vendor/github.com/leibowitz/moment/diff.go | 75 ++ vendor/github.com/leibowitz/moment/moment.go | 1185 +++++++++++++++++ .../leibowitz/moment/moment_parser.go | 100 ++ .../github.com/leibowitz/moment/parse_day.go | 32 + .../leibowitz/moment/strftime_parser.go | 68 + 9 files changed, 1559 insertions(+), 8 deletions(-) create mode 100644 pkg/tsdb/elasticsearch/model_parser_test.go create mode 100644 vendor/github.com/leibowitz/moment/diff.go create mode 100644 vendor/github.com/leibowitz/moment/moment.go create mode 100644 vendor/github.com/leibowitz/moment/moment_parser.go create mode 100644 vendor/github.com/leibowitz/moment/parse_day.go create mode 100644 vendor/github.com/leibowitz/moment/strftime_parser.go diff --git a/Gopkg.lock b/Gopkg.lock index ebadad8331b..78316b77664 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -295,6 +295,12 @@ packages = ["."] revision = "7cafcd837844e784b526369c9bce262804aebc60" +[[projects]] + branch = "master" + name = "github.com/leibowitz/moment" + packages = ["."] + revision = "8548108dcca204a1110b99e5fec966817499fe84" + [[projects]] branch = "master" name = "github.com/lib/pq" @@ -642,6 +648,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "5e65aeace832f1b4be17e7ff5d5714513c40f31b94b885f64f98f2332968d7c6" + inputs-digest = "9895ff7b1516b9639d0fc280ca155c8958486656a2086fc45e91f727fccea0d2" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index df163e01ed3..1f8cbba6e11 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -201,3 +201,7 @@ ignored = [ [[constraint]] name = "github.com/denisenkom/go-mssqldb" revision = "270bc3860bb94dd3a3ffd047377d746c5e276726" + +[[constraint]] + branch = "master" + name = "github.com/leibowitz/moment" diff --git a/pkg/tsdb/elasticsearch/model_parser.go b/pkg/tsdb/elasticsearch/model_parser.go index 233a35efdc6..7da6765e06c 100644 --- a/pkg/tsdb/elasticsearch/model_parser.go +++ b/pkg/tsdb/elasticsearch/model_parser.go @@ -7,6 +7,7 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/tsdb" + "github.com/leibowitz/moment" "src/github.com/davecgh/go-spew/spew" "strconv" "strings" @@ -63,7 +64,7 @@ func (qp *ElasticSearchQueryParser) getQueryHeader() *QueryHeader { } header.SearchType = searchType header.IgnoreUnavailable = true - header.Index = qp.getIndexList() + header.Index = getIndexList(qp.DsInfo.Database, qp.DsInfo.JsonData.Get("interval").MustString(""), qp.TimeRange) if esVersion >= 56 { header.MaxConcurrentShardRequests = qp.DsInfo.JsonData.Get("maxConcurrentShardRequests").MustInt() @@ -87,11 +88,42 @@ func (qp *ElasticSearchQueryParser) payloadReplace(payload string, model *simple return payload, nil } -func (qp *ElasticSearchQueryParser) getIndexList() string { - _, err := qp.DsInfo.JsonData.Get("interval").String() - if err != nil { - return qp.DsInfo.Database +func getIndexList(pattern string, interval string, timeRange *tsdb.TimeRange) string { + if interval == "" { + return pattern } - // todo: support interval - return qp.DsInfo.Database + + var indexes []string + indexParts := strings.Split(strings.TrimLeft(pattern, "["), "]") + indexBase := indexParts[0] + if len(indexParts) <= 1 { + return pattern + } + + indexDateFormat := indexParts[1] + + start := moment.NewMoment(timeRange.MustGetFrom()) + end := moment.NewMoment(timeRange.MustGetTo()) + + indexes = append(indexes, fmt.Sprintf("%s%s", indexBase, start.Format(indexDateFormat))) + for start.IsBefore(*end) { + switch interval { + case "Hourly": + start = start.AddHours(1) + + case "Daily": + start = start.AddDay() + + case "Weekly": + start = start.AddWeeks(1) + + case "Monthly": + start = start.AddMonths(1) + + case "Yearly": + start = start.AddYears(1) + } + indexes = append(indexes, fmt.Sprintf("%s%s", indexBase, start.Format(indexDateFormat))) + } + return strings.Join(indexes, ",") } diff --git a/pkg/tsdb/elasticsearch/model_parser_test.go b/pkg/tsdb/elasticsearch/model_parser_test.go new file mode 100644 index 00000000000..aa7336fb69b --- /dev/null +++ b/pkg/tsdb/elasticsearch/model_parser_test.go @@ -0,0 +1,49 @@ +package elasticsearch + +import ( + "github.com/grafana/grafana/pkg/tsdb" + . "github.com/smartystreets/goconvey/convey" + "strconv" + "strings" + "testing" +) + +func makeTime(hour int) string { + //unixtime 1500000000 == 2017-07-14T02:40:00+00:00 + return strconv.Itoa((1500000000 + hour*60*60) * 1000) +} + +func getIndexListByTime(pattern string, interval string, hour int) string { + timeRange := &tsdb.TimeRange{ + From: makeTime(0), + To: makeTime(hour), + } + return getIndexList(pattern, interval, timeRange) +} + +func TestElasticsearchGetIndexList(t *testing.T) { + Convey("Test Elasticsearch getIndex ", t, func() { + + Convey("Parse Interval Formats", func() { + So(getIndexListByTime("[logstash-]YYYY.MM.DD", "Daily", 48), + ShouldEqual, "logstash-2017.07.14,logstash-2017.07.15,logstash-2017.07.16") + + So(len(strings.Split(getIndexListByTime("[logstash-]YYYY.MM.DD.HH", "Hourly", 3), ",")), + ShouldEqual, 4) + + So(getIndexListByTime("[logstash-]YYYY.W", "Weekly", 100), + ShouldEqual, "logstash-2017.28,logstash-2017.29") + + So(getIndexListByTime("[logstash-]YYYY.MM", "Monthly", 700), + ShouldEqual, "logstash-2017.07,logstash-2017.08") + + So(getIndexListByTime("[logstash-]YYYY", "Yearly", 10000), + ShouldEqual, "logstash-2017,logstash-2018,logstash-2019") + }) + + Convey("No Interval", func() { + index := getIndexListByTime("logstash-test", "", 1) + So(index, ShouldEqual, "logstash-test") + }) + }) +} diff --git a/vendor/github.com/leibowitz/moment/diff.go b/vendor/github.com/leibowitz/moment/diff.go new file mode 100644 index 00000000000..0d6b3935adf --- /dev/null +++ b/vendor/github.com/leibowitz/moment/diff.go @@ -0,0 +1,75 @@ +package moment + +import ( + "fmt" + "math" + "time" +) + +// @todo In months/years requires the old and new to calculate correctly, right? +// @todo decide how to handle rounding (i.e. always floor?) +type Diff struct { + duration time.Duration +} + +func (d *Diff) InSeconds() int { + return int(d.duration.Seconds()) +} + +func (d *Diff) InMinutes() int { + return int(d.duration.Minutes()) +} + +func (d *Diff) InHours() int { + return int(d.duration.Hours()) +} + +func (d *Diff) InDays() int { + return int(math.Floor(float64(d.InSeconds()) / 86400)) +} + +// This depends on where the weeks fall? +func (d *Diff) InWeeks() int { + return int(math.Floor(float64(d.InDays() / 7))) +} + +func (d *Diff) InMonths() int { + return 0 +} + +func (d *Diff) InYears() int { + return 0 +} + +// http://momentjs.com/docs/#/durations/humanize/ +func (d *Diff) Humanize() string { + diffInSeconds := d.InSeconds() + + if diffInSeconds <= 45 { + return fmt.Sprintf("%d seconds ago", diffInSeconds) + } else if diffInSeconds <= 90 { + return "a minute ago" + } + + diffInMinutes := d.InMinutes() + + if diffInMinutes <= 45 { + return fmt.Sprintf("%d minutes ago", diffInMinutes) + } else if diffInMinutes <= 90 { + return "an hour ago" + } + + diffInHours := d.InHours() + + if diffInHours <= 22 { + return fmt.Sprintf("%d hours ago", diffInHours) + } else if diffInHours <= 36 { + return "a day ago" + } + + return "diff is in days" +} + +// In Months + +// In years diff --git a/vendor/github.com/leibowitz/moment/moment.go b/vendor/github.com/leibowitz/moment/moment.go new file mode 100644 index 00000000000..13c8ef7dbef --- /dev/null +++ b/vendor/github.com/leibowitz/moment/moment.go @@ -0,0 +1,1185 @@ +package moment + +import ( + "fmt" + "regexp" + "strconv" + "strings" + "time" +) + +// links +// http://en.wikipedia.org/wiki/ISO_week_date +// http://golang.org/src/pkg/time/format.go +// http://www.php.net/manual/en/class.datetime.php#datetime.constants.rfc822 +// http://php.net/manual/en/function.date.php +// http://www.php.net/manual/en/datetime.formats.relative.php + +// @todo are these constants needed if they are in the time package? +// There are a lot of extras here, and RFC822 doesn't match up. Why? +// Also, is timezone usage wrong? Double-check +const ( + ATOM = "2006-01-02T15:04:05Z07:00" + COOKIE = "Monday, 02-Jan-06 15:04:05 MST" + ISO8601 = "2006-01-02T15:04:05Z0700" + RFC822 = "Mon, 02 Jan 06 15:04:05 Z0700" + RFC850 = "Monday, 02-Jan-06 15:04:05 MST" + RFC1036 = "Mon, 02 Jan 06 15:04:05 Z0700" + RFC1123 = "Mon, 02 Jan 2006 15:04:05 Z0700" + RFC2822 = "Mon, 02 Jan 2006 15:04:05 Z0700" + RFC3339 = "2006-01-02T15:04:05Z07:00" + RSS = "Mon, 02 Jan 2006 15:04:05 Z0700" + W3C = "2006-01-02T15:04:05Z07:00" +) + +var ( + regex_days = "monday|mon|tuesday|tues|wednesday|wed|thursday|thurs|friday|fri|saturday|sat|sunday|sun" + regex_period = "second|minute|hour|day|week|month|year" + regex_numbers = "one|two|three|four|five|six|seven|eight|nine|ten" +) + +// regexp +var ( + compiled = regexp.MustCompile(`\s{2,}`) + relativeday = regexp.MustCompile(`(yesterday|today|tomorrow)`) + //relative1 = regexp.MustCompile(`(first|last) day of (this|next|last|previous) (week|month|year)`) + //relative2 = regexp.MustCompile(`(first|last) day of (` + "jan|january|feb|february|mar|march|apr|april|may|jun|june|jul|july|aug|august|sep|september|oct|october|nov|november|dec|december" + `)(?:\s(\d{4,4}))?`) + relative3 = regexp.MustCompile(`((?Pthis|next|last|previous) )?(` + regex_days + `)`) + //relativeval = regexp.MustCompile(`([0-9]+) (day|week|month|year)s? ago`) + ago = regexp.MustCompile(`([0-9]+) (` + regex_period + `)s? ago`) + ordinal = regexp.MustCompile("([0-9]+)(st|nd|rd|th)") + written = regexp.MustCompile(regex_numbers) + relativediff = regexp.MustCompile(`([\+\-])?([0-9]+),? ?(` + regex_period + `)s?`) + relativetime = regexp.MustCompile(`(?P\d\d?):(?P\d\d?)(:(?P\d\d?))?\s?(?Pam|pm)?\s?(?P[a-z]{3,3})?|(?Pnoon|midnight)`) + yearmonthday = regexp.MustCompile(`(?P\d{4})-(?P\d{1,2})-(?P\d{1,2})`) + relativeperiod = regexp.MustCompile(`(?Pthis|next|last) (week|month|year)`) + numberRegex = regexp.MustCompile("([0-9]+)(?:)") +) + +// http://golang.org/src/pkg/time/format.go?s=12686:12728#L404 + +// Timezone implementation +// https://groups.google.com/forum/#!topic/golang-nuts/XEVN4QwTvHw +// http://en.wikipedia.org/wiki/Zone.tab + +// Support ISO8601 Duration Parsing? +// http://en.wikipedia.org/wiki/ISO_8601 + +// Differences +// Months are NOT zero-index, MOmentJS they are +// Weeks are 0 indexed +// -- Sunday being the last day of the week ISO-8601 - is that diff from Moment? +// From/FromNow Return a Diff object rather than strings + +// Support for locale and languages with English as default + +// Support for strftime +// https://github.com/benjaminoakes/moment-strftime +// Format: https://php.net/strftime + +type Moment struct { + time time.Time + + Parser +} + +type Parser interface { + Convert(string) string +} + +func New() *Moment { + m := &Moment{time.Now(), new(MomentParser)} + + return m +} + +func NewMoment(t time.Time) *Moment { + m := &Moment{t, new(MomentParser)} + + return m +} + +func (m *Moment) GetTime() time.Time { + return m.time +} + +func (m *Moment) Now() *Moment { + m.time = time.Now().In(m.GetTime().Location()) + + return m +} + +func (m *Moment) Moment(layout string, datetime string) *Moment { + return m.MomentGo(m.Convert(layout), datetime) +} + +func (m *Moment) MomentGo(layout string, datetime string) *Moment { + time, _ := time.Parse(layout, datetime) + + m.time = time + + return m +} + +// This method is nowhere near done - requires lots of work. +func (m *Moment) Strtotime(str string) *Moment { + str = strings.ToLower(strings.TrimSpace(str)) + str = compiled.ReplaceAllString(str, " ") + + // Replace written numbers (i.e. nine, ten) with actual numbers (9, 10) + str = written.ReplaceAllStringFunc(str, func(n string) string { + switch n { + case "one": + return "1" + case "two": + return "2" + case "three": + return "3" + case "four": + return "4" + case "five": + return "5" + case "six": + return "6" + case "seven": + return "7" + case "eight": + return "8" + case "nine": + return "9" + case "ten": + return "10" + } + + return "" + }) + + // Remove ordinal suffixes st, nd, rd, th + str = ordinal.ReplaceAllString(str, "$1") + + // Replace n second|minute|hour... ago to -n second|minute|hour... to consolidate parsing + str = ago.ReplaceAllString(str, "-$1 $2") + + // Look for relative +1day, +3 days 5 hours 15 minutes + if match := relativediff.FindAllStringSubmatch(str, -1); match != nil { + for i := range match { + switch match[i][1] { + case "-": + number, _ := strconv.Atoi(match[i][2]) + m.Subtract(match[i][3], number) + default: + number, _ := strconv.Atoi(match[i][2]) + m.Add(match[i][3], number) + } + + str = strings.Replace(str, match[i][0], "", 1) + } + } + + // Remove any words that aren't needed for consistency + str = strings.Replace(str, " at ", " ", -1) + str = strings.Replace(str, " on ", " ", -1) + + // Support for interchangeable previous/last + str = strings.Replace(str, "previous", "last", -1) + + var dateDefaults = map[string]int{ + "year": 0, + "month": 0, + "day": 0, + } + + dateMatches := dateDefaults + if match := yearmonthday.FindStringSubmatch(str); match != nil { + for i, name := range yearmonthday.SubexpNames() { + if i == 0 { + str = strings.Replace(str, match[i], "", 1) + continue + } + + if match[i] == "" { + continue + } + + if name == "year" || name == "month" || name == "day" { + dateMatches[name], _ = strconv.Atoi(match[i]) + } + + } + + defer m.strtotimeSetDate(dateMatches) + if str == "" { + // Nothing left to parse + return m + } + + str = strings.TrimSpace(str) + } + + // Try to parse out time from the string + var timeDefaults = map[string]int{ + "hour": 0, + "minutes": 0, + "seconds": 0, + } + + timeMatches := timeDefaults + var zone string + if match := relativetime.FindStringSubmatch(str); match != nil { + for i, name := range relativetime.SubexpNames() { + if i == 0 { + str = strings.Replace(str, match[i], "", 1) + continue + } + + if match[i] == "" { + continue + } + + // Midnight is all zero's so nothing to do + if name == "relativetime" && match[i] == "noon" { + timeDefaults["hour"] = 12 + } + + if name == "zone" { + zone = match[i] + } + + if name == "meridiem" && match[i] == "pm" && timeMatches["hour"] < 12 { + timeMatches["hour"] += 12 + } + + if name == "hour" || name == "minutes" || name == "seconds" { + timeMatches[name], _ = strconv.Atoi(match[i]) + } + } + + // Processing time is always last + defer m.strtotimeSetTime(timeMatches, zone) + + if str == "" { + // Nothing left to parse + return m + } + + str = strings.TrimSpace(str) + } + + // m.StartOf("month", "January").GoTo(time.Sunday) + + if match := relativeperiod.FindStringSubmatch(str); match != nil { + period := match[1] + unit := match[2] + + str = strings.Replace(str, match[0], "", 1) + + switch period { + case "next": + if unit == "year" { + m.AddYears(1) + } + if unit == "month" { + m.AddMonths(1) + } + if unit == "week" { + m.AddWeeks(1) + } + case "last": + if unit == "year" { + m.SubYears(1) + } + if unit == "month" { + m.SubMonths(1) + } + if unit == "week" { + m.SubWeeks(1) + } + } + + str = strings.TrimSpace(str) + + // first := regexp.MustCompile("(?Pfirst|last)?") + } + + /* + + relativeday: first day of + relativeperiod: this, last, next + relativeperiodunit week, month, year + day: monday, tues, wednesday + month: january, feb + + + YYYY-MM-DD (HH:MM:SS MST)? + MM-DD-YYYY (HH:MM:SS MST) + 10 September 2015 (HH:MM:SS MST)? + September, 10 2015 (HH:MM:SS MST)? + September 10 2015 (HH:MM:SS M + + this year 2014 + next year 2015 + last year 2013 + + this month April + next month May + last month Mar + + first day of April + last day of April + + + DONE 3PM + DONE 3:00 PM + DONE 3:00:05 MST + 3PM on January 5th + January 5th at 3:00PM + first saturday _of_ next month + first saturday _of_ next month _at_ 3:00PM + saturday of next week + saturday of last week + saturday next week + monday next week + saturday of this week + saturday at 3:00pm + saturday at 4:00PM + saturday at midn + first of january + last of january + january of next year + first day of january + last day of january + first day of February + + DONE midnight + DONE noon + DONE 3 days ago + DONE ten days + DONE 9 weeks ago // Convert to -9 weeks + DONE -9 weeks + + */ + + if match := relativeday.FindStringSubmatch(str); match != nil && len(match) > 1 { + day := match[1] + + str = strings.Replace(str, match[0], "", 1) + + switch day { + case "today": + m.Today() + case "yesterday": + m.Yesterday() + case "tomorrow": + m.Tomorrow() + } + } + + if match := relative3.FindStringSubmatch(str); match != nil { + var when string + for i, name := range relative3.SubexpNames() { + if name == "relperiod" { + when = match[i] + } + } + weekDay := match[len(match)-1] + + str = strings.Replace(str, match[0], "", 1) + + wDay, err := ParseWeekDay(weekDay) + if err == nil { + switch when { + case "last", "previous": + m.GoBackTo(wDay, true) + + case "next": + m.GoTo(wDay, true) + + case "", "this": + m.GoTo(wDay, false) + default: + m.GoTo(wDay, false) + } + } + } + + /* + + + yesterday 11:00 + today 11:00 + tomorrow 11:00 + midnight + noon + DONE +n (second|day|week|month|year)s? + DONE -n (second|day|week|month|year)s? + next (monday|tuesday|wednesday|thursday|friday|saturday|sunday) 11:00 + last (monday|tuesday|wednesday|thursday|friday|saturday|sunday) 11:00 + next (month|year) + last (month|year) + first day of (january|february|march...|december) 2014 + last day of (january|february|march...|december) 2014 + first day of (this|next|last) (week|month|year) + last day of (this|next|last) (week|month|year) + first (monday|tuesday|wednesday) of July 2014 + last (monday|tuesday|wednesday) of July 2014 + n (day|week|month|year)s? ago + Monday|Tuesday|Wednesday|Thursday|Friday + Monday (last|this|next) week + + DONE +1 week 2 days 3 hours 4 minutes 5 seconds + */ + + return m +} + +// @todo deal with timezone +func (m *Moment) strtotimeSetTime(time map[string]int, zone string) { + m.SetHour(time["hour"]).SetMinute(time["minutes"]).SetSecond(time["seconds"]) +} + +func (m *Moment) strtotimeSetDate(date map[string]int) { + m.SetYear(date["year"]).SetMonth(time.Month(date["month"])).SetDay(date["day"]) +} + +func (m Moment) Clone() *Moment { + copy := New() + copy.time = m.GetTime() + + return copy +} + +/** + * Getters + * + */ +// https://groups.google.com/forum/#!topic/golang-nuts/pret7hjDc70 +func (m *Moment) Millisecond() { + +} + +func (m *Moment) Second() int { + return m.GetTime().Second() +} + +func (m *Moment) Minute() int { + return m.GetTime().Minute() +} + +func (m *Moment) Hour() int { + return m.GetTime().Hour() +} + +// Day of month +func (m *Moment) Date() int { + return m.DayOfMonth() +} + +// Carbon convenience method +func (m *Moment) DayOfMonth() int { + return m.GetTime().Day() +} + +// Day of week (int or string) +func (m *Moment) Day() time.Weekday { + return m.DayOfWeek() +} + +// Carbon convenience method +func (m *Moment) DayOfWeek() time.Weekday { + return m.GetTime().Weekday() +} + +func (m *Moment) DayOfWeekISO() int { + day := m.GetTime().Weekday() + + if day == time.Sunday { + return 7 + } + + return int(day) +} + +func (m *Moment) DayOfYear() int { + return m.GetTime().YearDay() +} + +// Day of Year with zero padding +func (m *Moment) dayOfYearZero() string { + day := m.GetTime().YearDay() + + if day < 10 { + return fmt.Sprintf("00%d", day) + } + + if day < 100 { + return fmt.Sprintf("0%d", day) + } + + return fmt.Sprintf("%d", day) +} + +// todo panic? +func (m *Moment) Weekday(index int) string { + if index > 6 { + panic("Weekday index must be between 0 and 6") + } + + return time.Weekday(index).String() +} + +func (m *Moment) Week() int { + return 0 +} + +// Is this the week number where as ISOWeekYear is the number of weeks in the year? +// @see http://stackoverflow.com/questions/18478741/get-weeks-in-year +func (m *Moment) ISOWeek() int { + _, week := m.GetTime().ISOWeek() + + return week +} + +// @todo Consider language support +func (m *Moment) Month() time.Month { + return m.GetTime().Month() +} + +func (m *Moment) Quarter() (quarter int) { + quarter = 4 + + switch m.Month() { + case time.January, time.February, time.March: + quarter = 1 + case time.April, time.May, time.June: + quarter = 2 + case time.July, time.August, time.September: + quarter = 3 + } + + return +} + +func (m *Moment) Year() int { + return m.GetTime().Year() +} + +// @see comments for ISOWeek +func (m *Moment) WeekYear() { + +} + +func (m *Moment) ISOWeekYear() { + +} + +/** + * Manipulate + * + */ +func (m *Moment) Add(key string, value int) *Moment { + switch key { + case "years", "year", "y": + m.AddYears(value) + case "months", "month", "M": + m.AddMonths(value) + case "weeks", "week", "w": + m.AddWeeks(value) + case "days", "day", "d": + m.AddDays(value) + case "hours", "hour", "h": + m.AddHours(value) + case "minutes", "minute", "m": + m.AddMinutes(value) + case "seconds", "second", "s": + m.AddSeconds(value) + case "milliseconds", "millisecond", "ms": + + } + + return m +} + +// Carbon +func (m *Moment) AddSeconds(seconds int) *Moment { + return m.addTime(time.Second * time.Duration(seconds)) +} + +// Carbon +func (m *Moment) AddMinutes(minutes int) *Moment { + return m.addTime(time.Minute * time.Duration(minutes)) +} + +// Carbon +func (m *Moment) AddHours(hours int) *Moment { + return m.addTime(time.Hour * time.Duration(hours)) +} + +// Carbon +func (m *Moment) AddDay() *Moment { + return m.AddDays(1) +} + +// Carbon +func (m *Moment) AddDays(days int) *Moment { + m.time = m.GetTime().AddDate(0, 0, days) + + return m +} + +// Carbon +func (m *Moment) AddWeeks(weeks int) *Moment { + return m.AddDays(weeks * 7) +} + +// Carbon +func (m *Moment) AddMonths(months int) *Moment { + m.time = m.GetTime().AddDate(0, months, 0) + + return m +} + +// Carbon +func (m *Moment) AddYears(years int) *Moment { + m.time = m.GetTime().AddDate(years, 0, 0) + + return m +} + +func (m *Moment) addTime(d time.Duration) *Moment { + m.time = m.GetTime().Add(d) + + return m +} + +func (m *Moment) Subtract(key string, value int) *Moment { + switch key { + case "years", "year", "y": + m.SubYears(value) + case "months", "month", "M": + m.SubMonths(value) + case "weeks", "week", "w": + m.SubWeeks(value) + case "days", "day", "d": + m.SubDays(value) + case "hours", "hour", "h": + m.SubHours(value) + case "minutes", "minute", "m": + m.SubMinutes(value) + case "seconds", "second", "s": + m.SubSeconds(value) + case "milliseconds", "millisecond", "ms": + + } + + return m +} + +// Carbon +func (m *Moment) SubSeconds(seconds int) *Moment { + return m.addTime(time.Second * time.Duration(seconds*-1)) +} + +// Carbon +func (m *Moment) SubMinutes(minutes int) *Moment { + return m.addTime(time.Minute * time.Duration(minutes*-1)) +} + +// Carbon +func (m *Moment) SubHours(hours int) *Moment { + return m.addTime(time.Hour * time.Duration(hours*-1)) +} + +// Carbon +func (m *Moment) SubDay() *Moment { + return m.SubDays(1) +} + +// Carbon +func (m *Moment) SubDays(days int) *Moment { + return m.AddDays(days * -1) +} + +func (m *Moment) SubWeeks(weeks int) *Moment { + return m.SubDays(weeks * 7) +} + +// Carbon +func (m *Moment) SubMonths(months int) *Moment { + return m.AddMonths(months * -1) +} + +// Carbon +func (m *Moment) SubYears(years int) *Moment { + return m.AddYears(years * -1) +} + +// Carbon +func (m *Moment) Today() *Moment { + return m.Now() +} + +// Carbon +func (m *Moment) Tomorrow() *Moment { + return m.Today().AddDay() +} + +// Carbon +func (m *Moment) Yesterday() *Moment { + return m.Today().SubDay() +} + +func (m *Moment) StartOf(key string) *Moment { + switch key { + case "year", "y": + m.StartOfYear() + case "month", "M": + m.StartOfMonth() + case "week", "w": + m.StartOfWeek() + case "day", "d": + m.StartOfDay() + case "hour", "h": + if m.Minute() > 0 { + m.SubMinutes(m.Minute()) + } + + if m.Second() > 0 { + m.SubSeconds(m.Second()) + } + case "minute", "m": + if m.Second() > 0 { + m.SubSeconds(m.Second()) + } + case "second", "s": + + } + + return m +} + +// Carbon +func (m *Moment) StartOfDay() *Moment { + if m.Hour() > 0 { + _, timeOffset := m.GetTime().Zone() + m.SubHours(m.Hour()) + + _, newTimeOffset := m.GetTime().Zone() + diffOffset := timeOffset - newTimeOffset + if diffOffset != 0 { + // we need to adjust for time zone difference + m.AddSeconds(diffOffset) + } + } + + return m.StartOf("hour") +} + +// @todo ISO8601 Starts on Monday +func (m *Moment) StartOfWeek() *Moment { + return m.GoBackTo(time.Monday, false).StartOfDay() +} + +// Carbon +func (m *Moment) StartOfMonth() *Moment { + return m.SetDay(1).StartOfDay() +} + +// Carbon +func (m *Moment) StartOfYear() *Moment { + return m.SetMonth(time.January).SetDay(1).StartOfDay() +} + +// Carbon +func (m *Moment) EndOf(key string) *Moment { + switch key { + case "year", "y": + m.EndOfYear() + case "month", "M": + m.EndOfMonth() + case "week", "w": + m.EndOfWeek() + case "day", "d": + m.EndOfDay() + case "hour", "h": + if m.Minute() < 59 { + m.AddMinutes(59 - m.Minute()) + } + case "minute", "m": + if m.Second() < 59 { + m.AddSeconds(59 - m.Second()) + } + case "second", "s": + + } + + return m +} + +// Carbon +func (m *Moment) EndOfDay() *Moment { + if m.Hour() < 23 { + _, timeOffset := m.GetTime().Zone() + m.AddHours(23 - m.Hour()) + + _, newTimeOffset := m.GetTime().Zone() + diffOffset := newTimeOffset - timeOffset + if diffOffset != 0 { + // we need to adjust for time zone difference + m.SubSeconds(diffOffset) + } + } + + return m.EndOf("hour") +} + +// @todo ISO8601 Ends on Sunday +func (m *Moment) EndOfWeek() *Moment { + return m.GoTo(time.Sunday, false).EndOfDay() +} + +// Carbon +func (m *Moment) EndOfMonth() *Moment { + return m.SetDay(m.DaysInMonth()).EndOfDay() +} + +// Carbon +func (m *Moment) EndOfYear() *Moment { + return m.GoToMonth(time.December, false).EndOfMonth() +} + +// Custom +func (m *Moment) GoTo(day time.Weekday, next bool) *Moment { + if m.Day() == day { + if !next { + return m + } else { + m.AddDay() + } + } + + var diff int + if diff = int(day) - int(m.Day()); diff > 0 { + return m.AddDays(diff) + } + + return m.AddDays(7 + diff) +} + +// Custom +func (m *Moment) GoBackTo(day time.Weekday, previous bool) *Moment { + if m.Day() == day { + if !previous { + return m + } else { + m.SubDay() + } + } + + var diff int + if diff = int(day) - int(m.Day()); diff > 0 { + return m.SubDays(7 - diff) + } + + return m.SubDays(diff * -1) +} + +// Custom +func (m *Moment) GoToMonth(month time.Month, next bool) *Moment { + if m.Month() == month { + if !next { + return m + } else { + m.AddMonths(1) + } + } + + var diff int + if diff = int(month - m.Month()); diff > 0 { + return m.AddMonths(diff) + } + + return m.AddMonths(12 + diff) +} + +// Custom +func (m *Moment) GoBackToMonth(month time.Month, previous bool) *Moment { + if m.Month() == month { + if !previous { + return m + } else { + m.SubMonths(1) + } + } + + var diff int + if diff = int(month) - int(m.Month()); diff > 0 { + return m.SubMonths(12 - diff) + } + + return m.SubMonths(diff * -1) +} + +func (m *Moment) SetSecond(seconds int) *Moment { + if seconds >= 0 && seconds <= 60 { + return m.AddSeconds(seconds - m.Second()) + } + + return m +} + +func (m *Moment) SetMinute(minute int) *Moment { + if minute >= 0 && minute <= 60 { + return m.AddMinutes(minute - m.Minute()) + } + + return m +} + +func (m *Moment) SetHour(hour int) *Moment { + if hour >= 0 && hour <= 23 { + return m.AddHours(hour - m.Hour()) + } + + return m +} + +// Custom +func (m *Moment) SetDay(day int) *Moment { + if m.DayOfMonth() == day { + return m + } + + return m.AddDays(day - m.DayOfMonth()) +} + +// Custom +func (m *Moment) SetMonth(month time.Month) *Moment { + if m.Month() > month { + return m.GoBackToMonth(month, false) + } + + return m.GoToMonth(month, false) +} + +// Custom +func (m *Moment) SetYear(year int) *Moment { + if m.Year() == year { + return m + } + + return m.AddYears(year - m.Year()) +} + +// UTC Mode. @see http://momentjs.com/docs/#/parsing/utc/ +func (m *Moment) UTC() *Moment { + return m +} + +// http://momentjs.com/docs/#/manipulating/timezone-offset/ +func (m *Moment) Zone() int { + _, offset := m.GetTime().Zone() + + return (offset / 60) * -1 +} + +/** + * Display + * + */ +func (m *Moment) Format(layout string) string { + format := m.Convert(layout) + hasCustom := false + + formatted := m.GetTime().Format(format) + + if strings.Contains(formatted, "", fmt.Sprintf("%d", m.Unix()), -1) + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", m.ISOWeek()), -1) + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", m.DayOfWeek()), -1) + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", m.DayOfWeekISO()), -1) + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", m.DayOfYear()), -1) + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", m.Quarter()), -1) + formatted = strings.Replace(formatted, "", m.dayOfYearZero(), -1) + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", m.Hour()), -1) + } + + // This has to happen after time.Format + if hasCustom && strings.Contains(formatted, "") { + formatted = numberRegex.ReplaceAllStringFunc(formatted, func(n string) string { + ordinal, _ := strconv.Atoi(strings.Replace(n, "", "", 1)) + return m.ordinal(ordinal) + }) + } + + return formatted +} + +func (m *Moment) FormatGo(layout string) string { + return m.GetTime().Format(layout) +} + +// From Dmytro Shteflyuk @https://groups.google.com/forum/#!topic/golang-nuts/l8NhI74jl-4 +func (m *Moment) ordinal(x int) string { + suffix := "th" + switch x % 10 { + case 1: + if x%100 != 11 { + suffix = "st" + } + case 2: + if x%100 != 12 { + suffix = "nd" + } + case 3: + if x%100 != 13 { + suffix = "rd" + } + } + + return strconv.Itoa(x) + suffix +} + +func (m *Moment) FromNow() Diff { + now := new(Moment) + now.Now() + + return m.From(now) +} + +// Carbon +func (m *Moment) From(f *Moment) Diff { + return m.GetDiff(f) +} + +/** + * Difference + * + */ +func (m *Moment) Diff(t *Moment, unit string) int { + diff := m.GetDiff(t) + + switch unit { + case "years": + return diff.InYears() + case "months": + return diff.InMonths() + case "weeks": + return diff.InWeeks() + case "days": + return diff.InDays() + case "hours": + return diff.InHours() + case "minutes": + return diff.InMinutes() + case "seconds": + return diff.InSeconds() + } + + return 0 +} + +// Custom +func (m *Moment) GetDiff(t *Moment) Diff { + duration := m.GetTime().Sub(t.GetTime()) + + return Diff{duration} +} + +/** + * Display + * + */ +func (m *Moment) ValueOf() int64 { + return m.Unix() * 1000 +} + +func (m *Moment) Unix() int64 { + return m.GetTime().Unix() +} + +func (m *Moment) DaysInMonth() int { + days := 31 + switch m.Month() { + case time.April, time.June, time.September, time.November: + days = 30 + break + case time.February: + days = 28 + if m.IsLeapYear() { + days = 29 + } + break + } + + return days +} + +// or ToSlice? +func (m *Moment) ToArray() []int { + return []int{ + m.Year(), + int(m.Month()), + m.DayOfMonth(), + m.Hour(), + m.Minute(), + m.Second(), + } +} + +/** + * Query + * + */ +func (m *Moment) IsBefore(t Moment) bool { + return m.GetTime().Before(t.GetTime()) +} + +func (m *Moment) IsSame(t *Moment, layout string) bool { + return m.Format(layout) == t.Format(layout) +} + +func (m *Moment) IsAfter(t Moment) bool { + return m.GetTime().After(t.GetTime()) +} + +// Carbon +func (m *Moment) IsToday() bool { + today := m.Clone().Today() + + return m.Year() == today.Year() && m.Month() == today.Month() && m.Day() == today.Day() +} + +// Carbon +func (m *Moment) IsTomorrow() bool { + tomorrow := m.Clone().Tomorrow() + + return m.Year() == tomorrow.Year() && m.Month() == tomorrow.Month() && m.Day() == tomorrow.Day() +} + +// Carbon +func (m *Moment) IsYesterday() bool { + yesterday := m.Clone().Yesterday() + + return m.Year() == yesterday.Year() && m.Month() == yesterday.Month() && m.Day() == yesterday.Day() +} + +// Carbon +func (m *Moment) IsWeekday() bool { + return !m.IsWeekend() +} + +// Carbon +func (m *Moment) IsWeekend() bool { + return m.DayOfWeek() == time.Sunday || m.DayOfWeek() == time.Saturday +} + +func (m *Moment) IsLeapYear() bool { + year := m.Year() + return year%4 == 0 && (year%100 != 0 || year%400 == 0) +} + +// Custom +func (m *Moment) Range(start Moment, end Moment) bool { + return m.IsAfter(start) && m.IsBefore(end) +} diff --git a/vendor/github.com/leibowitz/moment/moment_parser.go b/vendor/github.com/leibowitz/moment/moment_parser.go new file mode 100644 index 00000000000..3361cfba113 --- /dev/null +++ b/vendor/github.com/leibowitz/moment/moment_parser.go @@ -0,0 +1,100 @@ +package moment + +import ( + "regexp" + "strings" +) + +type MomentParser struct{} + +var ( + date_pattern = regexp.MustCompile("(LT|LL?L?L?|l{1,4}|Mo|MM?M?M?|Do|DDDo|DD?D?D?|ddd?d?|do?|w[o|w]?|W[o|W]?|YYYYY|YYYY|YY|gg(ggg?)?|GG(GGG?)?|e|E|a|A|hh?|HH?|mm?|ss?|SS?S?|X|zz?|ZZ?|Q)") +) + +/* + + S (makes any number before it ordinal) + + stdDayOfYear 1,2,365 + + stdDayOfYearZero 001, 002, 365 + + stdDayOfWeek w 0, 1, 2 numeric day of the week (0 = sunday) + + stdDayOfWeekISO N 1 = Monday + + stdWeekOfYear W Iso week number of year + + stdUnix U + + stdQuarter +*/ + +// Thanks to https://github.com/fightbulc/moment.php for replacement keys and regex +var moment_replacements = map[string]string{ + "M": "1", // stdNumMonth 1 2 ... 11 12 + "Mo": "1", // stdNumMonth 1st 2nd ... 11th 12th + "MM": "01", // stdZeroMonth 01 02 ... 11 12 + "MMM": "Jan", // stdMonth Jan Feb ... Nov Dec + "MMMM": "January", // stdLongMonth January February ... November December + "D": "2", // stdDay 1 2 ... 30 30 + "Do": "2", // stdDay 1st 2nd ... 30th 31st @todo support st nd th etch + "DD": "02", // stdZeroDay 01 02 ... 30 31 + "DDD": "", // Day of the year 1 2 ... 364 365 + "DDDo": "", // Day of the year 1st 2nd ... 364th 365th + "DDDD": "", // Day of the year 001 002 ... 364 365 @todo**** + "d": "", // Numeric representation of day of the week 0 1 ... 5 6 + "do": "", // 0th 1st ... 5th 6th + "dd": "Mon", // ***Su Mo ... Fr Sa @todo + "ddd": "Mon", // Sun Mon ... Fri Sat + "dddd": "Monday", // stdLongWeekDay Sunday Monday ... Friday Saturday + "e": "", // Numeric representation of day of the week 0 1 ... 5 6 @todo + "E": "", // ISO-8601 numeric representation of the day of the week (added in PHP 5.1.0) 1 2 ... 6 7 @todo + "w": "", // 1 2 ... 52 53 + "wo": "", // 1st 2nd ... 52nd 53rd + "ww": "", // ***01 02 ... 52 53 @todo + "W": "", // 1 2 ... 52 53 + "Wo": "", // 1st 2nd ... 52nd 53rd + "WW": "", // ***01 02 ... 52 53 @todo + "YY": "06", // stdYear 70 71 ... 29 30 + "YYYY": "2006", // stdLongYear 1970 1971 ... 2029 2030 + // "gg" : "o", // ISO-8601 year number 70 71 ... 29 30 @todo + // "gggg" : "o", // ***1970 1971 ... 2029 2030 @todo + // "GG" : "o", //70 71 ... 29 30 @todo + // "GGGG" : "o", // ***1970 1971 ... 2029 2030 @todo + "Q": "", + "A": "PM", // stdPM AM PM + "a": "pm", // stdpm am pm + "H": "", // stdHour 0 1 ... 22 23 + "HH": "15", // 00 01 ... 22 23 + "h": "3", // stdHour12 1 2 ... 11 12 + "hh": "03", // stdZeroHour12 01 02 ... 11 12 + "m": "4", // stdZeroMinute 0 1 ... 58 59 + "mm": "04", // stdZeroMinute 00 01 ... 58 59 + "s": "5", // stdSecond 0 1 ... 58 59 + "ss": "05", // stdZeroSecond ***00 01 ... 58 59 + // "S" : "", //0 1 ... 8 9 + // "SS" : "", //0 1 ... 98 99 + // "SSS" : "", //0 1 ... 998 999 + "z": "MST", //EST CST ... MST PST + "zz": "MST", //EST CST ... MST PST + "Z": "Z07:00", // stdNumColonTZ -07:00 -06:00 ... +06:00 +07:00 + "ZZ": "-0700", // stdNumTZ -0700 -0600 ... +0600 +0700 + "X": "", // Seconds since unix epoch 1360013296 + "LT": "3:04 PM", // 8:30 PM + "L": "01/02/2006", //09/04/1986 + "l": "1/2/2006", //9/4/1986 + "LL": "January 2 2006", //September 4th 1986 the php s flag isn't supported + "ll": "Jan 2 2006", //Sep 4 1986 + "LLL": "January 2 2006 3:04 PM", //September 4th 1986 8:30 PM @todo the php s flag isn't supported + "lll": "Jan 2 2006 3:04 PM", //Sep 4 1986 8:30 PM + "LLLL": "Monday, January 2 2006 3:04 PM", //Thursday, September 4th 1986 8:30 PM the php s flag isn't supported + "llll": "Mon, Jan 2 2006 3:04 PM", //Thu, Sep 4 1986 8:30 PM +} + +func (p *MomentParser) Convert(layout string) string { + var match [][]string + if match = date_pattern.FindAllStringSubmatch(layout, -1); match == nil { + return layout + } + + for i := range match { + if replace, ok := moment_replacements[match[i][0]]; ok { + layout = strings.Replace(layout, match[i][0], replace, 1) + } + } + + return layout +} diff --git a/vendor/github.com/leibowitz/moment/parse_day.go b/vendor/github.com/leibowitz/moment/parse_day.go new file mode 100644 index 00000000000..e8e890a462e --- /dev/null +++ b/vendor/github.com/leibowitz/moment/parse_day.go @@ -0,0 +1,32 @@ +package moment + +import ( + "fmt" + "strings" + "time" +) + +var ( + days = []time.Weekday{ + time.Sunday, + time.Monday, + time.Tuesday, + time.Wednesday, + time.Thursday, + time.Friday, + time.Saturday, + } +) + +func ParseWeekDay(day string) (time.Weekday, error) { + + day = strings.ToLower(day) + + for _, d := range days { + if day == strings.ToLower(d.String()) { + return d, nil + } + } + + return -1, fmt.Errorf("Unable to parse %s as week day", day) +} diff --git a/vendor/github.com/leibowitz/moment/strftime_parser.go b/vendor/github.com/leibowitz/moment/strftime_parser.go new file mode 100644 index 00000000000..3c024376535 --- /dev/null +++ b/vendor/github.com/leibowitz/moment/strftime_parser.go @@ -0,0 +1,68 @@ +package moment + +import ( + "regexp" + "strings" +) + +type StrftimeParser struct{} + +var ( + replacements_pattern = regexp.MustCompile("%[mbhBedjwuaAVgyGYpPkHlIMSZzsTrRTDFXx]") +) + +// Not implemented +// U +// C + +var strftime_replacements = map[string]string{ + "%m": "01", // stdZeroMonth 01 02 ... 11 12 + "%b": "Jan", // stdMonth Jan Feb ... Nov Dec + "%h": "Jan", + "%B": "January", // stdLongMonth January February ... November December + "%e": "2", // stdDay 1 2 ... 30 30 + "%d": "02", // stdZeroDay 01 02 ... 30 31 + "%j": "", // Day of the year ***001 002 ... 364 365 @todo**** + "%w": "", // Numeric representation of day of the week 0 1 ... 5 6 + "%u": "", // ISO-8601 numeric representation of the day of the week (added in PHP 5.1.0) 1 2 ... 6 7 @todo + "%a": "Mon", // Sun Mon ... Fri Sat + "%A": "Monday", // stdLongWeekDay Sunday Monday ... Friday Saturday + "%V": "", // ***01 02 ... 52 53 @todo begin with zeros + "%g": "06", // stdYear 70 71 ... 29 30 + "%y": "06", + "%G": "2006", // stdLongYear 1970 1971 ... 2029 2030 + "%Y": "2006", + "%p": "PM", // stdPM AM PM + "%P": "pm", // stdpm am pm + "%k": "15", // stdHour 0 1 ... 22 23 + "%H": "15", // 00 01 ... 22 23 + "%l": "3", // stdHour12 1 2 ... 11 12 + "%I": "03", // stdZeroHour12 01 02 ... 11 12 + "%M": "04", // stdZeroMinute 00 01 ... 58 59 + "%S": "05", // stdZeroSecond ***00 01 ... 58 59 + "%Z": "MST", //EST CST ... MST PST + "%z": "-0700", // stdNumTZ -0700 -0600 ... +0600 +0700 + "%s": "", // Seconds since unix epoch 1360013296 + "%r": "03:04:05 PM", + "%R": "15:04", + "%T": "15:04:05", + "%D": "01/02/06", + "%F": "2006-01-02", + "%X": "15:04:05", + "%x": "01/02/06", +} + +func (p *StrftimeParser) Convert(layout string) string { + var match [][]string + if match = replacements_pattern.FindAllStringSubmatch(layout, -1); match == nil { + return layout + } + + for i := range match { + if replace, ok := strftime_replacements[match[i][0]]; ok { + layout = strings.Replace(layout, match[i][0], replace, 1) + } + } + + return layout +} From 63a200686e065a79fdd7ade563fd942236c4feda Mon Sep 17 00:00:00 2001 From: wph95 Date: Mon, 26 Mar 2018 19:48:57 +0800 Subject: [PATCH 05/22] - pipeline aggs support - add some test --- pkg/tsdb/elasticsearch/elasticsearch.go | 42 ++- pkg/tsdb/elasticsearch/model_parser.go | 81 ++---- pkg/tsdb/elasticsearch/models.go | 21 +- pkg/tsdb/elasticsearch/query.go | 182 +++++++----- pkg/tsdb/elasticsearch/query_def.go | 18 ++ pkg/tsdb/elasticsearch/query_test.go | 331 ++++++++++++++++++++++ pkg/tsdb/elasticsearch/response_parser.go | 34 ++- 7 files changed, 557 insertions(+), 152 deletions(-) create mode 100644 pkg/tsdb/elasticsearch/query_test.go diff --git a/pkg/tsdb/elasticsearch/elasticsearch.go b/pkg/tsdb/elasticsearch/elasticsearch.go index 8fd82a179e8..0ce9eca0972 100644 --- a/pkg/tsdb/elasticsearch/elasticsearch.go +++ b/pkg/tsdb/elasticsearch/elasticsearch.go @@ -1,6 +1,7 @@ package elasticsearch import ( + "bytes" "context" "encoding/json" "errors" @@ -18,7 +19,8 @@ import ( ) type ElasticsearchExecutor struct { - Transport *http.Transport + QueryParser *ElasticSearchQueryParser + Transport *http.Transport } var ( @@ -47,17 +49,21 @@ func (e *ElasticsearchExecutor) Query(ctx context.Context, dsInfo *models.DataSo result := &tsdb.Response{} result.Results = make(map[string]*tsdb.QueryResult) - queryParser := ElasticSearchQueryParser{ - dsInfo, - tsdbQuery.TimeRange, - tsdbQuery.Queries, - } - - payload, targets, err := queryParser.Parse() + queries, err := e.getQuery(dsInfo, tsdbQuery) if err != nil { return nil, err } + buff := bytes.Buffer{} + for _, q := range queries { + s, err := q.Build(tsdbQuery, dsInfo) + if err != nil { + return nil, err + } + buff.WriteString(s) + } + payload := buff.String() + if setting.Env == setting.DEV { glog.Debug("Elasticsearch playload", "raw playload", payload) } @@ -96,12 +102,30 @@ func (e *ElasticsearchExecutor) Query(ctx context.Context, dsInfo *models.DataSo return nil, errors.New(res.getErrMsg()) } } - responseParser := ElasticsearchResponseParser{responses.Responses, targets} + responseParser := ElasticsearchResponseParser{responses.Responses, queries} queryRes := responseParser.getTimeSeries() result.Results["A"] = queryRes return result, nil } +func (e *ElasticsearchExecutor) getQuery(dsInfo *models.DataSource, context *tsdb.TsdbQuery) ([]*Query, error) { + queries := make([]*Query, 0) + if len(context.Queries) == 0 { + return nil, fmt.Errorf("query request contains no queries") + } + for _, v := range context.Queries { + + query, err := e.QueryParser.Parse(v.Model, dsInfo) + if err != nil { + return nil, err + } + queries = append(queries, query) + + } + return queries, nil + +} + func (e *ElasticsearchExecutor) createRequest(dsInfo *models.DataSource, query string) (*http.Request, error) { u, _ := url.Parse(dsInfo.Url) u.Path = path.Join(u.Path, "_msearch") diff --git a/pkg/tsdb/elasticsearch/model_parser.go b/pkg/tsdb/elasticsearch/model_parser.go index 7da6765e06c..0d016dc58a5 100644 --- a/pkg/tsdb/elasticsearch/model_parser.go +++ b/pkg/tsdb/elasticsearch/model_parser.go @@ -1,62 +1,45 @@ package elasticsearch import ( - "bytes" - "encoding/json" "fmt" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/tsdb" "github.com/leibowitz/moment" - "src/github.com/davecgh/go-spew/spew" - "strconv" "strings" "time" ) type ElasticSearchQueryParser struct { - DsInfo *models.DataSource - TimeRange *tsdb.TimeRange - Queries []*tsdb.Query } -func (qp *ElasticSearchQueryParser) Parse() (string, []*QueryBuilder, error) { - payload := bytes.Buffer{} - queryHeader := qp.getQueryHeader() - targets := make([]*QueryBuilder, 0) - for _, q := range qp.Queries { - timeField, err := q.Model.Get("timeField").String() - if err != nil { - return "", nil, err - } - rawQuery := q.Model.Get("query").MustString("") - bucketAggs := q.Model.Get("bucketAggs").MustArray() - metrics := q.Model.Get("metrics").MustArray() - alias := q.Model.Get("alias").MustString("") - builder := QueryBuilder{timeField, rawQuery, bucketAggs, metrics, alias} - targets = append(targets, &builder) - - query, err := builder.Build() - if err != nil { - return "", nil, err - } - queryBytes, err := json.Marshal(query) - if err != nil { - return "", nil, err - } - - payload.WriteString(queryHeader.String() + "\n") - payload.WriteString(string(queryBytes) + "\n") +func (qp *ElasticSearchQueryParser) Parse(model *simplejson.Json, dsInfo *models.DataSource) (*Query, error) { + //payload := bytes.Buffer{} + //queryHeader := qp.getQueryHeader() + timeField, err := model.Get("timeField").String() + if err != nil { + return nil, err + } + rawQuery := model.Get("query").MustString("") + bucketAggs := model.Get("bucketAggs").MustArray() + metrics := model.Get("metrics").MustArray() + alias := model.Get("alias").MustString("") + parsedInterval, err := tsdb.GetIntervalFrom(dsInfo, model, time.Millisecond) + if err != nil { + return nil, err } - p, err := qp.payloadReplace(payload.String(), qp.DsInfo.JsonData) - - return p, targets, err + return &Query{timeField, + rawQuery, + bucketAggs, + metrics, + alias, + parsedInterval}, nil } -func (qp *ElasticSearchQueryParser) getQueryHeader() *QueryHeader { +func getRequestHeader(timeRange *tsdb.TimeRange, dsInfo *models.DataSource) *QueryHeader { var header QueryHeader - esVersion := qp.DsInfo.JsonData.Get("esVersion").MustInt() + esVersion := dsInfo.JsonData.Get("esVersion").MustInt() searchType := "query_then_fetch" if esVersion < 5 { @@ -64,29 +47,13 @@ func (qp *ElasticSearchQueryParser) getQueryHeader() *QueryHeader { } header.SearchType = searchType header.IgnoreUnavailable = true - header.Index = getIndexList(qp.DsInfo.Database, qp.DsInfo.JsonData.Get("interval").MustString(""), qp.TimeRange) + header.Index = getIndexList(dsInfo.Database, dsInfo.JsonData.Get("interval").MustString(""), timeRange) if esVersion >= 56 { - header.MaxConcurrentShardRequests = qp.DsInfo.JsonData.Get("maxConcurrentShardRequests").MustInt() + header.MaxConcurrentShardRequests = dsInfo.JsonData.Get("maxConcurrentShardRequests").MustInt() } return &header } -func (qp *ElasticSearchQueryParser) payloadReplace(payload string, model *simplejson.Json) (string, error) { - parsedInterval, err := tsdb.GetIntervalFrom(qp.DsInfo, model, time.Millisecond) - if err != nil { - return "", nil - } - - interval := intervalCalculator.Calculate(qp.TimeRange, parsedInterval) - glog.Warn(spew.Sdump(interval)) - payload = strings.Replace(payload, "$timeFrom", fmt.Sprintf("%d", qp.TimeRange.GetFromAsMsEpoch()), -1) - payload = strings.Replace(payload, "$timeTo", fmt.Sprintf("%d", qp.TimeRange.GetToAsMsEpoch()), -1) - payload = strings.Replace(payload, "$interval", interval.Text, -1) - payload = strings.Replace(payload, "$__interval_ms", strconv.FormatInt(interval.Value.Nanoseconds()/int64(time.Millisecond), 10), -1) - payload = strings.Replace(payload, "$__interval", interval.Text, -1) - - return payload, nil -} func getIndexList(pattern string, interval string, timeRange *tsdb.TimeRange) string { if interval == "" { diff --git a/pkg/tsdb/elasticsearch/models.go b/pkg/tsdb/elasticsearch/models.go index d758e2159de..822df2dd4d1 100644 --- a/pkg/tsdb/elasticsearch/models.go +++ b/pkg/tsdb/elasticsearch/models.go @@ -1,25 +1,25 @@ package elasticsearch import ( - "github.com/grafana/grafana/pkg/components/simplejson" "bytes" - "fmt" "encoding/json" + "fmt" + "github.com/grafana/grafana/pkg/components/simplejson" ) type QueryHeader struct { SearchType string `json:"search_type"` IgnoreUnavailable bool `json:"ignore_unavailable"` Index interface{} `json:"index"` - MaxConcurrentShardRequests int `json:"max_concurrent_shard_requests"` + MaxConcurrentShardRequests int `json:"max_concurrent_shard_requests,omitempty"` } -func (q *QueryHeader) String() (string) { +func (q *QueryHeader) String() string { r, _ := json.Marshal(q) return string(r) } -type Query struct { +type Request struct { Query map[string]interface{} `json:"query"` Aggs Aggs `json:"aggs"` Size int `json:"size"` @@ -45,11 +45,10 @@ type FiltersAgg struct { } type TermsAggSetting struct { - Field string `json:"field"` - Size int `json:"size"` - Order map[string]interface{} `json:"order"` - MinDocCount int `json:"min_doc_count"` - Missing string `json:"missing"` + Field string `json:"field"` + Size int `json:"size"` + Order map[string]interface{} `json:"order"` + Missing string `json:"missing,omitempty"` } type TermsAgg struct { @@ -104,7 +103,7 @@ type Response struct { Aggregations map[string]interface{} `json:"aggregations"` } -func (r *Response) getErrMsg() (string) { +func (r *Response) getErrMsg() string { var msg bytes.Buffer errJson := simplejson.NewFromAny(r.Err) errType, err := errJson.Get("type").String() diff --git a/pkg/tsdb/elasticsearch/query.go b/pkg/tsdb/elasticsearch/query.go index d6d70e79a2a..51f1ebb5d7a 100644 --- a/pkg/tsdb/elasticsearch/query.go +++ b/pkg/tsdb/elasticsearch/query.go @@ -1,81 +1,103 @@ package elasticsearch import ( + "bytes" + "encoding/json" "errors" + "fmt" "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/tsdb" "strconv" + "strings" + "time" ) var rangeFilterSetting = RangeFilterSetting{Gte: "$timeFrom", - Lte: "$timeTo", + Lte: "$timeTo", Format: "epoch_millis"} -type QueryBuilder struct { - TimeField string - RawQuery string - BucketAggs []interface{} - Metrics []interface{} - Alias string +type Query struct { + TimeField string `json:"timeField"` + RawQuery string `json:"query"` + BucketAggs []interface{} `json:"bucketAggs"` + Metrics []interface{} `json:"metrics"` + Alias string `json:"Alias"` + Interval time.Duration } -func (b *QueryBuilder) Build() (Query, error) { - var err error - var res Query - res.Query = make(map[string]interface{}) - res.Size = 0 +func (q *Query) Build(queryContext *tsdb.TsdbQuery, dsInfo *models.DataSource) (string, error) { + var req Request + payload := bytes.Buffer{} - if err != nil { - return res, err - } - - boolQuery := BoolQuery{} - boolQuery.Filter = append(boolQuery.Filter, newRangeFilter(b.TimeField, rangeFilterSetting)) - boolQuery.Filter = append(boolQuery.Filter, newQueryStringFilter(true, b.RawQuery)) - res.Query["bool"] = boolQuery + req.Size = 0 + q.renderReqQuery(&req) // handle document query - if len(b.BucketAggs) == 0 { - if len(b.Metrics) > 0 { - metric := simplejson.NewFromAny(b.Metrics[0]) + if q.isRawDocumentQuery() { + return "", errors.New("alert not support Raw_Document") + } + + err := q.parseAggs(&req) + if err != nil { + return "", err + } + + reqBytes, err := json.Marshal(req) + reqHeader := getRequestHeader(queryContext.TimeRange, dsInfo) + payload.WriteString(reqHeader.String() + "\n") + payload.WriteString(string(reqBytes) + "\n") + return q.renderTemplate(payload.String(), queryContext) +} + +func (q *Query) isRawDocumentQuery() bool { + if len(q.BucketAggs) == 0 { + if len(q.Metrics) > 0 { + metric := simplejson.NewFromAny(q.Metrics[0]) if metric.Get("type").MustString("") == "raw_document" { - return res, errors.New("alert not support Raw_Document") + return true } } } - aggs, err := b.parseAggs(b.BucketAggs, b.Metrics) - res.Aggs = aggs["aggs"].(Aggs) - - return res, err + return false } -func (b *QueryBuilder) parseAggs(bucketAggs []interface{}, metrics []interface{}) (Aggs, error) { - query := make(Aggs) - nestedAggs := query - for _, aggRaw := range bucketAggs { +func (q *Query) renderReqQuery(req *Request) { + req.Query = make(map[string]interface{}) + boolQuery := BoolQuery{} + boolQuery.Filter = append(boolQuery.Filter, newRangeFilter(q.TimeField, rangeFilterSetting)) + boolQuery.Filter = append(boolQuery.Filter, newQueryStringFilter(true, q.RawQuery)) + req.Query["bool"] = boolQuery +} + +func (q *Query) parseAggs(req *Request) error { + aggs := make(Aggs) + nestedAggs := aggs + for _, aggRaw := range q.BucketAggs { esAggs := make(Aggs) aggJson := simplejson.NewFromAny(aggRaw) aggType, err := aggJson.Get("type").String() if err != nil { - return nil, err + return err } id, err := aggJson.Get("id").String() if err != nil { - return nil, err + return err } switch aggType { case "date_histogram": - esAggs["date_histogram"] = b.getDateHistogramAgg(aggJson) + esAggs["date_histogram"] = q.getDateHistogramAgg(aggJson) case "histogram": - esAggs["histogram"] = b.getHistogramAgg(aggJson) + esAggs["histogram"] = q.getHistogramAgg(aggJson) case "filters": - esAggs["filters"] = b.getFilters(aggJson) + esAggs["filters"] = q.getFilters(aggJson) case "terms": - terms := b.getTerms(aggJson) + terms := q.getTerms(aggJson) esAggs["terms"] = terms.Terms esAggs["aggs"] = terms.Aggs case "geohash_grid": - return nil, errors.New("alert not support Geo_Hash_Grid") + return errors.New("alert not support Geo_Hash_Grid") } if _, ok := nestedAggs["aggs"]; !ok { @@ -90,40 +112,51 @@ func (b *QueryBuilder) parseAggs(bucketAggs []interface{}, metrics []interface{} } nestedAggs["aggs"] = make(Aggs) - for _, metricRaw := range metrics { + for _, metricRaw := range q.Metrics { metric := make(Metric) metricJson := simplejson.NewFromAny(metricRaw) id, err := metricJson.Get("id").String() if err != nil { - return nil, err + return err } metricType, err := metricJson.Get("type").String() if err != nil { - return nil, err + return err } if metricType == "count" { continue } - // todo support pipeline Agg + settings := metricJson.Get("settings").MustMap(map[string]interface{}{}) + + if isPipelineAgg(metricType) { + pipelineAgg := metricJson.Get("pipelineAgg").MustString("") + if _, err := strconv.Atoi(pipelineAgg); err == nil { + settings["buckets_path"] = pipelineAgg + } else { + continue + } + + } else { + settings["field"] = metricJson.Get("field").MustString() + } - settings := metricJson.Get("settings").MustMap() - settings["field"] = metricJson.Get("field").MustString() metric[metricType] = settings nestedAggs["aggs"].(Aggs)[id] = metric } - return query, nil + req.Aggs = aggs["aggs"].(Aggs) + return nil } -func (b *QueryBuilder) getDateHistogramAgg(model *simplejson.Json) DateHistogramAgg { +func (q *Query) getDateHistogramAgg(model *simplejson.Json) *DateHistogramAgg { agg := &DateHistogramAgg{} settings := simplejson.NewFromAny(model.Get("settings").Interface()) interval, err := settings.Get("interval").String() if err == nil { agg.Interval = interval } - agg.Field = b.TimeField + agg.Field = q.TimeField agg.MinDocCount = settings.Get("min_doc_count").MustInt(0) agg.ExtendedBounds = ExtendedBounds{"$timeFrom", "$timeTo"} agg.Format = "epoch_millis" @@ -136,10 +169,10 @@ func (b *QueryBuilder) getDateHistogramAgg(model *simplejson.Json) DateHistogram if err == nil { agg.Missing = missing } - return *agg + return agg } -func (b *QueryBuilder) getHistogramAgg(model *simplejson.Json) HistogramAgg { +func (q *Query) getHistogramAgg(model *simplejson.Json) *HistogramAgg { agg := &HistogramAgg{} settings := simplejson.NewFromAny(model.Get("settings").Interface()) interval, err := settings.Get("interval").String() @@ -155,10 +188,10 @@ func (b *QueryBuilder) getHistogramAgg(model *simplejson.Json) HistogramAgg { if err == nil { agg.Missing = missing } - return *agg + return agg } -func (b *QueryBuilder) getFilters(model *simplejson.Json) FiltersAgg { +func (q *Query) getFilters(model *simplejson.Json) *FiltersAgg { agg := &FiltersAgg{} settings := simplejson.NewFromAny(model.Get("settings").Interface()) for filter := range settings.Get("filters").MustArray() { @@ -170,15 +203,15 @@ func (b *QueryBuilder) getFilters(model *simplejson.Json) FiltersAgg { } agg.Filter[label] = newQueryStringFilter(true, query) } - return *agg + return agg } -func (b *QueryBuilder) getTerms(model *simplejson.Json) TermsAgg { +func (q *Query) getTerms(model *simplejson.Json) *TermsAgg { agg := &TermsAgg{Aggs: make(Aggs)} settings := simplejson.NewFromAny(model.Get("settings").Interface()) agg.Terms.Field = model.Get("field").MustString() if settings == nil { - return *agg + return agg } sizeStr := settings.Get("size").MustString("") size, err := strconv.Atoi(sizeStr) @@ -186,17 +219,25 @@ func (b *QueryBuilder) getTerms(model *simplejson.Json) TermsAgg { size = 500 } agg.Terms.Size = size - orderBy := settings.Get("orderBy").MustString("") - if orderBy != "" { + orderBy, err := settings.Get("orderBy").String() + if err == nil { agg.Terms.Order = make(map[string]interface{}) agg.Terms.Order[orderBy] = settings.Get("order").MustString("") - // if orderBy is a int, means this fields is metric result value - // TODO set subAggs - } - - minDocCount, err := settings.Get("min_doc_count").Int() - if err == nil { - agg.Terms.MinDocCount = minDocCount + if _, err := strconv.Atoi(orderBy); err != nil { + for _, metricI := range q.Metrics { + metric := simplejson.NewFromAny(metricI) + metricId := metric.Get("id").MustString() + if metricId == orderBy { + subAggs := make(Aggs) + metricField := metric.Get("field").MustString() + metricType := metric.Get("type").MustString() + subAggs[metricType] = map[string]string{"field": metricField} + agg.Aggs = make(Aggs) + agg.Aggs[metricId] = subAggs + break + } + } + } } missing, err := settings.Get("missing").String() @@ -204,5 +245,16 @@ func (b *QueryBuilder) getTerms(model *simplejson.Json) TermsAgg { agg.Terms.Missing = missing } - return *agg + return agg +} + +func (q *Query) renderTemplate(payload string, queryContext *tsdb.TsdbQuery) (string, error) { + timeRange := queryContext.TimeRange + interval := intervalCalculator.Calculate(timeRange, q.Interval) + payload = strings.Replace(payload, "$timeFrom", fmt.Sprintf("%d", timeRange.GetFromAsMsEpoch()), -1) + payload = strings.Replace(payload, "$timeTo", fmt.Sprintf("%d", timeRange.GetToAsMsEpoch()), -1) + payload = strings.Replace(payload, "$interval", interval.Text, -1) + payload = strings.Replace(payload, "$__interval_ms", strconv.FormatInt(interval.Value.Nanoseconds()/int64(time.Millisecond), 10), -1) + payload = strings.Replace(payload, "$__interval", interval.Text, -1) + return payload, nil } diff --git a/pkg/tsdb/elasticsearch/query_def.go b/pkg/tsdb/elasticsearch/query_def.go index 5dc02aa359e..6f78f02f346 100644 --- a/pkg/tsdb/elasticsearch/query_def.go +++ b/pkg/tsdb/elasticsearch/query_def.go @@ -24,3 +24,21 @@ var extendedStats = map[string]string{ "std_deviation_bounds_upper": "Std Dev Upper", "std_deviation_bounds_lower": "Std Dev Lower", } + +var pipelineOptions = map[string]string{ + "moving_avg": "moving_avg", + "derivative": "derivative", +} + +func isPipelineAgg(metricType string) bool { + if _, ok := pipelineOptions[metricType]; ok { + return true + } + return false +} + +func describeMetric(metricType, field string) string { + text := metricAggType[metricType] + return text + " " + field + +} diff --git a/pkg/tsdb/elasticsearch/query_test.go b/pkg/tsdb/elasticsearch/query_test.go new file mode 100644 index 00000000000..992469175b6 --- /dev/null +++ b/pkg/tsdb/elasticsearch/query_test.go @@ -0,0 +1,331 @@ +package elasticsearch + +import ( + "encoding/json" + "fmt" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/tsdb" + . "github.com/smartystreets/goconvey/convey" + "reflect" + "strconv" + "strings" + "testing" +) + +func testElasticSearchResponse(requestJSON string, expectedElasticSearchRequestJSON string) { + var queryExpectedJSONInterface, queryJSONInterface interface{} + parser := ElasticSearchQueryParser{} + model := &Query{} + + err := json.Unmarshal([]byte(requestJSON), model) + So(err, ShouldBeNil) + jsonDate, _ := simplejson.NewJson([]byte(`{"esVersion":2}`)) + dsInfo := &models.DataSource{ + Database: "grafana-test", + JsonData: jsonDate, + } + + testTimeRange := tsdb.NewTimeRange("5m", "now") + + req, _ := simplejson.NewJson([]byte(requestJSON)) + query, err := parser.Parse(req, dsInfo) + s, err := query.Build(&tsdb.TsdbQuery{TimeRange: testTimeRange}, dsInfo) + + queryJSON := strings.Split(s, "\n")[1] + err = json.Unmarshal([]byte(queryJSON), &queryJSONInterface) + So(err, ShouldBeNil) + + expectedElasticSearchRequestJSON = strings.Replace( + expectedElasticSearchRequestJSON, + "", + strconv.FormatInt(testTimeRange.GetFromAsMsEpoch(), 10), + -1, + ) + + expectedElasticSearchRequestJSON = strings.Replace( + expectedElasticSearchRequestJSON, + "", + strconv.FormatInt(testTimeRange.GetToAsMsEpoch(), 10), + -1, + ) + + err = json.Unmarshal([]byte(expectedElasticSearchRequestJSON), &queryExpectedJSONInterface) + So(err, ShouldBeNil) + + result := reflect.DeepEqual(queryExpectedJSONInterface, queryJSONInterface) + if !result { + fmt.Printf("ERROR: %s \n != \n %s", expectedElasticSearchRequestJSON, queryJSON) + } + So(result, ShouldBeTrue) +} +func TestElasticSearchQueryBuilder(t *testing.T) { + Convey("Elasticsearch QueryBuilder query testing", t, func() { + Convey("Build test average metric with moving average", func() { + var testElasticsearchModelRequestJSON = ` + { + "bucketAggs": [ + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": 0, + "trimEdges": 0 + }, + "type": "date_histogram" + } + ], + "dsType": "elasticsearch", + "metrics": [ + { + "field": "value", + "id": "1", + "inlineScript": "_value * 2", + "meta": {}, + "settings": { + "script": { + "inline": "_value * 2" + } + }, + "type": "avg" + }, + { + "field": "1", + "id": "3", + "meta": {}, + "pipelineAgg": "1", + "settings": { + "minimize": false, + "model": "simple", + "window": 5 + }, + "type": "moving_avg" + } + ], + "query": "(test:query) AND (name:sample)", + "refId": "A", + "timeField": "timestamp" + } + ` + + var expectedElasticsearchQueryJSON = ` + { + "size": 0, + "query": { + "bool": { + "filter": [ + { + "range": { + "timestamp": { + "gte": "", + "lte": "", + "format": "epoch_millis" + } + } + }, + { + "query_string": { + "analyze_wildcard": true, + "query": "(test:query) AND (name:sample)" + } + } + ] + } + }, + "aggs": { + "2": { + "date_histogram": { + "interval": "200ms", + "field": "timestamp", + "min_doc_count": 0, + "extended_bounds": { + "min": "", + "max": "" + }, + "format": "epoch_millis" + }, + "aggs": { + "1": { + "avg": { + "field": "value", + "script": { + "inline": "_value * 2" + } + } + }, + "3": { + "moving_avg": { + "buckets_path": "1", + "window": 5, + "model": "simple", + "minimize": false + } + } + } + } + } + }` + + testElasticSearchResponse(testElasticsearchModelRequestJSON, expectedElasticsearchQueryJSON) + }) + Convey("Test Wildcards and Quotes", func() { + testElasticsearchModelRequestJSON := ` + { + "alias": "New", + "bucketAggs": [ + { + "field": "timestamp", + "id": "2", + "type": "date_histogram" + } + ], + "dsType": "elasticsearch", + "metrics": [ + { + "type": "sum", + "field": "value", + "id": "1" + } + ], + "query": "scope:$location.leagueconnect.api AND name:*CreateRegistration AND name:\"*.201-responses.rate\"", + "refId": "A", + "timeField": "timestamp" + }` + + expectedElasticsearchQueryJSON := ` + { + "size": 0, + "query": { + "bool": { + "filter": [ + { + "range": { + "timestamp": { + "gte": "", + "lte": "", + "format": "epoch_millis" + } + } + }, + { + "query_string": { + "analyze_wildcard": true, + "query": "scope:$location.leagueconnect.api AND name:*CreateRegistration AND name:\"*.201-responses.rate\"" + } + } + ] + } + }, + "aggs": { + "2": { + "aggs": { + "1": { + "sum": { + "field": "value" + } + } + }, + "date_histogram": { + "extended_bounds": { + "max": "", + "min": "" + }, + "field": "timestamp", + "format": "epoch_millis", + "min_doc_count": 0 + } + } + } + }` + + testElasticSearchResponse(testElasticsearchModelRequestJSON, expectedElasticsearchQueryJSON) + }) + Convey("Test Term Aggregates", func() { + testElasticsearchModelRequestJSON := ` + { + "bucketAggs": [{ + "field": "name_raw", + "id": "4", + "settings": { + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "1m", + "min_doc_count": 0, + "trimEdges": 0 + }, + "type": "date_histogram" + }], + "dsType": "elasticsearch", + "filters": [{ + "boolOp": "AND", + "not": false, + "type": "rfc190Scope", + "value": "*.hmp.metricsd" + }, { + "boolOp": "AND", + "not": false, + "type": "name_raw", + "value": "builtin.general.*_instance_count" + }], + "metricObject": {}, + "metrics": [{ + "field": "value", + "id": "1", + "meta": {}, + "options": {}, + "settings": {}, + "type": "sum" + }], + "mode": 0, + "numToGraph": 10, + "prependHostName": false, + "query": "(scope:*.hmp.metricsd) AND (name_raw:builtin.general.*_instance_count)", + "refId": "A", + "regexAlias": false, + "selectedApplication": "", + "selectedHost": "", + "selectedLocation": "", + "timeField": "timestamp", + "useFullHostName": "", + "useQuery": false + }` + + expectedElasticsearchQueryJSON := ` + { + "size": 0, + "query": { + "bool": { + "filter": [ + { + "range": { + "timestamp": { + "gte": "", + "lte": "", + "format": "epoch_millis" + } + } + }, + { + "query_string": { + "analyze_wildcard": true, + "query": "(scope:*.hmp.metricsd) AND (name_raw:builtin.general.*_instance_count)" + } + } + ] + } + }, + "aggs": {"4":{"aggs":{"2":{"aggs":{"1":{"sum":{"field":"value"}}},"date_histogram":{"extended_bounds":{"max":"","min":""},"field":"timestamp","format":"epoch_millis","interval":"1m","min_doc_count":0}}},"terms":{"field":"name_raw","order":{"_term":"desc"},"size":10}}} + }` + + testElasticSearchResponse(testElasticsearchModelRequestJSON, expectedElasticsearchQueryJSON) + }) + }) +} diff --git a/pkg/tsdb/elasticsearch/response_parser.go b/pkg/tsdb/elasticsearch/response_parser.go index a2a8565641f..01b8cb1d235 100644 --- a/pkg/tsdb/elasticsearch/response_parser.go +++ b/pkg/tsdb/elasticsearch/response_parser.go @@ -6,14 +6,14 @@ import ( "github.com/grafana/grafana/pkg/components/null" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/tsdb" - "strconv" "regexp" + "strconv" "strings" ) type ElasticsearchResponseParser struct { Responses []Response - Targets []*QueryBuilder + Targets []*Query } func (rp *ElasticsearchResponseParser) getTimeSeries() *tsdb.QueryResult { @@ -29,7 +29,7 @@ func (rp *ElasticsearchResponseParser) getTimeSeries() *tsdb.QueryResult { return queryRes } -func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{}, target *QueryBuilder, series *[]*tsdb.TimeSeries, props map[string]string, depth int) (error) { +func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{}, target *Query, series *[]*tsdb.TimeSeries, props map[string]string, depth int) error { var err error maxDepth := len(target.BucketAggs) - 1 for aggId, v := range aggs { @@ -71,7 +71,7 @@ func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{ } -func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, target *QueryBuilder, series *[]*tsdb.TimeSeries, props map[string]string) (error) { +func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, target *Query, series *[]*tsdb.TimeSeries, props map[string]string) error { for _, v := range target.Metrics { metric := simplejson.NewFromAny(v) if metric.Get("hide").MustBool(false) { @@ -143,7 +143,7 @@ func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, ta return nil } -func (rp *ElasticsearchResponseParser) nameSeries(seriesList *[]*tsdb.TimeSeries, target *QueryBuilder) { +func (rp *ElasticsearchResponseParser) nameSeries(seriesList *[]*tsdb.TimeSeries, target *Query) { set := make(map[string]string) for _, v := range *seriesList { if metricType, exists := v.Tags["metric"]; exists { @@ -159,8 +159,9 @@ func (rp *ElasticsearchResponseParser) nameSeries(seriesList *[]*tsdb.TimeSeries } -func (rp *ElasticsearchResponseParser) getSeriesName(series *tsdb.TimeSeries, target *QueryBuilder, metricTypeCount int) (string) { - metricName := rp.getMetricName(series.Tags["metric"]) +func (rp *ElasticsearchResponseParser) getSeriesName(series *tsdb.TimeSeries, target *Query, metricTypeCount int) string { + metricType := series.Tags["metric"] + metricName := rp.getMetricName(metricType) delete(series.Tags, "metric") field := "" @@ -172,7 +173,7 @@ func (rp *ElasticsearchResponseParser) getSeriesName(series *tsdb.TimeSeries, ta if target.Alias != "" { var re = regexp.MustCompile(`{{([\s\S]+?)}}`) for _, match := range re.FindAllString(target.Alias, -1) { - group := match[2:len(match)-2] + group := match[2 : len(match)-2] if strings.HasPrefix(group, "term ") { if term, ok := series.Tags["term "]; ok { @@ -193,7 +194,20 @@ func (rp *ElasticsearchResponseParser) getSeriesName(series *tsdb.TimeSeries, ta } } // todo, if field and pipelineAgg - if field != "" { + if field != "" && isPipelineAgg(metricType) { + found := false + for _, targetMetricI := range target.Metrics { + targetMetric := simplejson.NewFromAny(targetMetricI) + if targetMetric.Get("id").MustString() == field { + metricName += " " + describeMetric(targetMetric.Get("type").MustString(), field) + found = true + } + } + if !found { + metricName = "Unset" + } + + } else if field != "" { metricName += " " + field } @@ -241,7 +255,7 @@ func castToNullFloat(j *simplejson.Json) null.Float { return null.NewFloat(0, false) } -func findAgg(target *QueryBuilder, aggId string) (*simplejson.Json, error) { +func findAgg(target *Query, aggId string) (*simplejson.Json, error) { for _, v := range target.BucketAggs { aggDef := simplejson.NewFromAny(v) if aggId == aggDef.Get("id").MustString() { From 4042e4b225ad4b989f3f1ecabb52271547ff2af2 Mon Sep 17 00:00:00 2001 From: wph95 Date: Tue, 27 Mar 2018 02:12:43 +0800 Subject: [PATCH 06/22] fix a terms bug and add test --- pkg/tsdb/elasticsearch/models.go | 2 +- pkg/tsdb/elasticsearch/query.go | 6 +- pkg/tsdb/elasticsearch/query_test.go | 97 ++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 3 deletions(-) diff --git a/pkg/tsdb/elasticsearch/models.go b/pkg/tsdb/elasticsearch/models.go index 822df2dd4d1..6ab6fa9f43e 100644 --- a/pkg/tsdb/elasticsearch/models.go +++ b/pkg/tsdb/elasticsearch/models.go @@ -41,7 +41,7 @@ type DateHistogramAgg struct { } type FiltersAgg struct { - Filter map[string]interface{} `json:"filter"` + Filters map[string]interface{} `json:"filters"` } type TermsAggSetting struct { diff --git a/pkg/tsdb/elasticsearch/query.go b/pkg/tsdb/elasticsearch/query.go index 51f1ebb5d7a..c4e30cfcbf4 100644 --- a/pkg/tsdb/elasticsearch/query.go +++ b/pkg/tsdb/elasticsearch/query.go @@ -193,15 +193,17 @@ func (q *Query) getHistogramAgg(model *simplejson.Json) *HistogramAgg { func (q *Query) getFilters(model *simplejson.Json) *FiltersAgg { agg := &FiltersAgg{} + agg.Filters = map[string]interface{}{} settings := simplejson.NewFromAny(model.Get("settings").Interface()) - for filter := range settings.Get("filters").MustArray() { + + for _, filter := range settings.Get("filters").MustArray() { filterJson := simplejson.NewFromAny(filter) query := filterJson.Get("query").MustString("") label := filterJson.Get("label").MustString("") if label == "" { label = query } - agg.Filter[label] = newQueryStringFilter(true, query) + agg.Filters[label] = newQueryStringFilter(true, query) } return agg } diff --git a/pkg/tsdb/elasticsearch/query_test.go b/pkg/tsdb/elasticsearch/query_test.go index 992469175b6..4f7b4d9147e 100644 --- a/pkg/tsdb/elasticsearch/query_test.go +++ b/pkg/tsdb/elasticsearch/query_test.go @@ -325,6 +325,103 @@ func TestElasticSearchQueryBuilder(t *testing.T) { "aggs": {"4":{"aggs":{"2":{"aggs":{"1":{"sum":{"field":"value"}}},"date_histogram":{"extended_bounds":{"max":"","min":""},"field":"timestamp","format":"epoch_millis","interval":"1m","min_doc_count":0}}},"terms":{"field":"name_raw","order":{"_term":"desc"},"size":10}}} }` + testElasticSearchResponse(testElasticsearchModelRequestJSON, expectedElasticsearchQueryJSON) + }) + Convey("Test Filters Aggregates", func() { + testElasticsearchModelRequestJSON := ` + { + "bucketAggs": [ + { + "id": "3", + "settings": { + "filters": [{ + "label": "hello", + "query": "host:\"67.65.185.232\"" + }] + }, + "type": "filters" + }, + { + "field": "time", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": 0, + "trimEdges": 0 + }, + "type": "date_histogram" + } + ], + "metrics": [ + { + "pipelineAgg": "select metric", + "field": "bytesSent", + "id": "1", + "meta": {}, + "settings": {}, + "type": "count" + } + ], + "query": "*", + "refId": "A", + "timeField": "time" + }` + + expectedElasticsearchQueryJSON := `{ + "size": 0, + "query": { + "bool": { + "filter": [ + { + "range": { + "time": { + "gte": "", + "lte": "", + "format": "epoch_millis" + } + } + }, + { + "query_string": { + "analyze_wildcard": true, + "query": "*" + } + } + ] + } + }, + "aggs": { + "3": { + "filters": { + "filters": { + "hello": { + "query_string": { + "query": "host:\"67.65.185.232\"", + "analyze_wildcard": true + } + } + } + }, + "aggs": { + "2": { + "date_histogram": { + "interval": "200ms", + "field": "time", + "min_doc_count": 0, + "extended_bounds": { + "min": "", + "max": "" + }, + "format": "epoch_millis" + }, + "aggs": {} + } + } + } + } + } + ` + testElasticSearchResponse(testElasticsearchModelRequestJSON, expectedElasticsearchQueryJSON) }) }) From 06f73321560defb2ac074e4d90af1c94f459943d Mon Sep 17 00:00:00 2001 From: wph95 Date: Wed, 28 Mar 2018 01:42:25 +0800 Subject: [PATCH 07/22] cleanup and add more test --- pkg/tsdb/elasticsearch/elasticsearch_test.go | 121 ++++++++++++ pkg/tsdb/elasticsearch/model_parser.go | 65 ++++++- pkg/tsdb/elasticsearch/models.go | 26 ++- pkg/tsdb/elasticsearch/query.go | 113 +++++------ pkg/tsdb/elasticsearch/query_def.go | 1 - pkg/tsdb/elasticsearch/query_test.go | 186 +------------------ pkg/tsdb/elasticsearch/response_parser.go | 50 +++-- 7 files changed, 274 insertions(+), 288 deletions(-) create mode 100644 pkg/tsdb/elasticsearch/elasticsearch_test.go diff --git a/pkg/tsdb/elasticsearch/elasticsearch_test.go b/pkg/tsdb/elasticsearch/elasticsearch_test.go new file mode 100644 index 00000000000..ad905299166 --- /dev/null +++ b/pkg/tsdb/elasticsearch/elasticsearch_test.go @@ -0,0 +1,121 @@ +package elasticsearch + +import ( + "github.com/grafana/grafana/pkg/components/simplejson" + "time" +) + +var avgWithMovingAvg = Query{ + TimeField: "timestamp", + RawQuery: "(test:query) AND (name:sample)", + Interval: time.Millisecond, + BucketAggs: []*BucketAgg{{ + Field: "timestamp", + ID: "2", + Type: "date_histogram", + Settings: simplejson.NewFromAny(map[string]interface{}{ + "interval": "auto", + "min_doc_count": 0, + "trimEdges": 0, + }), + }}, + Metrics: []*Metric{{ + Field: "value", + ID: "1", + Type: "avg", + Settings: simplejson.NewFromAny(map[string]interface{}{ + "script": map[string]string{ + "inline": "_value * 2", + }, + }), + }, { + Field: "1", + ID: "3", + Type: "moving_avg", + PipelineAggregate: "1", + Settings: simplejson.NewFromAny(map[string]interface{}{ + "minimize": false, + "model": "simple", + "window": 5, + }), + }}, +} + +var wildcardsAndQuotes = Query{ + TimeField: "timestamp", + RawQuery: "scope:$location.leagueconnect.api AND name:*CreateRegistration AND name:\"*.201-responses.rate\"", + Interval: time.Millisecond, + BucketAggs: []*BucketAgg{{ + Field: "timestamp", + ID: "2", + Type: "date_histogram", + Settings: simplejson.NewFromAny(map[string]interface{}{}), + }}, + Metrics: []*Metric{{ + Field: "value", + ID: "1", + Type: "sum", + Settings: simplejson.NewFromAny(map[string]interface{}{}), + }}, +} +var termAggs = Query{ + TimeField: "timestamp", + RawQuery: "(scope:*.hmp.metricsd) AND (name_raw:builtin.general.*_instance_count)", + Interval: time.Millisecond, + BucketAggs: []*BucketAgg{{ + Field: "name_raw", + ID: "4", + Type: "terms", + Settings: simplejson.NewFromAny(map[string]interface{}{ + "order": "desc", + "orderBy": "_term", + "size": "10", + }), + }, { + Field: "timestamp", + ID: "2", + Type: "date_histogram", + Settings: simplejson.NewFromAny(map[string]interface{}{ + "interval": "auto", + "min_doc_count": 0, + "trimEdges": 0, + }), + }}, + Metrics: []*Metric{{ + Field: "value", + ID: "1", + Type: "sum", + Settings: simplejson.NewFromAny(map[string]interface{}{}), + }}, +} + +var filtersAggs = Query{ + TimeField: "time", + RawQuery: "*", + Interval: time.Millisecond, + BucketAggs: []*BucketAgg{{ + ID: "3", + Type: "filters", + Settings: simplejson.NewFromAny(map[string]interface{}{ + "filters": []interface{}{ + map[string]interface{}{"label": "hello", "query": "host:\"67.65.185.232\""}, + }, + }), + }, { + Field: "timestamp", + ID: "2", + Type: "date_histogram", + Settings: simplejson.NewFromAny(map[string]interface{}{ + "interval": "auto", + "min_doc_count": 0, + "trimEdges": 0, + }), + }}, + Metrics: []*Metric{{ + Field: "bytesSent", + ID: "1", + Type: "count", + PipelineAggregate: "select metric", + Settings: simplejson.NewFromAny(map[string]interface{}{}), + }}, +} diff --git a/pkg/tsdb/elasticsearch/model_parser.go b/pkg/tsdb/elasticsearch/model_parser.go index 0d016dc58a5..5d94aebef1a 100644 --- a/pkg/tsdb/elasticsearch/model_parser.go +++ b/pkg/tsdb/elasticsearch/model_parser.go @@ -20,9 +20,15 @@ func (qp *ElasticSearchQueryParser) Parse(model *simplejson.Json, dsInfo *models if err != nil { return nil, err } - rawQuery := model.Get("query").MustString("") - bucketAggs := model.Get("bucketAggs").MustArray() - metrics := model.Get("metrics").MustArray() + rawQuery := model.Get("query").MustString() + bucketAggs, err := qp.parseBucketAggs(model) + if err != nil { + return nil, err + } + metrics, err := qp.parseMetrics(model) + if err != nil { + return nil, err + } alias := model.Get("alias").MustString("") parsedInterval, err := tsdb.GetIntervalFrom(dsInfo, model, time.Millisecond) if err != nil { @@ -37,6 +43,57 @@ func (qp *ElasticSearchQueryParser) Parse(model *simplejson.Json, dsInfo *models parsedInterval}, nil } +func (qp *ElasticSearchQueryParser) parseBucketAggs(model *simplejson.Json) ([]*BucketAgg, error) { + var err error + var result []*BucketAgg + for _, t := range model.Get("bucketAggs").MustArray() { + aggJson := simplejson.NewFromAny(t) + agg := &BucketAgg{} + + agg.Type, err = aggJson.Get("type").String() + if err != nil { + return nil, err + } + + agg.ID, err = aggJson.Get("id").String() + if err != nil { + return nil, err + } + + agg.Field = aggJson.Get("field").MustString() + agg.Settings = simplejson.NewFromAny(aggJson.Get("settings").MustMap()) + + result = append(result, agg) + } + return result, nil +} + +func (qp *ElasticSearchQueryParser) parseMetrics(model *simplejson.Json) ([]*Metric, error) { + var err error + var result []*Metric + for _, t := range model.Get("metrics").MustArray() { + metricJson := simplejson.NewFromAny(t) + metric := &Metric{} + + metric.Field = metricJson.Get("field").MustString() + metric.Hide = metricJson.Get("hide").MustBool(false) + metric.ID, err = metricJson.Get("id").String() + if err != nil { + return nil, err + } + + metric.PipelineAggregate = metricJson.Get("pipelineAgg").MustString() + metric.Settings = simplejson.NewFromAny(metricJson.Get("settings").MustMap()) + + metric.Type, err = metricJson.Get("type").String() + if err != nil { + return nil, err + } + + result = append(result, metric) + } + return result, nil +} func getRequestHeader(timeRange *tsdb.TimeRange, dsInfo *models.DataSource) *QueryHeader { var header QueryHeader esVersion := dsInfo.JsonData.Get("esVersion").MustInt() @@ -47,7 +104,7 @@ func getRequestHeader(timeRange *tsdb.TimeRange, dsInfo *models.DataSource) *Que } header.SearchType = searchType header.IgnoreUnavailable = true - header.Index = getIndexList(dsInfo.Database, dsInfo.JsonData.Get("interval").MustString(""), timeRange) + header.Index = getIndexList(dsInfo.Database, dsInfo.JsonData.Get("interval").MustString(), timeRange) if esVersion >= 56 { header.MaxConcurrentShardRequests = dsInfo.JsonData.Get("maxConcurrentShardRequests").MustInt() diff --git a/pkg/tsdb/elasticsearch/models.go b/pkg/tsdb/elasticsearch/models.go index 6ab6fa9f43e..9cf295cbd0e 100644 --- a/pkg/tsdb/elasticsearch/models.go +++ b/pkg/tsdb/elasticsearch/models.go @@ -7,6 +7,22 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" ) +type BucketAgg struct { + Field string `json:"field"` + ID string `json:"id"` + Settings *simplejson.Json `json:"settings"` + Type string `jsons:"type"` +} + +type Metric struct { + Field string `json:"field"` + Hide bool `json:"hide"` + ID string `json:"id"` + PipelineAggregate string `json:"pipelineAgg"` + Settings *simplejson.Json `json:"settings"` + Type string `json:"type"` +} + type QueryHeader struct { SearchType string `json:"search_type"` IgnoreUnavailable bool `json:"ignore_unavailable"` @@ -44,16 +60,16 @@ type FiltersAgg struct { Filters map[string]interface{} `json:"filters"` } -type TermsAggSetting struct { +type TermsAgg struct { Field string `json:"field"` Size int `json:"size"` Order map[string]interface{} `json:"order"` Missing string `json:"missing,omitempty"` } -type TermsAgg struct { - Terms TermsAggSetting `json:"terms"` - Aggs Aggs `json:"aggs"` +type TermsAggWrap struct { + Terms TermsAgg `json:"terms"` + Aggs Aggs `json:"aggs"` } type ExtendedBounds struct { @@ -91,8 +107,6 @@ type BoolQuery struct { Filter []interface{} `json:"filter"` } -type Metric map[string]interface{} - type Responses struct { Responses []Response `json:"responses"` } diff --git a/pkg/tsdb/elasticsearch/query.go b/pkg/tsdb/elasticsearch/query.go index c4e30cfcbf4..a63529df2df 100644 --- a/pkg/tsdb/elasticsearch/query.go +++ b/pkg/tsdb/elasticsearch/query.go @@ -18,11 +18,11 @@ var rangeFilterSetting = RangeFilterSetting{Gte: "$timeFrom", Format: "epoch_millis"} type Query struct { - TimeField string `json:"timeField"` - RawQuery string `json:"query"` - BucketAggs []interface{} `json:"bucketAggs"` - Metrics []interface{} `json:"metrics"` - Alias string `json:"Alias"` + TimeField string `json:"timeField"` + RawQuery string `json:"query"` + BucketAggs []*BucketAgg `json:"bucketAggs"` + Metrics []*Metric `json:"metrics"` + Alias string `json:"Alias"` Interval time.Duration } @@ -73,27 +73,17 @@ func (q *Query) renderReqQuery(req *Request) { func (q *Query) parseAggs(req *Request) error { aggs := make(Aggs) nestedAggs := aggs - for _, aggRaw := range q.BucketAggs { + for _, agg := range q.BucketAggs { esAggs := make(Aggs) - aggJson := simplejson.NewFromAny(aggRaw) - aggType, err := aggJson.Get("type").String() - if err != nil { - return err - } - id, err := aggJson.Get("id").String() - if err != nil { - return err - } - - switch aggType { + switch agg.Type { case "date_histogram": - esAggs["date_histogram"] = q.getDateHistogramAgg(aggJson) + esAggs["date_histogram"] = q.getDateHistogramAgg(agg) case "histogram": - esAggs["histogram"] = q.getHistogramAgg(aggJson) + esAggs["histogram"] = q.getHistogramAgg(agg) case "filters": - esAggs["filters"] = q.getFilters(aggJson) + esAggs["filters"] = q.getFilters(agg) case "terms": - terms := q.getTerms(aggJson) + terms := q.getTerms(agg) esAggs["terms"] = terms.Terms esAggs["aggs"] = terms.Aggs case "geohash_grid": @@ -105,59 +95,47 @@ func (q *Query) parseAggs(req *Request) error { } if aggs, ok := (nestedAggs["aggs"]).(Aggs); ok { - aggs[id] = esAggs + aggs[agg.ID] = esAggs } nestedAggs = esAggs } nestedAggs["aggs"] = make(Aggs) - for _, metricRaw := range q.Metrics { - metric := make(Metric) - metricJson := simplejson.NewFromAny(metricRaw) + for _, metric := range q.Metrics { + subAgg := make(Aggs) - id, err := metricJson.Get("id").String() - if err != nil { - return err - } - metricType, err := metricJson.Get("type").String() - if err != nil { - return err - } - if metricType == "count" { + if metric.Type == "count" { continue } + settings := metric.Settings.MustMap(make(map[string]interface{})) - settings := metricJson.Get("settings").MustMap(map[string]interface{}{}) - - if isPipelineAgg(metricType) { - pipelineAgg := metricJson.Get("pipelineAgg").MustString("") - if _, err := strconv.Atoi(pipelineAgg); err == nil { - settings["buckets_path"] = pipelineAgg + if isPipelineAgg(metric.Type) { + if _, err := strconv.Atoi(metric.PipelineAggregate); err == nil { + settings["buckets_path"] = metric.PipelineAggregate } else { continue } } else { - settings["field"] = metricJson.Get("field").MustString() + settings["field"] = metric.Field } - metric[metricType] = settings - nestedAggs["aggs"].(Aggs)[id] = metric + subAgg[metric.Type] = settings + nestedAggs["aggs"].(Aggs)[metric.ID] = subAgg } req.Aggs = aggs["aggs"].(Aggs) return nil } -func (q *Query) getDateHistogramAgg(model *simplejson.Json) *DateHistogramAgg { +func (q *Query) getDateHistogramAgg(target *BucketAgg) *DateHistogramAgg { agg := &DateHistogramAgg{} - settings := simplejson.NewFromAny(model.Get("settings").Interface()) - interval, err := settings.Get("interval").String() + interval, err := target.Settings.Get("interval").String() if err == nil { agg.Interval = interval } agg.Field = q.TimeField - agg.MinDocCount = settings.Get("min_doc_count").MustInt(0) + agg.MinDocCount = target.Settings.Get("min_doc_count").MustInt(0) agg.ExtendedBounds = ExtendedBounds{"$timeFrom", "$timeTo"} agg.Format = "epoch_millis" @@ -165,66 +143,63 @@ func (q *Query) getDateHistogramAgg(model *simplejson.Json) *DateHistogramAgg { agg.Interval = "$__interval" } - missing, err := settings.Get("missing").String() + missing, err := target.Settings.Get("missing").String() if err == nil { agg.Missing = missing } return agg } -func (q *Query) getHistogramAgg(model *simplejson.Json) *HistogramAgg { +func (q *Query) getHistogramAgg(target *BucketAgg) *HistogramAgg { agg := &HistogramAgg{} - settings := simplejson.NewFromAny(model.Get("settings").Interface()) - interval, err := settings.Get("interval").String() + interval, err := target.Settings.Get("interval").String() if err == nil { agg.Interval = interval } - field, err := model.Get("field").String() - if err == nil { - agg.Field = field + + if target.Field != "" { + agg.Field = target.Field } - agg.MinDocCount = settings.Get("min_doc_count").MustInt(0) - missing, err := settings.Get("missing").String() + agg.MinDocCount = target.Settings.Get("min_doc_count").MustInt(0) + missing, err := target.Settings.Get("missing").String() if err == nil { agg.Missing = missing } return agg } -func (q *Query) getFilters(model *simplejson.Json) *FiltersAgg { +func (q *Query) getFilters(target *BucketAgg) *FiltersAgg { agg := &FiltersAgg{} agg.Filters = map[string]interface{}{} - settings := simplejson.NewFromAny(model.Get("settings").Interface()) - - for _, filter := range settings.Get("filters").MustArray() { + for _, filter := range target.Settings.Get("filters").MustArray() { filterJson := simplejson.NewFromAny(filter) query := filterJson.Get("query").MustString("") label := filterJson.Get("label").MustString("") if label == "" { label = query } + agg.Filters[label] = newQueryStringFilter(true, query) } return agg } -func (q *Query) getTerms(model *simplejson.Json) *TermsAgg { - agg := &TermsAgg{Aggs: make(Aggs)} - settings := simplejson.NewFromAny(model.Get("settings").Interface()) - agg.Terms.Field = model.Get("field").MustString() - if settings == nil { +func (q *Query) getTerms(target *BucketAgg) *TermsAggWrap { + agg := &TermsAggWrap{Aggs: make(Aggs)} + agg.Terms.Field = target.Field + if len(target.Settings.MustMap()) == 0 { return agg } - sizeStr := settings.Get("size").MustString("") + sizeStr := target.Settings.Get("size").MustString("") size, err := strconv.Atoi(sizeStr) if err != nil { size = 500 } agg.Terms.Size = size - orderBy, err := settings.Get("orderBy").String() + orderBy, err := target.Settings.Get("orderBy").String() if err == nil { agg.Terms.Order = make(map[string]interface{}) - agg.Terms.Order[orderBy] = settings.Get("order").MustString("") + agg.Terms.Order[orderBy] = target.Settings.Get("order").MustString("") if _, err := strconv.Atoi(orderBy); err != nil { for _, metricI := range q.Metrics { metric := simplejson.NewFromAny(metricI) @@ -242,7 +217,7 @@ func (q *Query) getTerms(model *simplejson.Json) *TermsAgg { } } - missing, err := settings.Get("missing").String() + missing, err := target.Settings.Get("missing").String() if err == nil { agg.Terms.Missing = missing } diff --git a/pkg/tsdb/elasticsearch/query_def.go b/pkg/tsdb/elasticsearch/query_def.go index 6f78f02f346..128e752d97a 100644 --- a/pkg/tsdb/elasticsearch/query_def.go +++ b/pkg/tsdb/elasticsearch/query_def.go @@ -40,5 +40,4 @@ func isPipelineAgg(metricType string) bool { func describeMetric(metricType, field string) string { text := metricAggType[metricType] return text + " " + field - } diff --git a/pkg/tsdb/elasticsearch/query_test.go b/pkg/tsdb/elasticsearch/query_test.go index 4f7b4d9147e..aecca9f4734 100644 --- a/pkg/tsdb/elasticsearch/query_test.go +++ b/pkg/tsdb/elasticsearch/query_test.go @@ -13,13 +13,8 @@ import ( "testing" ) -func testElasticSearchResponse(requestJSON string, expectedElasticSearchRequestJSON string) { +func testElasticSearchResponse(query Query, expectedElasticSearchRequestJSON string) { var queryExpectedJSONInterface, queryJSONInterface interface{} - parser := ElasticSearchQueryParser{} - model := &Query{} - - err := json.Unmarshal([]byte(requestJSON), model) - So(err, ShouldBeNil) jsonDate, _ := simplejson.NewJson([]byte(`{"esVersion":2}`)) dsInfo := &models.DataSource{ Database: "grafana-test", @@ -28,10 +23,8 @@ func testElasticSearchResponse(requestJSON string, expectedElasticSearchRequestJ testTimeRange := tsdb.NewTimeRange("5m", "now") - req, _ := simplejson.NewJson([]byte(requestJSON)) - query, err := parser.Parse(req, dsInfo) s, err := query.Build(&tsdb.TsdbQuery{TimeRange: testTimeRange}, dsInfo) - + So(err, ShouldBeNil) queryJSON := strings.Split(s, "\n")[1] err = json.Unmarshal([]byte(queryJSON), &queryJSONInterface) So(err, ShouldBeNil) @@ -62,53 +55,6 @@ func testElasticSearchResponse(requestJSON string, expectedElasticSearchRequestJ func TestElasticSearchQueryBuilder(t *testing.T) { Convey("Elasticsearch QueryBuilder query testing", t, func() { Convey("Build test average metric with moving average", func() { - var testElasticsearchModelRequestJSON = ` - { - "bucketAggs": [ - { - "field": "timestamp", - "id": "2", - "settings": { - "interval": "auto", - "min_doc_count": 0, - "trimEdges": 0 - }, - "type": "date_histogram" - } - ], - "dsType": "elasticsearch", - "metrics": [ - { - "field": "value", - "id": "1", - "inlineScript": "_value * 2", - "meta": {}, - "settings": { - "script": { - "inline": "_value * 2" - } - }, - "type": "avg" - }, - { - "field": "1", - "id": "3", - "meta": {}, - "pipelineAgg": "1", - "settings": { - "minimize": false, - "model": "simple", - "window": 5 - }, - "type": "moving_avg" - } - ], - "query": "(test:query) AND (name:sample)", - "refId": "A", - "timeField": "timestamp" - } - ` - var expectedElasticsearchQueryJSON = ` { "size": 0, @@ -167,32 +113,9 @@ func TestElasticSearchQueryBuilder(t *testing.T) { } }` - testElasticSearchResponse(testElasticsearchModelRequestJSON, expectedElasticsearchQueryJSON) + testElasticSearchResponse(avgWithMovingAvg, expectedElasticsearchQueryJSON) }) Convey("Test Wildcards and Quotes", func() { - testElasticsearchModelRequestJSON := ` - { - "alias": "New", - "bucketAggs": [ - { - "field": "timestamp", - "id": "2", - "type": "date_histogram" - } - ], - "dsType": "elasticsearch", - "metrics": [ - { - "type": "sum", - "field": "value", - "id": "1" - } - ], - "query": "scope:$location.leagueconnect.api AND name:*CreateRegistration AND name:\"*.201-responses.rate\"", - "refId": "A", - "timeField": "timestamp" - }` - expectedElasticsearchQueryJSON := ` { "size": 0, @@ -239,65 +162,9 @@ func TestElasticSearchQueryBuilder(t *testing.T) { } }` - testElasticSearchResponse(testElasticsearchModelRequestJSON, expectedElasticsearchQueryJSON) + testElasticSearchResponse(wildcardsAndQuotes, expectedElasticsearchQueryJSON) }) Convey("Test Term Aggregates", func() { - testElasticsearchModelRequestJSON := ` - { - "bucketAggs": [{ - "field": "name_raw", - "id": "4", - "settings": { - "order": "desc", - "orderBy": "_term", - "size": "10" - }, - "type": "terms" - }, { - "field": "timestamp", - "id": "2", - "settings": { - "interval": "1m", - "min_doc_count": 0, - "trimEdges": 0 - }, - "type": "date_histogram" - }], - "dsType": "elasticsearch", - "filters": [{ - "boolOp": "AND", - "not": false, - "type": "rfc190Scope", - "value": "*.hmp.metricsd" - }, { - "boolOp": "AND", - "not": false, - "type": "name_raw", - "value": "builtin.general.*_instance_count" - }], - "metricObject": {}, - "metrics": [{ - "field": "value", - "id": "1", - "meta": {}, - "options": {}, - "settings": {}, - "type": "sum" - }], - "mode": 0, - "numToGraph": 10, - "prependHostName": false, - "query": "(scope:*.hmp.metricsd) AND (name_raw:builtin.general.*_instance_count)", - "refId": "A", - "regexAlias": false, - "selectedApplication": "", - "selectedHost": "", - "selectedLocation": "", - "timeField": "timestamp", - "useFullHostName": "", - "useQuery": false - }` - expectedElasticsearchQueryJSON := ` { "size": 0, @@ -322,51 +189,12 @@ func TestElasticSearchQueryBuilder(t *testing.T) { ] } }, - "aggs": {"4":{"aggs":{"2":{"aggs":{"1":{"sum":{"field":"value"}}},"date_histogram":{"extended_bounds":{"max":"","min":""},"field":"timestamp","format":"epoch_millis","interval":"1m","min_doc_count":0}}},"terms":{"field":"name_raw","order":{"_term":"desc"},"size":10}}} + "aggs": {"4":{"aggs":{"2":{"aggs":{"1":{"sum":{"field":"value"}}},"date_histogram":{"extended_bounds":{"max":"","min":""},"field":"timestamp","format":"epoch_millis","interval":"200ms","min_doc_count":0}}},"terms":{"field":"name_raw","order":{"_term":"desc"},"size":10}}} }` - testElasticSearchResponse(testElasticsearchModelRequestJSON, expectedElasticsearchQueryJSON) + testElasticSearchResponse(termAggs, expectedElasticsearchQueryJSON) }) Convey("Test Filters Aggregates", func() { - testElasticsearchModelRequestJSON := ` - { - "bucketAggs": [ - { - "id": "3", - "settings": { - "filters": [{ - "label": "hello", - "query": "host:\"67.65.185.232\"" - }] - }, - "type": "filters" - }, - { - "field": "time", - "id": "2", - "settings": { - "interval": "auto", - "min_doc_count": 0, - "trimEdges": 0 - }, - "type": "date_histogram" - } - ], - "metrics": [ - { - "pipelineAgg": "select metric", - "field": "bytesSent", - "id": "1", - "meta": {}, - "settings": {}, - "type": "count" - } - ], - "query": "*", - "refId": "A", - "timeField": "time" - }` - expectedElasticsearchQueryJSON := `{ "size": 0, "query": { @@ -422,7 +250,7 @@ func TestElasticSearchQueryBuilder(t *testing.T) { } ` - testElasticSearchResponse(testElasticsearchModelRequestJSON, expectedElasticsearchQueryJSON) + testElasticSearchResponse(filtersAggs, expectedElasticsearchQueryJSON) }) }) } diff --git a/pkg/tsdb/elasticsearch/response_parser.go b/pkg/tsdb/elasticsearch/response_parser.go index 01b8cb1d235..24d5ebebfc4 100644 --- a/pkg/tsdb/elasticsearch/response_parser.go +++ b/pkg/tsdb/elasticsearch/response_parser.go @@ -40,27 +40,26 @@ func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{ } if depth == maxDepth { - if aggDef.Get("type").MustString() == "date_histogram" { + if aggDef.Type == "date_histogram" { err = rp.processMetrics(esAgg, target, series, props) if err != nil { return err } } else { - return fmt.Errorf("not support type:%s", aggDef.Get("type").MustString()) + return fmt.Errorf("not support type:%s", aggDef.Type) } } else { for i, b := range esAgg.Get("buckets").MustArray() { - field := aggDef.Get("field").MustString() bucket := simplejson.NewFromAny(b) newProps := props if key, err := bucket.Get("key").String(); err == nil { - newProps[field] = key + newProps[aggDef.Field] = key } else { props["filter"] = strconv.Itoa(i) } if key, err := bucket.Get("key_as_string").String(); err == nil { - props[field] = key + props[aggDef.Field] = key } rp.processBuckets(bucket.MustMap(), target, series, newProps, depth+1) } @@ -72,17 +71,12 @@ func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{ } func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, target *Query, series *[]*tsdb.TimeSeries, props map[string]string) error { - for _, v := range target.Metrics { - metric := simplejson.NewFromAny(v) - if metric.Get("hide").MustBool(false) { + for _, metric := range target.Metrics { + if metric.Hide { continue } - metricId := metric.Get("id").MustString() - metricField := metric.Get("field").MustString() - metricType := metric.Get("type").MustString() - - switch metricType { + switch metric.Type { case "count": newSeries := tsdb.TimeSeries{} for _, v := range esAgg.Get("buckets").MustArray() { @@ -102,16 +96,16 @@ func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, ta } firstBucket := simplejson.NewFromAny(buckets[0]) - percentiles := firstBucket.GetPath(metricId, "values").MustMap() + percentiles := firstBucket.GetPath(metric.ID, "values").MustMap() for percentileName := range percentiles { newSeries := tsdb.TimeSeries{} newSeries.Tags = props newSeries.Tags["metric"] = "p" + percentileName - newSeries.Tags["field"] = metricField + newSeries.Tags["field"] = metric.Field for _, v := range buckets { bucket := simplejson.NewFromAny(v) - value := castToNullFloat(bucket.GetPath(metricId, "values", percentileName)) + value := castToNullFloat(bucket.GetPath(metric.ID, "values", percentileName)) key := castToNullFloat(bucket.Get("key")) newSeries.Points = append(newSeries.Points, tsdb.TimePoint{value, key}) } @@ -120,20 +114,20 @@ func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, ta default: newSeries := tsdb.TimeSeries{} newSeries.Tags = props - newSeries.Tags["metric"] = metricType - newSeries.Tags["field"] = metricField + newSeries.Tags["metric"] = metric.Type + newSeries.Tags["field"] = metric.Field for _, v := range esAgg.Get("buckets").MustArray() { bucket := simplejson.NewFromAny(v) key := castToNullFloat(bucket.Get("key")) - valueObj, err := bucket.Get(metricId).Map() + valueObj, err := bucket.Get(metric.ID).Map() if err != nil { break } var value null.Float if _, ok := valueObj["normalized_value"]; ok { - value = castToNullFloat(bucket.GetPath(metricId, "normalized_value")) + value = castToNullFloat(bucket.GetPath(metric.ID, "normalized_value")) } else { - value = castToNullFloat(bucket.GetPath(metricId, "value")) + value = castToNullFloat(bucket.GetPath(metric.ID, "value")) } newSeries.Points = append(newSeries.Points, tsdb.TimePoint{value, key}) } @@ -196,10 +190,9 @@ func (rp *ElasticsearchResponseParser) getSeriesName(series *tsdb.TimeSeries, ta // todo, if field and pipelineAgg if field != "" && isPipelineAgg(metricType) { found := false - for _, targetMetricI := range target.Metrics { - targetMetric := simplejson.NewFromAny(targetMetricI) - if targetMetric.Get("id").MustString() == field { - metricName += " " + describeMetric(targetMetric.Get("type").MustString(), field) + for _, metric := range target.Metrics { + if metric.ID == field { + metricName += " " + describeMetric(metric.Type, field) found = true } } @@ -255,11 +248,10 @@ func castToNullFloat(j *simplejson.Json) null.Float { return null.NewFloat(0, false) } -func findAgg(target *Query, aggId string) (*simplejson.Json, error) { +func findAgg(target *Query, aggId string) (*BucketAgg, error) { for _, v := range target.BucketAggs { - aggDef := simplejson.NewFromAny(v) - if aggId == aggDef.Get("id").MustString() { - return aggDef, nil + if aggId == v.ID { + return v, nil } } return nil, errors.New("can't found aggDef, aggID:" + aggId) From 4050fce2205f9fdb6c217eae49686730cabd92c7 Mon Sep 17 00:00:00 2001 From: wph95 Date: Wed, 28 Mar 2018 12:35:05 +0800 Subject: [PATCH 08/22] add response_parser test --- pkg/tsdb/elasticsearch/response_parser.go | 9 +- .../elasticsearch/response_parser_test.go | 109 ++++++++++++++++++ 2 files changed, 116 insertions(+), 2 deletions(-) create mode 100644 pkg/tsdb/elasticsearch/response_parser_test.go diff --git a/pkg/tsdb/elasticsearch/response_parser.go b/pkg/tsdb/elasticsearch/response_parser.go index 24d5ebebfc4..ec7d2f9eb08 100644 --- a/pkg/tsdb/elasticsearch/response_parser.go +++ b/pkg/tsdb/elasticsearch/response_parser.go @@ -30,6 +30,7 @@ func (rp *ElasticsearchResponseParser) getTimeSeries() *tsdb.QueryResult { } func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{}, target *Query, series *[]*tsdb.TimeSeries, props map[string]string, depth int) error { + var err error maxDepth := len(target.BucketAggs) - 1 for aggId, v := range aggs { @@ -113,7 +114,11 @@ func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, ta } default: newSeries := tsdb.TimeSeries{} - newSeries.Tags = props + newSeries.Tags = map[string]string{} + for k, v := range props { + newSeries.Tags[k] = v + } + newSeries.Tags["metric"] = metric.Type newSeries.Tags["field"] = metric.Field for _, v := range esAgg.Get("buckets").MustArray() { @@ -121,7 +126,7 @@ func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, ta key := castToNullFloat(bucket.Get("key")) valueObj, err := bucket.Get(metric.ID).Map() if err != nil { - break + continue } var value null.Float if _, ok := valueObj["normalized_value"]; ok { diff --git a/pkg/tsdb/elasticsearch/response_parser_test.go b/pkg/tsdb/elasticsearch/response_parser_test.go new file mode 100644 index 00000000000..c5b877c1925 --- /dev/null +++ b/pkg/tsdb/elasticsearch/response_parser_test.go @@ -0,0 +1,109 @@ +package elasticsearch + +import ( + "encoding/json" + "github.com/grafana/grafana/pkg/tsdb" + . "github.com/smartystreets/goconvey/convey" + "testing" +) + +func testElasticsearchResponse(body string, target Query) *tsdb.QueryResult { + var responses Responses + err := json.Unmarshal([]byte(body), &responses) + So(err, ShouldBeNil) + + responseParser := ElasticsearchResponseParser{responses.Responses, []*Query{&target}} + return responseParser.getTimeSeries() +} + +func TestElasticSearchResponseParser(t *testing.T) { + Convey("Elasticsearch Response query testing", t, func() { + Convey("Build test average metric with moving average", func() { + responses := `{ + "responses": [ + { + "took": 1, + "timed_out": false, + "_shards": { + "total": 5, + "successful": 5, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": 4500, + "max_score": 0, + "hits": [] + }, + "aggregations": { + "2": { + "buckets": [ + { + "1": { + "value": null + }, + "key_as_string": "1522205880000", + "key": 1522205880000, + "doc_count": 0 + }, + { + "1": { + "value": 10 + }, + "key_as_string": "1522205940000", + "key": 1522205940000, + "doc_count": 300 + }, + { + "1": { + "value": 10 + }, + "3": { + "value": 20 + }, + "key_as_string": "1522206000000", + "key": 1522206000000, + "doc_count": 300 + }, + { + "1": { + "value": 10 + }, + "3": { + "value": 20 + }, + "key_as_string": "1522206060000", + "key": 1522206060000, + "doc_count": 300 + } + ] + } + }, + "status": 200 + } + ] +} +` + res := testElasticsearchResponse(responses, avgWithMovingAvg) + So(len(res.Series), ShouldEqual, 2) + So(res.Series[0].Name, ShouldEqual, "Average value") + So(len(res.Series[0].Points), ShouldEqual, 4) + for i, p := range res.Series[0].Points { + if i == 0 { + So(p[0].Valid, ShouldBeFalse) + } else { + So(p[0].Float64, ShouldEqual, 10) + } + So(p[1].Float64, ShouldEqual, 1522205880000+60000*i) + } + + So(res.Series[1].Name, ShouldEqual, "Moving Average Average 1") + So(len(res.Series[1].Points), ShouldEqual, 2) + + for _, p := range res.Series[1].Points { + So(p[0].Float64, ShouldEqual, 20) + } + + }) + }) +} From 77400cef08b3c868bea9d4106c7fcbb82d3d0e65 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 23 May 2018 14:36:41 +0200 Subject: [PATCH 09/22] elasticsearch: refactor and cleanup Move time series query logic to specific file. Remove model parser and move to time series query file, adds parser test. --- pkg/tsdb/elasticsearch/elasticsearch.go | 105 +--------- pkg/tsdb/elasticsearch/model_parser.go | 153 --------------- pkg/tsdb/elasticsearch/model_parser_test.go | 49 ----- pkg/tsdb/elasticsearch/query.go | 71 ++++++- pkg/tsdb/elasticsearch/query_test.go | 49 ++++- .../elasticsearch/response_parser_test.go | 3 +- pkg/tsdb/elasticsearch/time_series_query.go | 182 ++++++++++++++++++ .../elasticsearch/time_series_query_test.go | 118 ++++++++++++ 8 files changed, 421 insertions(+), 309 deletions(-) delete mode 100644 pkg/tsdb/elasticsearch/model_parser.go delete mode 100644 pkg/tsdb/elasticsearch/model_parser_test.go create mode 100644 pkg/tsdb/elasticsearch/time_series_query.go create mode 100644 pkg/tsdb/elasticsearch/time_series_query_test.go diff --git a/pkg/tsdb/elasticsearch/elasticsearch.go b/pkg/tsdb/elasticsearch/elasticsearch.go index 0ce9eca0972..abf25feac06 100644 --- a/pkg/tsdb/elasticsearch/elasticsearch.go +++ b/pkg/tsdb/elasticsearch/elasticsearch.go @@ -1,27 +1,20 @@ package elasticsearch import ( - "bytes" "context" - "encoding/json" - "errors" "fmt" - "github.com/grafana/grafana/pkg/log" - "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/setting" - "github.com/grafana/grafana/pkg/tsdb" - "golang.org/x/net/context/ctxhttp" "net/http" "net/url" "path" "strings" "time" + + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/tsdb" ) -type ElasticsearchExecutor struct { - QueryParser *ElasticSearchQueryParser - Transport *http.Transport -} +type ElasticsearchExecutor struct{} var ( glog log.Logger @@ -29,14 +22,7 @@ var ( ) func NewElasticsearchExecutor(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) { - transport, err := dsInfo.GetHttpTransport() - if err != nil { - return nil, err - } - - return &ElasticsearchExecutor{ - Transport: transport, - }, nil + return &ElasticsearchExecutor{}, nil } func init() { @@ -46,84 +32,11 @@ func init() { } func (e *ElasticsearchExecutor) Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) { - result := &tsdb.Response{} - result.Results = make(map[string]*tsdb.QueryResult) - - queries, err := e.getQuery(dsInfo, tsdbQuery) - if err != nil { - return nil, err + if len(tsdbQuery.Queries) == 0 { + return nil, fmt.Errorf("query contains no queries") } - buff := bytes.Buffer{} - for _, q := range queries { - s, err := q.Build(tsdbQuery, dsInfo) - if err != nil { - return nil, err - } - buff.WriteString(s) - } - payload := buff.String() - - if setting.Env == setting.DEV { - glog.Debug("Elasticsearch playload", "raw playload", payload) - } - glog.Info("Elasticsearch playload", "raw playload", payload) - - req, err := e.createRequest(dsInfo, payload) - if err != nil { - return nil, err - } - - httpClient, err := dsInfo.GetHttpClient() - if err != nil { - return nil, err - } - - resp, err := ctxhttp.Do(ctx, httpClient, req) - if err != nil { - return nil, err - } - - if resp.StatusCode/100 != 2 { - return nil, fmt.Errorf("elasticsearch returned statuscode invalid status code: %v", resp.Status) - } - - var responses Responses - dec := json.NewDecoder(resp.Body) - defer resp.Body.Close() - dec.UseNumber() - err = dec.Decode(&responses) - if err != nil { - return nil, err - } - - for _, res := range responses.Responses { - if res.Err != nil { - return nil, errors.New(res.getErrMsg()) - } - } - responseParser := ElasticsearchResponseParser{responses.Responses, queries} - queryRes := responseParser.getTimeSeries() - result.Results["A"] = queryRes - return result, nil -} - -func (e *ElasticsearchExecutor) getQuery(dsInfo *models.DataSource, context *tsdb.TsdbQuery) ([]*Query, error) { - queries := make([]*Query, 0) - if len(context.Queries) == 0 { - return nil, fmt.Errorf("query request contains no queries") - } - for _, v := range context.Queries { - - query, err := e.QueryParser.Parse(v.Model, dsInfo) - if err != nil { - return nil, err - } - queries = append(queries, query) - - } - return queries, nil - + return e.executeTimeSeriesQuery(ctx, dsInfo, tsdbQuery) } func (e *ElasticsearchExecutor) createRequest(dsInfo *models.DataSource, query string) (*http.Request, error) { diff --git a/pkg/tsdb/elasticsearch/model_parser.go b/pkg/tsdb/elasticsearch/model_parser.go deleted file mode 100644 index 5d94aebef1a..00000000000 --- a/pkg/tsdb/elasticsearch/model_parser.go +++ /dev/null @@ -1,153 +0,0 @@ -package elasticsearch - -import ( - "fmt" - "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/tsdb" - "github.com/leibowitz/moment" - "strings" - "time" -) - -type ElasticSearchQueryParser struct { -} - -func (qp *ElasticSearchQueryParser) Parse(model *simplejson.Json, dsInfo *models.DataSource) (*Query, error) { - //payload := bytes.Buffer{} - //queryHeader := qp.getQueryHeader() - timeField, err := model.Get("timeField").String() - if err != nil { - return nil, err - } - rawQuery := model.Get("query").MustString() - bucketAggs, err := qp.parseBucketAggs(model) - if err != nil { - return nil, err - } - metrics, err := qp.parseMetrics(model) - if err != nil { - return nil, err - } - alias := model.Get("alias").MustString("") - parsedInterval, err := tsdb.GetIntervalFrom(dsInfo, model, time.Millisecond) - if err != nil { - return nil, err - } - - return &Query{timeField, - rawQuery, - bucketAggs, - metrics, - alias, - parsedInterval}, nil -} - -func (qp *ElasticSearchQueryParser) parseBucketAggs(model *simplejson.Json) ([]*BucketAgg, error) { - var err error - var result []*BucketAgg - for _, t := range model.Get("bucketAggs").MustArray() { - aggJson := simplejson.NewFromAny(t) - agg := &BucketAgg{} - - agg.Type, err = aggJson.Get("type").String() - if err != nil { - return nil, err - } - - agg.ID, err = aggJson.Get("id").String() - if err != nil { - return nil, err - } - - agg.Field = aggJson.Get("field").MustString() - agg.Settings = simplejson.NewFromAny(aggJson.Get("settings").MustMap()) - - result = append(result, agg) - } - return result, nil -} - -func (qp *ElasticSearchQueryParser) parseMetrics(model *simplejson.Json) ([]*Metric, error) { - var err error - var result []*Metric - for _, t := range model.Get("metrics").MustArray() { - metricJson := simplejson.NewFromAny(t) - metric := &Metric{} - - metric.Field = metricJson.Get("field").MustString() - metric.Hide = metricJson.Get("hide").MustBool(false) - metric.ID, err = metricJson.Get("id").String() - if err != nil { - return nil, err - } - - metric.PipelineAggregate = metricJson.Get("pipelineAgg").MustString() - metric.Settings = simplejson.NewFromAny(metricJson.Get("settings").MustMap()) - - metric.Type, err = metricJson.Get("type").String() - if err != nil { - return nil, err - } - - result = append(result, metric) - } - return result, nil -} -func getRequestHeader(timeRange *tsdb.TimeRange, dsInfo *models.DataSource) *QueryHeader { - var header QueryHeader - esVersion := dsInfo.JsonData.Get("esVersion").MustInt() - - searchType := "query_then_fetch" - if esVersion < 5 { - searchType = "count" - } - header.SearchType = searchType - header.IgnoreUnavailable = true - header.Index = getIndexList(dsInfo.Database, dsInfo.JsonData.Get("interval").MustString(), timeRange) - - if esVersion >= 56 { - header.MaxConcurrentShardRequests = dsInfo.JsonData.Get("maxConcurrentShardRequests").MustInt() - } - return &header -} - -func getIndexList(pattern string, interval string, timeRange *tsdb.TimeRange) string { - if interval == "" { - return pattern - } - - var indexes []string - indexParts := strings.Split(strings.TrimLeft(pattern, "["), "]") - indexBase := indexParts[0] - if len(indexParts) <= 1 { - return pattern - } - - indexDateFormat := indexParts[1] - - start := moment.NewMoment(timeRange.MustGetFrom()) - end := moment.NewMoment(timeRange.MustGetTo()) - - indexes = append(indexes, fmt.Sprintf("%s%s", indexBase, start.Format(indexDateFormat))) - for start.IsBefore(*end) { - switch interval { - case "Hourly": - start = start.AddHours(1) - - case "Daily": - start = start.AddDay() - - case "Weekly": - start = start.AddWeeks(1) - - case "Monthly": - start = start.AddMonths(1) - - case "Yearly": - start = start.AddYears(1) - } - indexes = append(indexes, fmt.Sprintf("%s%s", indexBase, start.Format(indexDateFormat))) - } - return strings.Join(indexes, ",") -} diff --git a/pkg/tsdb/elasticsearch/model_parser_test.go b/pkg/tsdb/elasticsearch/model_parser_test.go deleted file mode 100644 index aa7336fb69b..00000000000 --- a/pkg/tsdb/elasticsearch/model_parser_test.go +++ /dev/null @@ -1,49 +0,0 @@ -package elasticsearch - -import ( - "github.com/grafana/grafana/pkg/tsdb" - . "github.com/smartystreets/goconvey/convey" - "strconv" - "strings" - "testing" -) - -func makeTime(hour int) string { - //unixtime 1500000000 == 2017-07-14T02:40:00+00:00 - return strconv.Itoa((1500000000 + hour*60*60) * 1000) -} - -func getIndexListByTime(pattern string, interval string, hour int) string { - timeRange := &tsdb.TimeRange{ - From: makeTime(0), - To: makeTime(hour), - } - return getIndexList(pattern, interval, timeRange) -} - -func TestElasticsearchGetIndexList(t *testing.T) { - Convey("Test Elasticsearch getIndex ", t, func() { - - Convey("Parse Interval Formats", func() { - So(getIndexListByTime("[logstash-]YYYY.MM.DD", "Daily", 48), - ShouldEqual, "logstash-2017.07.14,logstash-2017.07.15,logstash-2017.07.16") - - So(len(strings.Split(getIndexListByTime("[logstash-]YYYY.MM.DD.HH", "Hourly", 3), ",")), - ShouldEqual, 4) - - So(getIndexListByTime("[logstash-]YYYY.W", "Weekly", 100), - ShouldEqual, "logstash-2017.28,logstash-2017.29") - - So(getIndexListByTime("[logstash-]YYYY.MM", "Monthly", 700), - ShouldEqual, "logstash-2017.07,logstash-2017.08") - - So(getIndexListByTime("[logstash-]YYYY", "Yearly", 10000), - ShouldEqual, "logstash-2017,logstash-2018,logstash-2019") - }) - - Convey("No Interval", func() { - index := getIndexListByTime("logstash-test", "", 1) - So(index, ShouldEqual, "logstash-test") - }) - }) -} diff --git a/pkg/tsdb/elasticsearch/query.go b/pkg/tsdb/elasticsearch/query.go index a63529df2df..123f8dd5667 100644 --- a/pkg/tsdb/elasticsearch/query.go +++ b/pkg/tsdb/elasticsearch/query.go @@ -5,12 +5,14 @@ import ( "encoding/json" "errors" "fmt" - "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/tsdb" "strconv" "strings" "time" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/tsdb" + "github.com/leibowitz/moment" ) var rangeFilterSetting = RangeFilterSetting{Gte: "$timeFrom", @@ -22,14 +24,12 @@ type Query struct { RawQuery string `json:"query"` BucketAggs []*BucketAgg `json:"bucketAggs"` Metrics []*Metric `json:"metrics"` - Alias string `json:"Alias"` + Alias string `json:"alias"` Interval time.Duration } func (q *Query) Build(queryContext *tsdb.TsdbQuery, dsInfo *models.DataSource) (string, error) { var req Request - payload := bytes.Buffer{} - req.Size = 0 q.renderReqQuery(&req) @@ -45,6 +45,7 @@ func (q *Query) Build(queryContext *tsdb.TsdbQuery, dsInfo *models.DataSource) ( reqBytes, err := json.Marshal(req) reqHeader := getRequestHeader(queryContext.TimeRange, dsInfo) + payload := bytes.Buffer{} payload.WriteString(reqHeader.String() + "\n") payload.WriteString(string(reqBytes) + "\n") return q.renderTemplate(payload.String(), queryContext) @@ -235,3 +236,61 @@ func (q *Query) renderTemplate(payload string, queryContext *tsdb.TsdbQuery) (st payload = strings.Replace(payload, "$__interval", interval.Text, -1) return payload, nil } + +func getRequestHeader(timeRange *tsdb.TimeRange, dsInfo *models.DataSource) *QueryHeader { + var header QueryHeader + esVersion := dsInfo.JsonData.Get("esVersion").MustInt() + + searchType := "query_then_fetch" + if esVersion < 5 { + searchType = "count" + } + header.SearchType = searchType + header.IgnoreUnavailable = true + header.Index = getIndexList(dsInfo.Database, dsInfo.JsonData.Get("interval").MustString(), timeRange) + + if esVersion >= 56 { + header.MaxConcurrentShardRequests = dsInfo.JsonData.Get("maxConcurrentShardRequests").MustInt() + } + return &header +} + +func getIndexList(pattern string, interval string, timeRange *tsdb.TimeRange) string { + if interval == "" { + return pattern + } + + var indexes []string + indexParts := strings.Split(strings.TrimLeft(pattern, "["), "]") + indexBase := indexParts[0] + if len(indexParts) <= 1 { + return pattern + } + + indexDateFormat := indexParts[1] + + start := moment.NewMoment(timeRange.MustGetFrom()) + end := moment.NewMoment(timeRange.MustGetTo()) + + indexes = append(indexes, fmt.Sprintf("%s%s", indexBase, start.Format(indexDateFormat))) + for start.IsBefore(*end) { + switch interval { + case "Hourly": + start = start.AddHours(1) + + case "Daily": + start = start.AddDay() + + case "Weekly": + start = start.AddWeeks(1) + + case "Monthly": + start = start.AddMonths(1) + + case "Yearly": + start = start.AddYears(1) + } + indexes = append(indexes, fmt.Sprintf("%s%s", indexBase, start.Format(indexDateFormat))) + } + return strings.Join(indexes, ",") +} diff --git a/pkg/tsdb/elasticsearch/query_test.go b/pkg/tsdb/elasticsearch/query_test.go index aecca9f4734..1ce6e5ac7bb 100644 --- a/pkg/tsdb/elasticsearch/query_test.go +++ b/pkg/tsdb/elasticsearch/query_test.go @@ -3,14 +3,15 @@ package elasticsearch import ( "encoding/json" "fmt" - "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/tsdb" - . "github.com/smartystreets/goconvey/convey" "reflect" "strconv" "strings" "testing" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/tsdb" + . "github.com/smartystreets/goconvey/convey" ) func testElasticSearchResponse(query Query, expectedElasticSearchRequestJSON string) { @@ -254,3 +255,43 @@ func TestElasticSearchQueryBuilder(t *testing.T) { }) }) } + +func makeTime(hour int) string { + //unixtime 1500000000 == 2017-07-14T02:40:00+00:00 + return strconv.Itoa((1500000000 + hour*60*60) * 1000) +} + +func getIndexListByTime(pattern string, interval string, hour int) string { + timeRange := &tsdb.TimeRange{ + From: makeTime(0), + To: makeTime(hour), + } + return getIndexList(pattern, interval, timeRange) +} + +func TestElasticsearchGetIndexList(t *testing.T) { + Convey("Test Elasticsearch getIndex ", t, func() { + + Convey("Parse Interval Formats", func() { + So(getIndexListByTime("[logstash-]YYYY.MM.DD", "Daily", 48), + ShouldEqual, "logstash-2017.07.14,logstash-2017.07.15,logstash-2017.07.16") + + So(len(strings.Split(getIndexListByTime("[logstash-]YYYY.MM.DD.HH", "Hourly", 3), ",")), + ShouldEqual, 4) + + So(getIndexListByTime("[logstash-]YYYY.W", "Weekly", 100), + ShouldEqual, "logstash-2017.28,logstash-2017.29") + + So(getIndexListByTime("[logstash-]YYYY.MM", "Monthly", 700), + ShouldEqual, "logstash-2017.07,logstash-2017.08") + + So(getIndexListByTime("[logstash-]YYYY", "Yearly", 10000), + ShouldEqual, "logstash-2017,logstash-2018,logstash-2019") + }) + + Convey("No Interval", func() { + index := getIndexListByTime("logstash-test", "", 1) + So(index, ShouldEqual, "logstash-test") + }) + }) +} diff --git a/pkg/tsdb/elasticsearch/response_parser_test.go b/pkg/tsdb/elasticsearch/response_parser_test.go index c5b877c1925..1df2c4551ae 100644 --- a/pkg/tsdb/elasticsearch/response_parser_test.go +++ b/pkg/tsdb/elasticsearch/response_parser_test.go @@ -2,9 +2,10 @@ package elasticsearch import ( "encoding/json" + "testing" + "github.com/grafana/grafana/pkg/tsdb" . "github.com/smartystreets/goconvey/convey" - "testing" ) func testElasticsearchResponse(body string, target Query) *tsdb.QueryResult { diff --git a/pkg/tsdb/elasticsearch/time_series_query.go b/pkg/tsdb/elasticsearch/time_series_query.go new file mode 100644 index 00000000000..af8f61eb144 --- /dev/null +++ b/pkg/tsdb/elasticsearch/time_series_query.go @@ -0,0 +1,182 @@ +package elasticsearch + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/tsdb" + "golang.org/x/net/context/ctxhttp" +) + +type timeSeriesQuery struct { + queries []*Query +} + +func (e *ElasticsearchExecutor) executeTimeSeriesQuery(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) { + result := &tsdb.Response{} + result.Results = make(map[string]*tsdb.QueryResult) + + tsQueryParser := newTimeSeriesQueryParser(dsInfo) + query, err := tsQueryParser.parse(tsdbQuery) + if err != nil { + return nil, err + } + + buff := bytes.Buffer{} + for _, q := range query.queries { + s, err := q.Build(tsdbQuery, dsInfo) + if err != nil { + return nil, err + } + buff.WriteString(s) + } + payload := buff.String() + + if setting.Env == setting.DEV { + glog.Debug("Elasticsearch playload", "raw playload", payload) + } + glog.Info("Elasticsearch playload", "raw playload", payload) + + req, err := e.createRequest(dsInfo, payload) + if err != nil { + return nil, err + } + + httpClient, err := dsInfo.GetHttpClient() + if err != nil { + return nil, err + } + + resp, err := ctxhttp.Do(ctx, httpClient, req) + if err != nil { + return nil, err + } + + if resp.StatusCode/100 != 2 { + return nil, fmt.Errorf("elasticsearch returned statuscode invalid status code: %v", resp.Status) + } + + var responses Responses + defer resp.Body.Close() + dec := json.NewDecoder(resp.Body) + dec.UseNumber() + err = dec.Decode(&responses) + if err != nil { + return nil, err + } + + for _, res := range responses.Responses { + if res.Err != nil { + return nil, errors.New(res.getErrMsg()) + } + } + responseParser := ElasticsearchResponseParser{responses.Responses, query.queries} + queryRes := responseParser.getTimeSeries() + result.Results["A"] = queryRes + return result, nil +} + +type timeSeriesQueryParser struct { + ds *models.DataSource +} + +func newTimeSeriesQueryParser(ds *models.DataSource) *timeSeriesQueryParser { + return &timeSeriesQueryParser{ + ds: ds, + } +} + +func (p *timeSeriesQueryParser) parse(tsdbQuery *tsdb.TsdbQuery) (*timeSeriesQuery, error) { + queries := make([]*Query, 0) + for _, q := range tsdbQuery.Queries { + model := q.Model + timeField, err := model.Get("timeField").String() + if err != nil { + return nil, err + } + rawQuery := model.Get("query").MustString() + bucketAggs, err := p.parseBucketAggs(model) + if err != nil { + return nil, err + } + metrics, err := p.parseMetrics(model) + if err != nil { + return nil, err + } + alias := model.Get("alias").MustString("") + parsedInterval, err := tsdb.GetIntervalFrom(p.ds, model, time.Millisecond) + if err != nil { + return nil, err + } + + queries = append(queries, &Query{ + TimeField: timeField, + RawQuery: rawQuery, + BucketAggs: bucketAggs, + Metrics: metrics, + Alias: alias, + Interval: parsedInterval, + }) + } + + return &timeSeriesQuery{queries: queries}, nil +} + +func (p *timeSeriesQueryParser) parseBucketAggs(model *simplejson.Json) ([]*BucketAgg, error) { + var err error + var result []*BucketAgg + for _, t := range model.Get("bucketAggs").MustArray() { + aggJson := simplejson.NewFromAny(t) + agg := &BucketAgg{} + + agg.Type, err = aggJson.Get("type").String() + if err != nil { + return nil, err + } + + agg.ID, err = aggJson.Get("id").String() + if err != nil { + return nil, err + } + + agg.Field = aggJson.Get("field").MustString() + agg.Settings = simplejson.NewFromAny(aggJson.Get("settings").MustMap()) + + result = append(result, agg) + } + return result, nil +} + +func (p *timeSeriesQueryParser) parseMetrics(model *simplejson.Json) ([]*Metric, error) { + var err error + var result []*Metric + for _, t := range model.Get("metrics").MustArray() { + metricJSON := simplejson.NewFromAny(t) + metric := &Metric{} + + metric.Field = metricJSON.Get("field").MustString() + metric.Hide = metricJSON.Get("hide").MustBool(false) + metric.ID, err = metricJSON.Get("id").String() + if err != nil { + return nil, err + } + + metric.PipelineAggregate = metricJSON.Get("pipelineAgg").MustString() + metric.Settings = simplejson.NewFromAny(metricJSON.Get("settings").MustMap()) + + metric.Type, err = metricJSON.Get("type").String() + if err != nil { + return nil, err + } + + result = append(result, metric) + } + return result, nil +} diff --git a/pkg/tsdb/elasticsearch/time_series_query_test.go b/pkg/tsdb/elasticsearch/time_series_query_test.go new file mode 100644 index 00000000000..4950bc811de --- /dev/null +++ b/pkg/tsdb/elasticsearch/time_series_query_test.go @@ -0,0 +1,118 @@ +package elasticsearch + +import ( + "testing" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/tsdb" + . "github.com/smartystreets/goconvey/convey" +) + +func TestTimeSeriesQueryParser(t *testing.T) { + Convey("Test time series query parser", t, func() { + ds := &models.DataSource{} + p := newTimeSeriesQueryParser(ds) + + Convey("Should be able to parse query", func() { + json, err := simplejson.NewJson([]byte(`{ + "timeField": "@timestamp", + "query": "@metric:cpu", + "alias": "{{@hostname}} {{metric}}", + "metrics": [ + { + "field": "@value", + "id": "1", + "meta": {}, + "settings": { + "percents": [ + "90" + ] + }, + "type": "percentiles" + }, + { + "type": "count", + "field": "select field", + "id": "4", + "settings": {}, + "meta": {} + } + ], + "bucketAggs": [ + { + "fake": true, + "field": "@hostname", + "id": "3", + "settings": { + "min_doc_count": 1, + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "5m", + "min_doc_count": 0, + "trimEdges": 0 + }, + "type": "date_histogram" + } + ] + }`)) + So(err, ShouldBeNil) + tsdbQuery := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + DataSource: ds, + Model: json, + }, + }, + } + tsQuery, err := p.parse(tsdbQuery) + So(err, ShouldBeNil) + So(tsQuery.queries, ShouldHaveLength, 1) + + q := tsQuery.queries[0] + + So(q.TimeField, ShouldEqual, "@timestamp") + So(q.RawQuery, ShouldEqual, "@metric:cpu") + So(q.Alias, ShouldEqual, "{{@hostname}} {{metric}}") + + So(q.Metrics, ShouldHaveLength, 2) + So(q.Metrics[0].Field, ShouldEqual, "@value") + So(q.Metrics[0].ID, ShouldEqual, "1") + So(q.Metrics[0].Type, ShouldEqual, "percentiles") + So(q.Metrics[0].Hide, ShouldBeFalse) + So(q.Metrics[0].PipelineAggregate, ShouldEqual, "") + So(q.Metrics[0].Settings.Get("percents").MustStringArray()[0], ShouldEqual, "90") + + So(q.Metrics[1].Field, ShouldEqual, "select field") + So(q.Metrics[1].ID, ShouldEqual, "4") + So(q.Metrics[1].Type, ShouldEqual, "count") + So(q.Metrics[1].Hide, ShouldBeFalse) + So(q.Metrics[1].PipelineAggregate, ShouldEqual, "") + So(q.Metrics[1].Settings.MustMap(), ShouldBeEmpty) + + So(q.BucketAggs, ShouldHaveLength, 2) + So(q.BucketAggs[0].Field, ShouldEqual, "@hostname") + So(q.BucketAggs[0].ID, ShouldEqual, "3") + So(q.BucketAggs[0].Type, ShouldEqual, "terms") + So(q.BucketAggs[0].Settings.Get("min_doc_count").MustInt64(), ShouldEqual, 1) + So(q.BucketAggs[0].Settings.Get("order").MustString(), ShouldEqual, "desc") + So(q.BucketAggs[0].Settings.Get("orderBy").MustString(), ShouldEqual, "_term") + So(q.BucketAggs[0].Settings.Get("size").MustString(), ShouldEqual, "10") + + So(q.BucketAggs[1].Field, ShouldEqual, "@timestamp") + So(q.BucketAggs[1].ID, ShouldEqual, "2") + So(q.BucketAggs[1].Type, ShouldEqual, "date_histogram") + So(q.BucketAggs[1].Settings.Get("interval").MustString(), ShouldEqual, "5m") + So(q.BucketAggs[1].Settings.Get("min_doc_count").MustInt64(), ShouldEqual, 0) + So(q.BucketAggs[1].Settings.Get("trimEdges").MustInt64(), ShouldEqual, 0) + }) + }) +} From e171ed89102ff136f48488f38c98e71e03a77813 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 23 May 2018 14:59:12 +0200 Subject: [PATCH 10/22] elasticsearch: new simple client for communicating with elasticsearch Handles minor differences of es 2, 5 and 5.6. Implements index pattern logic. Exposes builders for building search requests. --- pkg/tsdb/elasticsearch/client/client.go | 286 +++++++++++ pkg/tsdb/elasticsearch/client/client_test.go | 215 ++++++++ .../elasticsearch/client/index_pattern.go | 312 ++++++++++++ .../client/index_pattern_test.go | 244 +++++++++ pkg/tsdb/elasticsearch/client/models.go | 304 +++++++++++ .../elasticsearch/client/search_request.go | 446 +++++++++++++++++ .../client/search_request_test.go | 471 ++++++++++++++++++ 7 files changed, 2278 insertions(+) create mode 100644 pkg/tsdb/elasticsearch/client/client.go create mode 100644 pkg/tsdb/elasticsearch/client/client_test.go create mode 100644 pkg/tsdb/elasticsearch/client/index_pattern.go create mode 100644 pkg/tsdb/elasticsearch/client/index_pattern_test.go create mode 100644 pkg/tsdb/elasticsearch/client/models.go create mode 100644 pkg/tsdb/elasticsearch/client/search_request.go create mode 100644 pkg/tsdb/elasticsearch/client/search_request_test.go diff --git a/pkg/tsdb/elasticsearch/client/client.go b/pkg/tsdb/elasticsearch/client/client.go new file mode 100644 index 00000000000..3dae343bd37 --- /dev/null +++ b/pkg/tsdb/elasticsearch/client/client.go @@ -0,0 +1,286 @@ +package es + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "path" + "strings" + "time" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/tsdb" + + "github.com/grafana/grafana/pkg/models" + "golang.org/x/net/context/ctxhttp" +) + +const loggerName = "tsdb.elasticsearch.client" + +var ( + clientLog = log.New(loggerName) + intervalCalculator = tsdb.NewIntervalCalculator(&tsdb.IntervalOptions{MinInterval: 15 * time.Second}) +) + +// Client represents a client which can interact with elasticsearch api +type Client interface { + GetVersion() int + GetTimeField() string + GetMinInterval(queryInterval string) (time.Duration, error) + ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearchResponse, error) + MultiSearch() *MultiSearchRequestBuilder +} + +// NewClient creates a new elasticsearch client +var NewClient = func(ctx context.Context, ds *models.DataSource, timeRange *tsdb.TimeRange) (Client, error) { + version, err := ds.JsonData.Get("esVersion").Int() + if err != nil { + return nil, fmt.Errorf("eleasticsearch version is required, err=%v", err) + } + + timeField, err := ds.JsonData.Get("timeField").String() + if err != nil { + return nil, fmt.Errorf("eleasticsearch time field name is required, err=%v", err) + } + + indexInterval := ds.JsonData.Get("interval").MustString() + ip, err := newIndexPattern(indexInterval, ds.Database) + if err != nil { + return nil, err + } + + indices, err := ip.GetIndices(timeRange) + if err != nil { + return nil, err + } + + bc := &baseClientImpl{ + ctx: ctx, + ds: ds, + version: version, + timeField: timeField, + indices: indices, + } + + clientLog.Debug("Creating new client", "version", version, "timeField", timeField, "indices", strings.Join(indices, ", ")) + + switch version { + case 2: + return newV2Client(bc) + case 5: + return newV5Client(bc) + case 56: + return newV56Client(bc) + } + + return nil, fmt.Errorf("elasticsearch version=%d is not supported", version) +} + +type baseClient interface { + Client + getSettings() *simplejson.Json + executeBatchRequest(uriPath string, requests []*multiRequest) (*http.Response, error) + executeRequest(method, uriPath string, body []byte) (*http.Response, error) + createMultiSearchRequests(searchRequests []*SearchRequest) []*multiRequest +} + +type baseClientImpl struct { + ctx context.Context + ds *models.DataSource + version int + timeField string + indices []string +} + +func (c *baseClientImpl) GetVersion() int { + return c.version +} + +func (c *baseClientImpl) GetTimeField() string { + return c.timeField +} + +func (c *baseClientImpl) GetMinInterval(queryInterval string) (time.Duration, error) { + return tsdb.GetIntervalFrom(c.ds, simplejson.NewFromAny(map[string]string{ + "interval": queryInterval, + }), 15*time.Second) +} + +func (c *baseClientImpl) getSettings() *simplejson.Json { + return c.ds.JsonData +} + +type multiRequest struct { + header map[string]interface{} + body interface{} +} + +func (c *baseClientImpl) executeBatchRequest(uriPath string, requests []*multiRequest) (*http.Response, error) { + payload := bytes.Buffer{} + for _, r := range requests { + reqHeader, err := json.Marshal(r.header) + if err != nil { + return nil, err + } + payload.WriteString(string(reqHeader) + "\n") + + reqBody, err := json.Marshal(r.body) + if err != nil { + return nil, err + } + payload.WriteString(string(reqBody) + "\n") + } + + return c.executeRequest(http.MethodPost, uriPath, payload.Bytes()) +} + +func (c *baseClientImpl) executeRequest(method, uriPath string, body []byte) (*http.Response, error) { + u, _ := url.Parse(c.ds.Url) + u.Path = path.Join(u.Path, uriPath) + + var req *http.Request + var err error + if method == http.MethodPost { + req, err = http.NewRequest(http.MethodPost, u.String(), bytes.NewBuffer(body)) + } else { + req, err = http.NewRequest(http.MethodGet, u.String(), nil) + } + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", "Grafana") + req.Header.Set("Content-Type", "application/json") + + if c.ds.BasicAuth { + clientLog.Debug("Request configured to use basic authentication") + req.SetBasicAuth(c.ds.BasicAuthUser, c.ds.BasicAuthPassword) + } + + if !c.ds.BasicAuth && c.ds.User != "" { + clientLog.Debug("Request configured to use basic authentication") + req.SetBasicAuth(c.ds.User, c.ds.Password) + } + + httpClient, err := c.ds.GetHttpClient() + if err != nil { + return nil, err + } + + if method == http.MethodPost { + clientLog.Debug("Executing request", "url", req.URL.String(), "method", method) + } else { + clientLog.Debug("Executing request", "url", req.URL.String(), "method", method) + } + + return ctxhttp.Do(c.ctx, httpClient, req) +} + +func (c *baseClientImpl) ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearchResponse, error) { + multiRequests := c.createMultiSearchRequests(r.Requests) + res, err := c.executeBatchRequest("_msearch", multiRequests) + if err != nil { + return nil, err + } + + var msr MultiSearchResponse + defer res.Body.Close() + dec := json.NewDecoder(res.Body) + err = dec.Decode(&msr) + if err != nil { + return nil, err + } + + clientLog.Debug("Received multisearch response", "code", res.StatusCode, "status", res.Status, "content-length", res.ContentLength) + + msr.status = res.StatusCode + + return &msr, nil +} + +func (c *baseClientImpl) createMultiSearchRequests(searchRequests []*SearchRequest) []*multiRequest { + multiRequests := []*multiRequest{} + + for _, searchReq := range searchRequests { + multiRequests = append(multiRequests, &multiRequest{ + header: map[string]interface{}{ + "search_type": "query_then_fetch", + "ignore_unavailable": true, + "index": strings.Join(c.indices, ","), + }, + body: searchReq, + }) + } + + return multiRequests +} + +type v2Client struct { + baseClient +} + +func newV2Client(bc baseClient) (*v2Client, error) { + c := v2Client{ + baseClient: bc, + } + + return &c, nil +} + +func (c *v2Client) createMultiSearchRequests(searchRequests []*SearchRequest) []*multiRequest { + multiRequests := c.baseClient.createMultiSearchRequests(searchRequests) + + for _, mr := range multiRequests { + mr.header["search_type"] = "count" + } + + return multiRequests +} + +type v5Client struct { + baseClient +} + +func newV5Client(bc baseClient) (*v5Client, error) { + c := v5Client{ + baseClient: bc, + } + + return &c, nil +} + +type v56Client struct { + *v5Client + maxConcurrentShardRequests int +} + +func newV56Client(bc baseClient) (*v56Client, error) { + v5Client := v5Client{ + baseClient: bc, + } + maxConcurrentShardRequests := bc.getSettings().Get("maxConcurrentShardRequests").MustInt(256) + + c := v56Client{ + v5Client: &v5Client, + maxConcurrentShardRequests: maxConcurrentShardRequests, + } + + return &c, nil +} + +func (c *v56Client) createMultiSearchRequests(searchRequests []*SearchRequest) []*multiRequest { + multiRequests := c.v5Client.createMultiSearchRequests(searchRequests) + + for _, mr := range multiRequests { + mr.header["max_concurrent_shard_requests"] = c.maxConcurrentShardRequests + } + + return multiRequests +} + +func (c *baseClientImpl) MultiSearch() *MultiSearchRequestBuilder { + return NewMultiSearchRequestBuilder(c.GetVersion()) +} diff --git a/pkg/tsdb/elasticsearch/client/client_test.go b/pkg/tsdb/elasticsearch/client/client_test.go new file mode 100644 index 00000000000..d557ceb28b1 --- /dev/null +++ b/pkg/tsdb/elasticsearch/client/client_test.go @@ -0,0 +1,215 @@ +package es + +import ( + "net/http" + "testing" + + "github.com/grafana/grafana/pkg/components/simplejson" + + "github.com/grafana/grafana/pkg/models" + . "github.com/smartystreets/goconvey/convey" +) + +func TestClient(t *testing.T) { + Convey("Test elasticsearch client", t, func() { + Convey("NewClient", func() { + Convey("When no version set should return error", func() { + ds := &models.DataSource{ + JsonData: simplejson.NewFromAny(make(map[string]interface{})), + } + + _, err := NewClient(nil, ds, nil) + So(err, ShouldNotBeNil) + }) + + Convey("When no time field name set should return error", func() { + ds := &models.DataSource{ + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 5, + }), + } + + _, err := NewClient(nil, ds, nil) + So(err, ShouldNotBeNil) + }) + + Convey("When unspported version set should return error", func() { + ds := &models.DataSource{ + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 6, + "timeField": "@timestamp", + }), + } + + _, err := NewClient(nil, ds, nil) + So(err, ShouldNotBeNil) + }) + + Convey("When version 2 should return v2 client", func() { + ds := &models.DataSource{ + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 2, + "timeField": "@timestamp", + }), + } + + c, err := NewClient(nil, ds, nil) + So(err, ShouldBeNil) + So(c.GetVersion(), ShouldEqual, 2) + }) + + Convey("When version 5 should return v5 client", func() { + ds := &models.DataSource{ + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 5, + "timeField": "@timestamp", + }), + } + + c, err := NewClient(nil, ds, nil) + So(err, ShouldBeNil) + So(c.GetVersion(), ShouldEqual, 5) + }) + + Convey("When version 56 should return v5.6 client", func() { + ds := &models.DataSource{ + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 56, + "timeField": "@timestamp", + }), + } + + c, err := NewClient(nil, ds, nil) + So(err, ShouldBeNil) + So(c.GetVersion(), ShouldEqual, 56) + }) + }) + + Convey("v2", func() { + ds := &models.DataSource{ + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 2, + }), + } + + c, err := newV2Client(newFakeBaseClient(ds, []string{"test-*"})) + So(err, ShouldBeNil) + So(c, ShouldNotBeNil) + + Convey("When creating multisearch requests should have correct headers", func() { + multiRequests := c.createMultiSearchRequests([]*SearchRequest{ + {Index: "test-*"}, + }) + So(multiRequests, ShouldHaveLength, 1) + header := multiRequests[0].header + So(header, ShouldHaveLength, 3) + So(header["index"], ShouldEqual, "test-*") + So(header["ignore_unavailable"], ShouldEqual, true) + So(header["search_type"], ShouldEqual, "count") + }) + }) + + Convey("v5", func() { + ds := &models.DataSource{ + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 5, + }), + } + + c, err := newV5Client(newFakeBaseClient(ds, []string{"test-*"})) + So(err, ShouldBeNil) + So(c, ShouldNotBeNil) + + Convey("When creating multisearch requests should have correct headers", func() { + multiRequests := c.createMultiSearchRequests([]*SearchRequest{ + {Index: "test-*"}, + }) + So(multiRequests, ShouldHaveLength, 1) + header := multiRequests[0].header + So(header, ShouldHaveLength, 3) + So(header["index"], ShouldEqual, "test-*") + So(header["ignore_unavailable"], ShouldEqual, true) + So(header["search_type"], ShouldEqual, "query_then_fetch") + }) + }) + + Convey("v5.6", func() { + Convey("With default settings", func() { + ds := models.DataSource{ + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 56, + }), + } + + c, err := newV56Client(newFakeBaseClient(&ds, []string{"test-*"})) + So(err, ShouldBeNil) + So(c, ShouldNotBeNil) + + Convey("When creating multisearch requests should have correct headers", func() { + multiRequests := c.createMultiSearchRequests([]*SearchRequest{ + {Index: "test-*"}, + }) + So(multiRequests, ShouldHaveLength, 1) + header := multiRequests[0].header + So(header, ShouldHaveLength, 4) + So(header["index"], ShouldEqual, "test-*") + So(header["ignore_unavailable"], ShouldEqual, true) + So(header["search_type"], ShouldEqual, "query_then_fetch") + So(header["max_concurrent_shard_requests"], ShouldEqual, 256) + }) + }) + + Convey("With custom settings", func() { + ds := models.DataSource{ + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 56, + "maxConcurrentShardRequests": 100, + }), + } + + c, err := newV56Client(newFakeBaseClient(&ds, []string{"test-*"})) + So(err, ShouldBeNil) + So(c, ShouldNotBeNil) + Convey("When creating multisearch requests should have correct headers", func() { + multiRequests := c.createMultiSearchRequests([]*SearchRequest{ + {Index: "test-*"}, + }) + So(multiRequests, ShouldHaveLength, 1) + header := multiRequests[0].header + So(header, ShouldHaveLength, 4) + So(header["index"], ShouldEqual, "test-*") + So(header["ignore_unavailable"], ShouldEqual, true) + So(header["search_type"], ShouldEqual, "query_then_fetch") + So(header["max_concurrent_shard_requests"], ShouldEqual, 100) + }) + }) + }) + }) +} + +type fakeBaseClient struct { + *baseClientImpl + ds *models.DataSource +} + +func newFakeBaseClient(ds *models.DataSource, indices []string) baseClient { + return &fakeBaseClient{ + baseClientImpl: &baseClientImpl{ + ds: ds, + indices: indices, + }, + ds: ds, + } +} + +func (c *fakeBaseClient) executeBatchRequest(uriPath string, requests []*multiRequest) (*http.Response, error) { + return nil, nil +} + +func (c *fakeBaseClient) executeRequest(method, uriPath string, body []byte) (*http.Response, error) { + return nil, nil +} + +func (c *fakeBaseClient) executeMultisearch(searchRequests []*SearchRequest) ([]*SearchResponse, error) { + return nil, nil +} diff --git a/pkg/tsdb/elasticsearch/client/index_pattern.go b/pkg/tsdb/elasticsearch/client/index_pattern.go new file mode 100644 index 00000000000..8391e902ea4 --- /dev/null +++ b/pkg/tsdb/elasticsearch/client/index_pattern.go @@ -0,0 +1,312 @@ +package es + +import ( + "fmt" + "regexp" + "strings" + "time" + + "github.com/grafana/grafana/pkg/tsdb" +) + +const ( + noInterval = "" + intervalHourly = "hourly" + intervalDaily = "daily" + intervalWeekly = "weekly" + intervalMonthly = "monthly" + intervalYearly = "yearly" +) + +type indexPattern interface { + GetIndices(timeRange *tsdb.TimeRange) ([]string, error) +} + +var newIndexPattern = func(interval string, pattern string) (indexPattern, error) { + if interval == noInterval { + return &staticIndexPattern{indexName: pattern}, nil + } + + return newDynamicIndexPattern(interval, pattern) +} + +type staticIndexPattern struct { + indexName string +} + +func (ip *staticIndexPattern) GetIndices(timeRange *tsdb.TimeRange) ([]string, error) { + return []string{ip.indexName}, nil +} + +type intervalGenerator interface { + Generate(from, to time.Time) []time.Time +} + +type dynamicIndexPattern struct { + interval string + pattern string + intervalGenerator intervalGenerator +} + +func newDynamicIndexPattern(interval, pattern string) (*dynamicIndexPattern, error) { + var generator intervalGenerator + + switch strings.ToLower(interval) { + case intervalHourly: + generator = &hourlyInterval{} + case intervalDaily: + generator = &dailyInterval{} + case intervalWeekly: + generator = &weeklyInterval{} + case intervalMonthly: + generator = &monthlyInterval{} + case intervalYearly: + generator = &yearlyInterval{} + default: + return nil, fmt.Errorf("unsupported interval '%s'", interval) + } + + return &dynamicIndexPattern{ + interval: interval, + pattern: pattern, + intervalGenerator: generator, + }, nil +} + +func (ip *dynamicIndexPattern) GetIndices(timeRange *tsdb.TimeRange) ([]string, error) { + from := timeRange.GetFromAsTimeUTC() + to := timeRange.GetToAsTimeUTC() + intervals := ip.intervalGenerator.Generate(from, to) + indices := make([]string, 0) + + for _, t := range intervals { + indices = append(indices, formatDate(t, ip.pattern)) + } + + return indices, nil +} + +type hourlyInterval struct{} + +func (i *hourlyInterval) Generate(from, to time.Time) []time.Time { + intervals := []time.Time{} + start := time.Date(from.Year(), from.Month(), from.Day(), from.Hour(), 0, 0, 0, time.UTC) + end := time.Date(to.Year(), to.Month(), to.Day(), to.Hour(), 0, 0, 0, time.UTC) + + intervals = append(intervals, start) + + for start.Before(end) { + start = start.Add(time.Hour) + intervals = append(intervals, start) + } + + return intervals +} + +type dailyInterval struct{} + +func (i *dailyInterval) Generate(from, to time.Time) []time.Time { + intervals := []time.Time{} + start := time.Date(from.Year(), from.Month(), from.Day(), 0, 0, 0, 0, time.UTC) + end := time.Date(to.Year(), to.Month(), to.Day(), 0, 0, 0, 0, time.UTC) + + intervals = append(intervals, start) + + for start.Before(end) { + start = start.Add(24 * time.Hour) + intervals = append(intervals, start) + } + + return intervals +} + +type weeklyInterval struct{} + +func (i *weeklyInterval) Generate(from, to time.Time) []time.Time { + intervals := []time.Time{} + start := time.Date(from.Year(), from.Month(), from.Day(), 0, 0, 0, 0, time.UTC) + end := time.Date(to.Year(), to.Month(), to.Day(), 0, 0, 0, 0, time.UTC) + + for start.Weekday() != time.Monday { + start = start.Add(-24 * time.Hour) + } + + for end.Weekday() != time.Monday { + end = end.Add(-24 * time.Hour) + } + + year, week := start.ISOWeek() + intervals = append(intervals, start) + + for start.Before(end) { + start = start.Add(24 * time.Hour) + nextYear, nextWeek := start.ISOWeek() + if nextYear != year || nextWeek != week { + intervals = append(intervals, start) + } + year = nextYear + week = nextWeek + } + + return intervals +} + +type monthlyInterval struct{} + +func (i *monthlyInterval) Generate(from, to time.Time) []time.Time { + intervals := []time.Time{} + start := time.Date(from.Year(), from.Month(), 1, 0, 0, 0, 0, time.UTC) + end := time.Date(to.Year(), to.Month(), 1, 0, 0, 0, 0, time.UTC) + + month := start.Month() + intervals = append(intervals, start) + + for start.Before(end) { + start = start.Add(24 * time.Hour) + nextMonth := start.Month() + if nextMonth != month { + intervals = append(intervals, start) + } + month = nextMonth + } + + return intervals +} + +type yearlyInterval struct{} + +func (i *yearlyInterval) Generate(from, to time.Time) []time.Time { + intervals := []time.Time{} + start := time.Date(from.Year(), 1, 1, 0, 0, 0, 0, time.UTC) + end := time.Date(to.Year(), 1, 1, 0, 0, 0, 0, time.UTC) + + year := start.Year() + intervals = append(intervals, start) + + for start.Before(end) { + start = start.Add(24 * time.Hour) + nextYear := start.Year() + if nextYear != year { + intervals = append(intervals, start) + } + year = nextYear + } + + return intervals +} + +var datePatternRegex = regexp.MustCompile("(LT|LL?L?L?|l{1,4}|Mo|MM?M?M?|Do|DDDo|DD?D?D?|ddd?d?|do?|w[o|w]?|W[o|W]?|YYYYY|YYYY|YY|gg(ggg?)?|GG(GGG?)?|e|E|a|A|hh?|HH?|mm?|ss?|SS?S?|X|zz?|ZZ?|Q)") + +var datePatternReplacements = map[string]string{ + "M": "1", // stdNumMonth 1 2 ... 11 12 + "MM": "01", // stdZeroMonth 01 02 ... 11 12 + "MMM": "Jan", // stdMonth Jan Feb ... Nov Dec + "MMMM": "January", // stdLongMonth January February ... November December + "D": "2", // stdDay 1 2 ... 30 30 + "DD": "02", // stdZeroDay 01 02 ... 30 31 + "DDD": "", // Day of the year 1 2 ... 364 365 + "DDDD": "", // Day of the year 001 002 ... 364 365 @todo**** + "d": "", // Numeric representation of day of the week 0 1 ... 5 6 + "dd": "Mon", // ***Su Mo ... Fr Sa @todo + "ddd": "Mon", // Sun Mon ... Fri Sat + "dddd": "Monday", // stdLongWeekDay Sunday Monday ... Friday Saturday + "e": "", // Numeric representation of day of the week 0 1 ... 5 6 @todo + "E": "", // ISO-8601 numeric representation of the day of the week (added in PHP 5.1.0) 1 2 ... 6 7 @todo + "w": "", // 1 2 ... 52 53 + "ww": "", // ***01 02 ... 52 53 @todo + "W": "", // 1 2 ... 52 53 + "WW": "", // ***01 02 ... 52 53 @todo + "YY": "06", // stdYear 70 71 ... 29 30 + "YYYY": "2006", // stdLongYear 1970 1971 ... 2029 2030 + "gg": "", // ISO-8601 year number 70 71 ... 29 30 + "gggg": "", // ***1970 1971 ... 2029 2030 + "GG": "", //70 71 ... 29 30 + "GGGG": "", // ***1970 1971 ... 2029 2030 + "Q": "", // 1, 2, 3, 4 + "A": "PM", // stdPM AM PM + "a": "pm", // stdpm am pm + "H": "", // stdHour 0 1 ... 22 23 + "HH": "15", // 00 01 ... 22 23 + "h": "3", // stdHour12 1 2 ... 11 12 + "hh": "03", // stdZeroHour12 01 02 ... 11 12 + "m": "4", // stdZeroMinute 0 1 ... 58 59 + "mm": "04", // stdZeroMinute 00 01 ... 58 59 + "s": "5", // stdSecond 0 1 ... 58 59 + "ss": "05", // stdZeroSecond ***00 01 ... 58 59 + "z": "MST", //EST CST ... MST PST + "zz": "MST", //EST CST ... MST PST + "Z": "Z07:00", // stdNumColonTZ -07:00 -06:00 ... +06:00 +07:00 + "ZZ": "-0700", // stdNumTZ -0700 -0600 ... +0600 +0700 + "X": "", // Seconds since unix epoch 1360013296 + "LT": "3:04 PM", // 8:30 PM + "L": "01/02/2006", //09/04/1986 + "l": "1/2/2006", //9/4/1986 + "ll": "Jan 2 2006", //Sep 4 1986 + "lll": "Jan 2 2006 3:04 PM", //Sep 4 1986 8:30 PM + "llll": "Mon, Jan 2 2006 3:04 PM", //Thu, Sep 4 1986 8:30 PM +} + +func formatDate(t time.Time, pattern string) string { + var datePattern string + parts := strings.Split(strings.TrimLeft(pattern, "["), "]") + base := parts[0] + if len(parts) == 2 { + datePattern = parts[1] + } else { + datePattern = base + base = "" + } + + formatted := t.Format(patternToLayout(datePattern)) + + if strings.Contains(formatted, "", fmt.Sprintf("%d", isoYear), -1) + formatted = strings.Replace(formatted, "", isoYearShort, -1) + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", isoWeek), -1) + + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", t.Unix()), -1) + + day := t.Weekday() + dayOfWeekIso := int(day) + if day == time.Sunday { + dayOfWeekIso = 7 + } + + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", day), -1) + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", dayOfWeekIso), -1) + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", t.YearDay()), -1) + + quarter := 4 + + switch t.Month() { + case time.January, time.February, time.March: + quarter = 1 + case time.April, time.May, time.June: + quarter = 2 + case time.July, time.August, time.September: + quarter = 3 + } + + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", quarter), -1) + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", t.Hour()), -1) + } + + return base + formatted +} + +func patternToLayout(pattern string) string { + var match [][]string + if match = datePatternRegex.FindAllStringSubmatch(pattern, -1); match == nil { + return pattern + } + + for i := range match { + if replace, ok := datePatternReplacements[match[i][0]]; ok { + pattern = strings.Replace(pattern, match[i][0], replace, 1) + } + } + + return pattern +} diff --git a/pkg/tsdb/elasticsearch/client/index_pattern_test.go b/pkg/tsdb/elasticsearch/client/index_pattern_test.go new file mode 100644 index 00000000000..3bd823d8c87 --- /dev/null +++ b/pkg/tsdb/elasticsearch/client/index_pattern_test.go @@ -0,0 +1,244 @@ +package es + +import ( + "fmt" + "testing" + "time" + + "github.com/grafana/grafana/pkg/tsdb" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestIndexPattern(t *testing.T) { + Convey("Static index patterns", t, func() { + indexPatternScenario(noInterval, "data-*", nil, func(indices []string) { + So(indices, ShouldHaveLength, 1) + So(indices[0], ShouldEqual, "data-*") + }) + + indexPatternScenario(noInterval, "es-index-name", nil, func(indices []string) { + So(indices, ShouldHaveLength, 1) + So(indices[0], ShouldEqual, "es-index-name") + }) + }) + + Convey("Dynamic index patterns", t, func() { + from := fmt.Sprintf("%d", time.Date(2018, 5, 15, 17, 50, 0, 0, time.UTC).UnixNano()/int64(time.Millisecond)) + to := fmt.Sprintf("%d", time.Date(2018, 5, 15, 17, 55, 0, 0, time.UTC).UnixNano()/int64(time.Millisecond)) + + indexPatternScenario(intervalHourly, "[data-]YYYY.MM.DD.HH", tsdb.NewTimeRange(from, to), func(indices []string) { + //So(indices, ShouldHaveLength, 1) + So(indices[0], ShouldEqual, "data-2018.05.15.17") + }) + + indexPatternScenario(intervalDaily, "[data-]YYYY.MM.DD", tsdb.NewTimeRange(from, to), func(indices []string) { + So(indices, ShouldHaveLength, 1) + So(indices[0], ShouldEqual, "data-2018.05.15") + }) + + indexPatternScenario(intervalWeekly, "[data-]GGGG.WW", tsdb.NewTimeRange(from, to), func(indices []string) { + So(indices, ShouldHaveLength, 1) + So(indices[0], ShouldEqual, "data-2018.20") + }) + + indexPatternScenario(intervalMonthly, "[data-]YYYY.MM", tsdb.NewTimeRange(from, to), func(indices []string) { + So(indices, ShouldHaveLength, 1) + So(indices[0], ShouldEqual, "data-2018.05") + }) + + indexPatternScenario(intervalYearly, "[data-]YYYY", tsdb.NewTimeRange(from, to), func(indices []string) { + So(indices, ShouldHaveLength, 1) + So(indices[0], ShouldEqual, "data-2018") + }) + }) + + Convey("Hourly interval", t, func() { + Convey("Should return 1 interval", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 1, 1, 23, 6, 0, 0, time.UTC) + intervals := (&hourlyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 1) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 23, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 2 intervals", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 1, 2, 0, 6, 0, 0, time.UTC) + intervals := (&hourlyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 2) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 23, 0, 0, 0, time.UTC)) + So(intervals[1], ShouldEqual, time.Date(2018, 1, 2, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 10 intervals", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 1, 2, 8, 6, 0, 0, time.UTC) + intervals := (&hourlyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 10) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 23, 0, 0, 0, time.UTC)) + So(intervals[4], ShouldEqual, time.Date(2018, 1, 2, 3, 0, 0, 0, time.UTC)) + So(intervals[9], ShouldEqual, time.Date(2018, 1, 2, 8, 0, 0, 0, time.UTC)) + }) + }) + + Convey("Daily interval", t, func() { + Convey("Should return 1 day", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 1, 1, 23, 6, 0, 0, time.UTC) + intervals := (&dailyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 1) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 2 days", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 1, 2, 0, 6, 0, 0, time.UTC) + intervals := (&dailyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 2) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + So(intervals[1], ShouldEqual, time.Date(2018, 1, 2, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 32 days", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 2, 1, 8, 6, 0, 0, time.UTC) + intervals := (&dailyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 32) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + So(intervals[30], ShouldEqual, time.Date(2018, 1, 31, 0, 0, 0, 0, time.UTC)) + So(intervals[31], ShouldEqual, time.Date(2018, 2, 1, 0, 0, 0, 0, time.UTC)) + }) + }) + + Convey("Weekly interval", t, func() { + Convey("Should return 1 week (1)", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 1, 1, 23, 6, 0, 0, time.UTC) + intervals := (&weeklyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 1) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 1 week (2)", func() { + from := time.Date(2017, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2017, 1, 1, 23, 6, 0, 0, time.UTC) + intervals := (&weeklyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 1) + So(intervals[0], ShouldEqual, time.Date(2016, 12, 26, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 2 weeks (1)", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 1, 10, 23, 6, 0, 0, time.UTC) + intervals := (&weeklyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 2) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + So(intervals[1], ShouldEqual, time.Date(2018, 1, 8, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 2 weeks (2)", func() { + from := time.Date(2017, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2017, 1, 8, 23, 6, 0, 0, time.UTC) + intervals := (&weeklyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 2) + So(intervals[0], ShouldEqual, time.Date(2016, 12, 26, 0, 0, 0, 0, time.UTC)) + So(intervals[1], ShouldEqual, time.Date(2017, 1, 2, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 3 weeks (1)", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 1, 21, 23, 6, 0, 0, time.UTC) + intervals := (&weeklyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 3) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + So(intervals[1], ShouldEqual, time.Date(2018, 1, 8, 0, 0, 0, 0, time.UTC)) + So(intervals[2], ShouldEqual, time.Date(2018, 1, 15, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 3 weeks (2)", func() { + from := time.Date(2017, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2017, 1, 9, 23, 6, 0, 0, time.UTC) + intervals := (&weeklyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 3) + So(intervals[0], ShouldEqual, time.Date(2016, 12, 26, 0, 0, 0, 0, time.UTC)) + So(intervals[1], ShouldEqual, time.Date(2017, 1, 2, 0, 0, 0, 0, time.UTC)) + So(intervals[2], ShouldEqual, time.Date(2017, 1, 9, 0, 0, 0, 0, time.UTC)) + }) + }) + + Convey("Monthly interval", t, func() { + Convey("Should return 1 month", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 1, 1, 23, 6, 0, 0, time.UTC) + intervals := (&monthlyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 1) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 2 months", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 2, 2, 0, 6, 0, 0, time.UTC) + intervals := (&monthlyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 2) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + So(intervals[1], ShouldEqual, time.Date(2018, 2, 1, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 14 months", func() { + from := time.Date(2017, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 2, 1, 8, 6, 0, 0, time.UTC) + intervals := (&monthlyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 14) + So(intervals[0], ShouldEqual, time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC)) + So(intervals[13], ShouldEqual, time.Date(2018, 2, 1, 0, 0, 0, 0, time.UTC)) + }) + }) + + Convey("Yearly interval", t, func() { + Convey("Should return 1 year (hour diff)", func() { + from := time.Date(2018, 2, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 2, 1, 23, 6, 0, 0, time.UTC) + intervals := (&yearlyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 1) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 1 year (month diff)", func() { + from := time.Date(2018, 2, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 12, 31, 23, 59, 59, 0, time.UTC) + intervals := (&yearlyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 1) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 2 years", func() { + from := time.Date(2018, 2, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2019, 1, 1, 23, 59, 59, 0, time.UTC) + intervals := (&yearlyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 2) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + So(intervals[1], ShouldEqual, time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 5 years", func() { + from := time.Date(2014, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 11, 1, 23, 59, 59, 0, time.UTC) + intervals := (&yearlyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 5) + So(intervals[0], ShouldEqual, time.Date(2014, 1, 1, 0, 0, 0, 0, time.UTC)) + So(intervals[4], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + }) + }) +} + +func indexPatternScenario(interval string, pattern string, timeRange *tsdb.TimeRange, fn func(indices []string)) { + Convey(fmt.Sprintf("Index pattern (interval=%s, index=%s", interval, pattern), func() { + ip, err := newIndexPattern(interval, pattern) + So(err, ShouldBeNil) + So(ip, ShouldNotBeNil) + indices, err := ip.GetIndices(timeRange) + So(err, ShouldBeNil) + fn(indices) + }) +} diff --git a/pkg/tsdb/elasticsearch/client/models.go b/pkg/tsdb/elasticsearch/client/models.go new file mode 100644 index 00000000000..3c86dcce825 --- /dev/null +++ b/pkg/tsdb/elasticsearch/client/models.go @@ -0,0 +1,304 @@ +package es + +import ( + "encoding/json" +) + +// SearchRequest represents a search request +type SearchRequest struct { + Index string + Size int + Sort map[string]interface{} + Query *Query + Aggs AggArray + CustomProps map[string]interface{} +} + +// MarshalJSON returns the JSON encoding of the request. +func (r *SearchRequest) MarshalJSON() ([]byte, error) { + root := make(map[string]interface{}) + + root["size"] = r.Size + if len(r.Sort) > 0 { + root["sort"] = r.Sort + } + + for key, value := range r.CustomProps { + root[key] = value + } + + root["query"] = r.Query + + if len(r.Aggs) > 0 { + root["aggs"] = r.Aggs + } + + return json.Marshal(root) +} + +// SearchResponseHits represents search response hits +type SearchResponseHits struct { + Hits []map[string]interface{} + Total int64 +} + +// SearchResponse represents a search response +type SearchResponse struct { + Error map[string]interface{} `json:"error"` + Aggregations map[string]interface{} `json:"aggregations"` + Hits *SearchResponseHits `json:"hits"` +} + +// func (r *Response) getErrMsg() string { +// var msg bytes.Buffer +// errJson := simplejson.NewFromAny(r.Err) +// errType, err := errJson.Get("type").String() +// if err == nil { +// msg.WriteString(fmt.Sprintf("type:%s", errType)) +// } + +// reason, err := errJson.Get("type").String() +// if err == nil { +// msg.WriteString(fmt.Sprintf("reason:%s", reason)) +// } +// return msg.String() +// } + +// MultiSearchRequest represents a multi search request +type MultiSearchRequest struct { + Requests []*SearchRequest +} + +// MultiSearchResponse represents a multi search response +type MultiSearchResponse struct { + status int `json:"status,omitempty"` + Responses []*SearchResponse `json:"responses"` +} + +// Query represents a query +type Query struct { + Bool *BoolQuery `json:"bool"` +} + +// BoolQuery represents a bool query +type BoolQuery struct { + Filters []Filter +} + +// NewBoolQuery create a new bool query +func NewBoolQuery() *BoolQuery { + return &BoolQuery{Filters: make([]Filter, 0)} +} + +// MarshalJSON returns the JSON encoding of the boolean query. +func (q *BoolQuery) MarshalJSON() ([]byte, error) { + root := make(map[string]interface{}) + + if len(q.Filters) > 0 { + if len(q.Filters) == 1 { + root["filter"] = q.Filters[0] + } else { + root["filter"] = q.Filters + } + } + return json.Marshal(root) +} + +// Filter represents a search filter +type Filter interface{} + +// QueryStringFilter represents a query string search filter +type QueryStringFilter struct { + Filter + Query string + AnalyzeWildcard bool +} + +// MarshalJSON returns the JSON encoding of the query string filter. +func (f *QueryStringFilter) MarshalJSON() ([]byte, error) { + root := map[string]interface{}{ + "query_string": map[string]interface{}{ + "query": f.Query, + "analyze_wildcard": f.AnalyzeWildcard, + }, + } + + return json.Marshal(root) +} + +// RangeFilter represents a range search filter +type RangeFilter struct { + Filter + Key string + Gte string + Lte string + Format string +} + +// DateFormatEpochMS represents a date format of epoch milliseconds (epoch_millis) +const DateFormatEpochMS = "epoch_millis" + +// MarshalJSON returns the JSON encoding of the query string filter. +func (f *RangeFilter) MarshalJSON() ([]byte, error) { + root := map[string]map[string]map[string]interface{}{ + "range": { + f.Key: { + "lte": f.Lte, + "gte": f.Gte, + }, + }, + } + + if f.Format != "" { + root["range"][f.Key]["format"] = f.Format + } + + return json.Marshal(root) +} + +// Aggregation represents an aggregation +type Aggregation interface{} + +// Agg represents a key and aggregation +type Agg struct { + Key string + Aggregation *aggContainer +} + +// MarshalJSON returns the JSON encoding of the agg +func (a *Agg) MarshalJSON() ([]byte, error) { + root := map[string]interface{}{ + a.Key: a.Aggregation, + } + + return json.Marshal(root) +} + +// AggArray represents a collection of key/aggregation pairs +type AggArray []*Agg + +// MarshalJSON returns the JSON encoding of the agg +func (a AggArray) MarshalJSON() ([]byte, error) { + aggsMap := make(map[string]Aggregation) + + for _, subAgg := range a { + aggsMap[subAgg.Key] = subAgg.Aggregation + } + + return json.Marshal(aggsMap) +} + +type aggContainer struct { + Type string + Aggregation Aggregation + Aggs AggArray +} + +// MarshalJSON returns the JSON encoding of the aggregation container +func (a *aggContainer) MarshalJSON() ([]byte, error) { + root := map[string]interface{}{ + a.Type: a.Aggregation, + } + + if len(a.Aggs) > 0 { + root["aggs"] = a.Aggs + } + + return json.Marshal(root) +} + +type aggDef struct { + key string + aggregation *aggContainer + builders []AggBuilder +} + +func newAggDef(key string, aggregation *aggContainer) *aggDef { + return &aggDef{ + key: key, + aggregation: aggregation, + builders: make([]AggBuilder, 0), + } +} + +// HistogramAgg represents a histogram aggregation +type HistogramAgg struct { + Interval int `json:"interval,omitempty"` + Field string `json:"field"` + MinDocCount int `json:"min_doc_count"` + Missing *int `json:"missing,omitempty"` +} + +// DateHistogramAgg represents a date histogram aggregation +type DateHistogramAgg struct { + Field string `json:"field"` + Interval string `json:"interval,omitempty"` + MinDocCount int `json:"min_doc_count"` + Missing *string `json:"missing,omitempty"` + ExtendedBounds *ExtendedBounds `json:"extended_bounds"` + Format string `json:"format"` +} + +// FiltersAggregation represents a filters aggregation +type FiltersAggregation struct { + Filters map[string]interface{} `json:"filters"` +} + +// TermsAggregation represents a terms aggregation +type TermsAggregation struct { + Field string `json:"field"` + Size int `json:"size"` + Order map[string]interface{} `json:"order"` + MinDocCount *int `json:"min_doc_count,omitempty"` + Missing *string `json:"missing,omitempty"` +} + +// ExtendedBounds represents extended bounds +type ExtendedBounds struct { + Min string `json:"min"` + Max string `json:"max"` +} + +// GeoHashGridAggregation represents a geo hash grid aggregation +type GeoHashGridAggregation struct { + Field string `json:"field"` + Precision int `json:"precision"` +} + +// MetricAggregation represents a metric aggregation +type MetricAggregation struct { + Field string + Settings map[string]interface{} +} + +// MarshalJSON returns the JSON encoding of the metric aggregation +func (a *MetricAggregation) MarshalJSON() ([]byte, error) { + root := map[string]interface{}{ + "field": a.Field, + } + + for k, v := range a.Settings { + root[k] = v + } + + return json.Marshal(root) +} + +// PipelineAggregation represents a metric aggregation +type PipelineAggregation struct { + BucketPath string + Settings map[string]interface{} +} + +// MarshalJSON returns the JSON encoding of the pipeline aggregation +func (a *PipelineAggregation) MarshalJSON() ([]byte, error) { + root := map[string]interface{}{ + "bucket_path": a.BucketPath, + } + + for k, v := range a.Settings { + root[k] = v + } + + return json.Marshal(root) +} diff --git a/pkg/tsdb/elasticsearch/client/search_request.go b/pkg/tsdb/elasticsearch/client/search_request.go new file mode 100644 index 00000000000..a582d8ec247 --- /dev/null +++ b/pkg/tsdb/elasticsearch/client/search_request.go @@ -0,0 +1,446 @@ +package es + +import ( + "strings" +) + +// SearchRequestBuilder represents a builder which can build a search request +type SearchRequestBuilder struct { + version int + index string + size int + sort map[string]interface{} + queryBuilder *QueryBuilder + aggBuilders []AggBuilder + customProps map[string]interface{} +} + +// NewSearchRequestBuilder create a new search request builder +func NewSearchRequestBuilder(version int) *SearchRequestBuilder { + builder := &SearchRequestBuilder{ + version: version, + sort: make(map[string]interface{}), + customProps: make(map[string]interface{}), + aggBuilders: make([]AggBuilder, 0), + } + return builder +} + +// Build builds and return a search request +func (b *SearchRequestBuilder) Build() (*SearchRequest, error) { + sr := SearchRequest{ + Index: b.index, + Size: b.size, + Sort: b.sort, + CustomProps: b.customProps, + } + + if b.queryBuilder != nil { + q, err := b.queryBuilder.Build() + if err != nil { + return nil, err + } + sr.Query = q + } + + if len(b.aggBuilders) > 0 { + sr.Aggs = make(AggArray, 0) + + for _, ab := range b.aggBuilders { + aggArray, err := ab.Build() + if err != nil { + return nil, err + } + for _, agg := range aggArray { + sr.Aggs = append(sr.Aggs, agg) + } + } + } + + return &sr, nil +} + +// Size sets the size of the search request +func (b *SearchRequestBuilder) Size(size int) *SearchRequestBuilder { + b.size = size + return b +} + +// SortDesc adds a sort to the search request +func (b *SearchRequestBuilder) SortDesc(field, unmappedType string) *SearchRequestBuilder { + props := map[string]string{ + "order": "desc", + } + + if unmappedType != "" { + props["unmapped_type"] = unmappedType + } + + b.sort[field] = props + + return b +} + +// AddDocValueField adds a doc value field to the search request +func (b *SearchRequestBuilder) AddDocValueField(field string) *SearchRequestBuilder { + // fields field not supported on version >= 5 + if b.version < 5 { + b.customProps["fields"] = []string{"*", "_source"} + } + + b.customProps["script_fields"] = make(map[string]interface{}) + + if b.version < 5 { + b.customProps["fielddata_fields"] = []string{field} + } else { + b.customProps["docvalue_fields"] = []string{field} + } + + return b +} + +// Query creates and return a query builder +func (b *SearchRequestBuilder) Query() *QueryBuilder { + if b.queryBuilder == nil { + b.queryBuilder = NewQueryBuilder() + } + return b.queryBuilder +} + +// Agg initaite and returns a new aggregation builder +func (b *SearchRequestBuilder) Agg() AggBuilder { + aggBuilder := newAggBuilder() + b.aggBuilders = append(b.aggBuilders, aggBuilder) + return aggBuilder +} + +// MultiSearchRequestBuilder represents a builder which can build a multi search request +type MultiSearchRequestBuilder struct { + version int + requestBuilders []*SearchRequestBuilder +} + +// NewMultiSearchRequestBuilder creates a new multi search request builder +func NewMultiSearchRequestBuilder(version int) *MultiSearchRequestBuilder { + return &MultiSearchRequestBuilder{ + version: version, + } +} + +// Search initiates and returns a new search request builder +func (m *MultiSearchRequestBuilder) Search() *SearchRequestBuilder { + b := NewSearchRequestBuilder(m.version) + m.requestBuilders = append(m.requestBuilders, b) + return b +} + +// Build builds and return a multi search request +func (m *MultiSearchRequestBuilder) Build() (*MultiSearchRequest, error) { + requests := []*SearchRequest{} + for _, sb := range m.requestBuilders { + searchRequest, err := sb.Build() + if err != nil { + return nil, err + } + requests = append(requests, searchRequest) + } + + return &MultiSearchRequest{ + Requests: requests, + }, nil +} + +// QueryBuilder represents a query builder +type QueryBuilder struct { + boolQueryBuilder *BoolQueryBuilder +} + +// NewQueryBuilder create a new query builder +func NewQueryBuilder() *QueryBuilder { + return &QueryBuilder{} +} + +// Build builds and return a query builder +func (b *QueryBuilder) Build() (*Query, error) { + q := Query{} + + if b.boolQueryBuilder != nil { + b, err := b.boolQueryBuilder.Build() + if err != nil { + return nil, err + } + q.Bool = b + } + + return &q, nil +} + +// Bool creates and return a query builder +func (b *QueryBuilder) Bool() *BoolQueryBuilder { + if b.boolQueryBuilder == nil { + b.boolQueryBuilder = NewBoolQueryBuilder() + } + return b.boolQueryBuilder +} + +// BoolQueryBuilder represents a bool query builder +type BoolQueryBuilder struct { + filterQueryBuilder *FilterQueryBuilder +} + +// NewBoolQueryBuilder create a new bool query builder +func NewBoolQueryBuilder() *BoolQueryBuilder { + return &BoolQueryBuilder{} +} + +// Filter creates and return a filter query builder +func (b *BoolQueryBuilder) Filter() *FilterQueryBuilder { + if b.filterQueryBuilder == nil { + b.filterQueryBuilder = NewFilterQueryBuilder() + } + return b.filterQueryBuilder +} + +// Build builds and return a bool query builder +func (b *BoolQueryBuilder) Build() (*BoolQuery, error) { + boolQuery := BoolQuery{} + + if b.filterQueryBuilder != nil { + filters, err := b.filterQueryBuilder.Build() + if err != nil { + return nil, err + } + boolQuery.Filters = filters + } + + return &boolQuery, nil +} + +// FilterQueryBuilder represents a filter query builder +type FilterQueryBuilder struct { + filters []Filter +} + +// NewFilterQueryBuilder creates a new filter query builder +func NewFilterQueryBuilder() *FilterQueryBuilder { + return &FilterQueryBuilder{ + filters: make([]Filter, 0), + } +} + +// Build builds and return a filter query builder +func (b *FilterQueryBuilder) Build() ([]Filter, error) { + return b.filters, nil +} + +// AddDateRangeFilter adds a new time range filter +func (b *FilterQueryBuilder) AddDateRangeFilter(timeField, lte, gte, format string) *FilterQueryBuilder { + b.filters = append(b.filters, &RangeFilter{ + Key: timeField, + Lte: lte, + Gte: gte, + Format: format, + }) + return b +} + +// AddQueryStringFilter adds a new query string filter +func (b *FilterQueryBuilder) AddQueryStringFilter(querystring string, analyseWildcard bool) *FilterQueryBuilder { + if len(strings.TrimSpace(querystring)) == 0 { + return b + } + + b.filters = append(b.filters, &QueryStringFilter{ + Query: querystring, + AnalyzeWildcard: analyseWildcard, + }) + return b +} + +// AggBuilder represents an aggregation builder +type AggBuilder interface { + Histogram(key, field string, fn func(a *HistogramAgg, b AggBuilder)) AggBuilder + DateHistogram(key, field string, fn func(a *DateHistogramAgg, b AggBuilder)) AggBuilder + Terms(key, field string, fn func(a *TermsAggregation, b AggBuilder)) AggBuilder + Filters(key string, fn func(a *FiltersAggregation, b AggBuilder)) AggBuilder + GeoHashGrid(key, field string, fn func(a *GeoHashGridAggregation, b AggBuilder)) AggBuilder + Metric(key, metricType, field string, fn func(a *MetricAggregation)) AggBuilder + Pipeline(key, pipelineType, bucketPath string, fn func(a *PipelineAggregation)) AggBuilder + Build() (AggArray, error) +} + +type aggBuilderImpl struct { + AggBuilder + aggDefs []*aggDef +} + +func newAggBuilder() *aggBuilderImpl { + return &aggBuilderImpl{ + aggDefs: make([]*aggDef, 0), + } +} + +func (b *aggBuilderImpl) Build() (AggArray, error) { + aggs := make(AggArray, 0) + + for _, aggDef := range b.aggDefs { + agg := &Agg{ + Key: aggDef.key, + Aggregation: aggDef.aggregation, + } + + for _, cb := range aggDef.builders { + childAggs, err := cb.Build() + if err != nil { + return nil, err + } + + for _, childAgg := range childAggs { + agg.Aggregation.Aggs = append(agg.Aggregation.Aggs, childAgg) + } + } + + aggs = append(aggs, agg) + } + + return aggs, nil +} + +func (b *aggBuilderImpl) Histogram(key, field string, fn func(a *HistogramAgg, b AggBuilder)) AggBuilder { + innerAgg := &HistogramAgg{ + Field: field, + } + aggDef := newAggDef(key, &aggContainer{ + Type: "histogram", + Aggregation: innerAgg, + }) + + if fn != nil { + builder := newAggBuilder() + aggDef.builders = append(aggDef.builders, builder) + fn(innerAgg, builder) + } + + b.aggDefs = append(b.aggDefs, aggDef) + + return b +} + +func (b *aggBuilderImpl) DateHistogram(key, field string, fn func(a *DateHistogramAgg, b AggBuilder)) AggBuilder { + innerAgg := &DateHistogramAgg{ + Field: field, + } + aggDef := newAggDef(key, &aggContainer{ + Type: "date_histogram", + Aggregation: innerAgg, + }) + + if fn != nil { + builder := newAggBuilder() + aggDef.builders = append(aggDef.builders, builder) + fn(innerAgg, builder) + } + + b.aggDefs = append(b.aggDefs, aggDef) + + return b +} + +func (b *aggBuilderImpl) Terms(key, field string, fn func(a *TermsAggregation, b AggBuilder)) AggBuilder { + innerAgg := &TermsAggregation{ + Field: field, + Order: make(map[string]interface{}), + } + aggDef := newAggDef(key, &aggContainer{ + Type: "terms", + Aggregation: innerAgg, + }) + + if fn != nil { + builder := newAggBuilder() + aggDef.builders = append(aggDef.builders, builder) + fn(innerAgg, builder) + } + + b.aggDefs = append(b.aggDefs, aggDef) + + return b +} + +func (b *aggBuilderImpl) Filters(key string, fn func(a *FiltersAggregation, b AggBuilder)) AggBuilder { + innerAgg := &FiltersAggregation{ + Filters: make(map[string]interface{}), + } + aggDef := newAggDef(key, &aggContainer{ + Type: "filters", + Aggregation: innerAgg, + }) + if fn != nil { + builder := newAggBuilder() + aggDef.builders = append(aggDef.builders, builder) + fn(innerAgg, builder) + } + + b.aggDefs = append(b.aggDefs, aggDef) + + return b +} + +func (b *aggBuilderImpl) GeoHashGrid(key, field string, fn func(a *GeoHashGridAggregation, b AggBuilder)) AggBuilder { + innerAgg := &GeoHashGridAggregation{ + Field: field, + Precision: 5, + } + aggDef := newAggDef(key, &aggContainer{ + Type: "geohash_grid", + Aggregation: innerAgg, + }) + + if fn != nil { + builder := newAggBuilder() + aggDef.builders = append(aggDef.builders, builder) + fn(innerAgg, builder) + } + + b.aggDefs = append(b.aggDefs, aggDef) + + return b +} + +func (b *aggBuilderImpl) Metric(key, metricType, field string, fn func(a *MetricAggregation)) AggBuilder { + innerAgg := &MetricAggregation{ + Field: field, + Settings: make(map[string]interface{}), + } + aggDef := newAggDef(key, &aggContainer{ + Type: metricType, + Aggregation: innerAgg, + }) + + if fn != nil { + fn(innerAgg) + } + + b.aggDefs = append(b.aggDefs, aggDef) + + return b +} + +func (b *aggBuilderImpl) Pipeline(key, pipelineType, bucketPath string, fn func(a *PipelineAggregation)) AggBuilder { + innerAgg := &PipelineAggregation{ + BucketPath: bucketPath, + Settings: make(map[string]interface{}), + } + aggDef := newAggDef(key, &aggContainer{ + Type: pipelineType, + Aggregation: innerAgg, + }) + + if fn != nil { + fn(innerAgg) + } + + b.aggDefs = append(b.aggDefs, aggDef) + + return b +} diff --git a/pkg/tsdb/elasticsearch/client/search_request_test.go b/pkg/tsdb/elasticsearch/client/search_request_test.go new file mode 100644 index 00000000000..d93f8826442 --- /dev/null +++ b/pkg/tsdb/elasticsearch/client/search_request_test.go @@ -0,0 +1,471 @@ +package es + +import ( + "encoding/json" + "testing" + + "github.com/grafana/grafana/pkg/components/simplejson" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestSearchRequest(t *testing.T) { + Convey("Test elasticsearch search request", t, func() { + timeField := "@timestamp" + Convey("Given new search request builder for es version 5", func() { + b := NewSearchRequestBuilder(5) + + Convey("When building search request", func() { + sr, err := b.Build() + So(err, ShouldBeNil) + + Convey("Should have size of zero", func() { + So(sr.Size, ShouldEqual, 0) + }) + + Convey("Should have no sorting", func() { + So(sr.Sort, ShouldHaveLength, 0) + }) + + Convey("When marshal to JSON should generate correct json", func() { + body, err := json.Marshal(sr) + So(err, ShouldBeNil) + json, err := simplejson.NewJson([]byte(body)) + So(err, ShouldBeNil) + So(json.Get("size").MustInt(500), ShouldEqual, 0) + So(json.Get("sort").Interface(), ShouldBeNil) + So(json.Get("aggs").Interface(), ShouldBeNil) + So(json.Get("query").Interface(), ShouldBeNil) + }) + }) + + Convey("When adding size, sort, filters", func() { + b.Size(200) + b.SortDesc(timeField, "boolean") + filters := b.Query().Bool().Filter() + filters.AddDateRangeFilter(timeField, "$timeTo", "$timeFrom", DateFormatEpochMS) + filters.AddQueryStringFilter("test", true) + + Convey("When building search request", func() { + sr, err := b.Build() + So(err, ShouldBeNil) + + Convey("Should have correct size", func() { + So(sr.Size, ShouldEqual, 200) + }) + + Convey("Should have correct sorting", func() { + sort, ok := sr.Sort[timeField].(map[string]string) + So(ok, ShouldBeTrue) + So(sort["order"], ShouldEqual, "desc") + So(sort["unmapped_type"], ShouldEqual, "boolean") + }) + + Convey("Should have range filter", func() { + f, ok := sr.Query.Bool.Filters[0].(*RangeFilter) + So(ok, ShouldBeTrue) + So(f.Gte, ShouldEqual, "$timeFrom") + So(f.Lte, ShouldEqual, "$timeTo") + So(f.Format, ShouldEqual, "epoch_millis") + }) + + Convey("Should have query string filter", func() { + f, ok := sr.Query.Bool.Filters[1].(*QueryStringFilter) + So(ok, ShouldBeTrue) + So(f.Query, ShouldEqual, "test") + So(f.AnalyzeWildcard, ShouldBeTrue) + }) + + Convey("When marshal to JSON should generate correct json", func() { + body, err := json.Marshal(sr) + So(err, ShouldBeNil) + json, err := simplejson.NewJson([]byte(body)) + So(err, ShouldBeNil) + So(json.Get("size").MustInt(0), ShouldEqual, 200) + + sort := json.GetPath("sort", timeField) + So(sort.Get("order").MustString(), ShouldEqual, "desc") + So(sort.Get("unmapped_type").MustString(), ShouldEqual, "boolean") + + timeRangeFilter := json.GetPath("query", "bool", "filter").GetIndex(0).Get("range").Get(timeField) + So(timeRangeFilter.Get("gte").MustString(""), ShouldEqual, "$timeFrom") + So(timeRangeFilter.Get("lte").MustString(""), ShouldEqual, "$timeTo") + So(timeRangeFilter.Get("format").MustString(""), ShouldEqual, DateFormatEpochMS) + + queryStringFilter := json.GetPath("query", "bool", "filter").GetIndex(1).Get("query_string") + So(queryStringFilter.Get("analyze_wildcard").MustBool(false), ShouldEqual, true) + So(queryStringFilter.Get("query").MustString(""), ShouldEqual, "test") + }) + }) + }) + + Convey("When adding doc value field", func() { + b.AddDocValueField(timeField) + + Convey("should set correct props", func() { + So(b.customProps["fields"], ShouldBeNil) + + scriptFields, ok := b.customProps["script_fields"].(map[string]interface{}) + So(ok, ShouldBeTrue) + So(scriptFields, ShouldHaveLength, 0) + + docValueFields, ok := b.customProps["docvalue_fields"].([]string) + So(ok, ShouldBeTrue) + So(docValueFields, ShouldHaveLength, 1) + So(docValueFields[0], ShouldEqual, timeField) + }) + + Convey("When building search request", func() { + sr, err := b.Build() + So(err, ShouldBeNil) + + Convey("When marshal to JSON should generate correct json", func() { + body, err := json.Marshal(sr) + So(err, ShouldBeNil) + json, err := simplejson.NewJson([]byte(body)) + So(err, ShouldBeNil) + + scriptFields, err := json.Get("script_fields").Map() + So(err, ShouldBeNil) + So(scriptFields, ShouldHaveLength, 0) + + _, err = json.Get("fields").StringArray() + So(err, ShouldNotBeNil) + + docValueFields, err := json.Get("docvalue_fields").StringArray() + So(err, ShouldBeNil) + So(docValueFields, ShouldHaveLength, 1) + So(docValueFields[0], ShouldEqual, timeField) + }) + }) + }) + + Convey("and adding multiple top level aggs", func() { + aggBuilder := b.Agg() + aggBuilder.Terms("1", "@hostname", nil) + aggBuilder.DateHistogram("2", "@timestamp", nil) + + Convey("When building search request", func() { + sr, err := b.Build() + So(err, ShouldBeNil) + + Convey("Should have 2 top level aggs", func() { + aggs := sr.Aggs + So(aggs, ShouldHaveLength, 2) + So(aggs[0].Key, ShouldEqual, "1") + So(aggs[0].Aggregation.Type, ShouldEqual, "terms") + So(aggs[1].Key, ShouldEqual, "2") + So(aggs[1].Aggregation.Type, ShouldEqual, "date_histogram") + }) + + Convey("When marshal to JSON should generate correct json", func() { + body, err := json.Marshal(sr) + So(err, ShouldBeNil) + json, err := simplejson.NewJson([]byte(body)) + So(err, ShouldBeNil) + + So(json.Get("aggs").MustMap(), ShouldHaveLength, 2) + So(json.GetPath("aggs", "1", "terms", "field").MustString(), ShouldEqual, "@hostname") + So(json.GetPath("aggs", "2", "date_histogram", "field").MustString(), ShouldEqual, "@timestamp") + }) + }) + }) + + Convey("and adding top level agg with child agg", func() { + aggBuilder := b.Agg() + aggBuilder.Terms("1", "@hostname", func(a *TermsAggregation, ib AggBuilder) { + ib.DateHistogram("2", "@timestamp", nil) + }) + + Convey("When building search request", func() { + sr, err := b.Build() + So(err, ShouldBeNil) + + Convey("Should have 1 top level agg and one child agg", func() { + aggs := sr.Aggs + So(aggs, ShouldHaveLength, 1) + + topAgg := aggs[0] + So(topAgg.Key, ShouldEqual, "1") + So(topAgg.Aggregation.Type, ShouldEqual, "terms") + So(topAgg.Aggregation.Aggs, ShouldHaveLength, 1) + + childAgg := aggs[0].Aggregation.Aggs[0] + So(childAgg.Key, ShouldEqual, "2") + So(childAgg.Aggregation.Type, ShouldEqual, "date_histogram") + }) + + Convey("When marshal to JSON should generate correct json", func() { + body, err := json.Marshal(sr) + So(err, ShouldBeNil) + json, err := simplejson.NewJson([]byte(body)) + So(err, ShouldBeNil) + + So(json.Get("aggs").MustMap(), ShouldHaveLength, 1) + firstLevelAgg := json.GetPath("aggs", "1") + secondLevelAgg := firstLevelAgg.GetPath("aggs", "2") + So(firstLevelAgg.GetPath("terms", "field").MustString(), ShouldEqual, "@hostname") + So(secondLevelAgg.GetPath("date_histogram", "field").MustString(), ShouldEqual, "@timestamp") + }) + }) + }) + + Convey("and adding two top level aggs with child agg", func() { + aggBuilder := b.Agg() + aggBuilder.Histogram("1", "@hostname", func(a *HistogramAgg, ib AggBuilder) { + ib.DateHistogram("2", "@timestamp", nil) + }) + aggBuilder.Filters("3", func(a *FiltersAggregation, ib AggBuilder) { + ib.Terms("4", "@test", nil) + }) + + Convey("When building search request", func() { + sr, err := b.Build() + So(err, ShouldBeNil) + + Convey("Should have 2 top level aggs with one child agg each", func() { + aggs := sr.Aggs + So(aggs, ShouldHaveLength, 2) + + topAggOne := aggs[0] + So(topAggOne.Key, ShouldEqual, "1") + So(topAggOne.Aggregation.Type, ShouldEqual, "histogram") + So(topAggOne.Aggregation.Aggs, ShouldHaveLength, 1) + + topAggOnechildAgg := topAggOne.Aggregation.Aggs[0] + So(topAggOnechildAgg.Key, ShouldEqual, "2") + So(topAggOnechildAgg.Aggregation.Type, ShouldEqual, "date_histogram") + + topAggTwo := aggs[1] + So(topAggTwo.Key, ShouldEqual, "3") + So(topAggTwo.Aggregation.Type, ShouldEqual, "filters") + So(topAggTwo.Aggregation.Aggs, ShouldHaveLength, 1) + + topAggTwochildAgg := topAggTwo.Aggregation.Aggs[0] + So(topAggTwochildAgg.Key, ShouldEqual, "4") + So(topAggTwochildAgg.Aggregation.Type, ShouldEqual, "terms") + }) + + Convey("When marshal to JSON should generate correct json", func() { + body, err := json.Marshal(sr) + So(err, ShouldBeNil) + json, err := simplejson.NewJson([]byte(body)) + So(err, ShouldBeNil) + + topAggOne := json.GetPath("aggs", "1") + So(topAggOne.GetPath("histogram", "field").MustString(), ShouldEqual, "@hostname") + topAggOnechildAgg := topAggOne.GetPath("aggs", "2") + So(topAggOnechildAgg.GetPath("date_histogram", "field").MustString(), ShouldEqual, "@timestamp") + + topAggTwo := json.GetPath("aggs", "3") + topAggTwochildAgg := topAggTwo.GetPath("aggs", "4") + So(topAggTwo.GetPath("filters").MustArray(), ShouldHaveLength, 0) + So(topAggTwochildAgg.GetPath("terms", "field").MustString(), ShouldEqual, "@test") + }) + }) + }) + + Convey("and adding top level agg with child agg with child agg", func() { + aggBuilder := b.Agg() + aggBuilder.Terms("1", "@hostname", func(a *TermsAggregation, ib AggBuilder) { + ib.Terms("2", "@app", func(a *TermsAggregation, ib AggBuilder) { + ib.DateHistogram("3", "@timestamp", nil) + }) + }) + + Convey("When building search request", func() { + sr, err := b.Build() + So(err, ShouldBeNil) + + Convey("Should have 1 top level agg with one child having a child", func() { + aggs := sr.Aggs + So(aggs, ShouldHaveLength, 1) + + topAgg := aggs[0] + So(topAgg.Key, ShouldEqual, "1") + So(topAgg.Aggregation.Type, ShouldEqual, "terms") + So(topAgg.Aggregation.Aggs, ShouldHaveLength, 1) + + childAgg := topAgg.Aggregation.Aggs[0] + So(childAgg.Key, ShouldEqual, "2") + So(childAgg.Aggregation.Type, ShouldEqual, "terms") + + childChildAgg := childAgg.Aggregation.Aggs[0] + So(childChildAgg.Key, ShouldEqual, "3") + So(childChildAgg.Aggregation.Type, ShouldEqual, "date_histogram") + }) + + Convey("When marshal to JSON should generate correct json", func() { + body, err := json.Marshal(sr) + So(err, ShouldBeNil) + json, err := simplejson.NewJson([]byte(body)) + So(err, ShouldBeNil) + + topAgg := json.GetPath("aggs", "1") + So(topAgg.GetPath("terms", "field").MustString(), ShouldEqual, "@hostname") + + childAgg := topAgg.GetPath("aggs", "2") + So(childAgg.GetPath("terms", "field").MustString(), ShouldEqual, "@app") + + childChildAgg := childAgg.GetPath("aggs", "3") + So(childChildAgg.GetPath("date_histogram", "field").MustString(), ShouldEqual, "@timestamp") + }) + }) + }) + + Convey("and adding bucket and metric aggs", func() { + aggBuilder := b.Agg() + aggBuilder.Terms("1", "@hostname", func(a *TermsAggregation, ib AggBuilder) { + ib.Terms("2", "@app", func(a *TermsAggregation, ib AggBuilder) { + ib.Metric("4", "avg", "@value", nil) + ib.DateHistogram("3", "@timestamp", func(a *DateHistogramAgg, ib AggBuilder) { + ib.Metric("4", "avg", "@value", nil) + ib.Metric("5", "max", "@value", nil) + }) + }) + }) + + Convey("When building search request", func() { + sr, err := b.Build() + So(err, ShouldBeNil) + + Convey("Should have 1 top level agg with one child having a child", func() { + aggs := sr.Aggs + So(aggs, ShouldHaveLength, 1) + + topAgg := aggs[0] + So(topAgg.Key, ShouldEqual, "1") + So(topAgg.Aggregation.Type, ShouldEqual, "terms") + So(topAgg.Aggregation.Aggs, ShouldHaveLength, 1) + + childAgg := topAgg.Aggregation.Aggs[0] + So(childAgg.Key, ShouldEqual, "2") + So(childAgg.Aggregation.Type, ShouldEqual, "terms") + + childChildOneAgg := childAgg.Aggregation.Aggs[0] + So(childChildOneAgg.Key, ShouldEqual, "4") + So(childChildOneAgg.Aggregation.Type, ShouldEqual, "avg") + + childChildTwoAgg := childAgg.Aggregation.Aggs[1] + So(childChildTwoAgg.Key, ShouldEqual, "3") + So(childChildTwoAgg.Aggregation.Type, ShouldEqual, "date_histogram") + + childChildTwoChildOneAgg := childChildTwoAgg.Aggregation.Aggs[0] + So(childChildTwoChildOneAgg.Key, ShouldEqual, "4") + So(childChildTwoChildOneAgg.Aggregation.Type, ShouldEqual, "avg") + + childChildTwoChildTwoAgg := childChildTwoAgg.Aggregation.Aggs[1] + So(childChildTwoChildTwoAgg.Key, ShouldEqual, "5") + So(childChildTwoChildTwoAgg.Aggregation.Type, ShouldEqual, "max") + }) + + Convey("When marshal to JSON should generate correct json", func() { + body, err := json.Marshal(sr) + So(err, ShouldBeNil) + json, err := simplejson.NewJson([]byte(body)) + So(err, ShouldBeNil) + + termsAgg := json.GetPath("aggs", "1") + So(termsAgg.GetPath("terms", "field").MustString(), ShouldEqual, "@hostname") + + termsAggTwo := termsAgg.GetPath("aggs", "2") + So(termsAggTwo.GetPath("terms", "field").MustString(), ShouldEqual, "@app") + + termsAggTwoAvg := termsAggTwo.GetPath("aggs", "4") + So(termsAggTwoAvg.GetPath("avg", "field").MustString(), ShouldEqual, "@value") + + dateHistAgg := termsAggTwo.GetPath("aggs", "3") + So(dateHistAgg.GetPath("date_histogram", "field").MustString(), ShouldEqual, "@timestamp") + + avgAgg := dateHistAgg.GetPath("aggs", "4") + So(avgAgg.GetPath("avg", "field").MustString(), ShouldEqual, "@value") + + maxAgg := dateHistAgg.GetPath("aggs", "5") + So(maxAgg.GetPath("max", "field").MustString(), ShouldEqual, "@value") + }) + }) + }) + }) + + Convey("Given new search request builder for es version 2", func() { + b := NewSearchRequestBuilder(2) + + Convey("When adding doc value field", func() { + b.AddDocValueField(timeField) + + Convey("should set correct props", func() { + fields, ok := b.customProps["fields"].([]string) + So(ok, ShouldBeTrue) + So(fields, ShouldHaveLength, 2) + So(fields[0], ShouldEqual, "*") + So(fields[1], ShouldEqual, "_source") + + scriptFields, ok := b.customProps["script_fields"].(map[string]interface{}) + So(ok, ShouldBeTrue) + So(scriptFields, ShouldHaveLength, 0) + + fieldDataFields, ok := b.customProps["fielddata_fields"].([]string) + So(ok, ShouldBeTrue) + So(fieldDataFields, ShouldHaveLength, 1) + So(fieldDataFields[0], ShouldEqual, timeField) + }) + + Convey("When building search request", func() { + sr, err := b.Build() + So(err, ShouldBeNil) + + Convey("When marshal to JSON should generate correct json", func() { + body, err := json.Marshal(sr) + So(err, ShouldBeNil) + json, err := simplejson.NewJson([]byte(body)) + So(err, ShouldBeNil) + + scriptFields, err := json.Get("script_fields").Map() + So(err, ShouldBeNil) + So(scriptFields, ShouldHaveLength, 0) + + fields, err := json.Get("fields").StringArray() + So(err, ShouldBeNil) + So(fields, ShouldHaveLength, 2) + So(fields[0], ShouldEqual, "*") + So(fields[1], ShouldEqual, "_source") + + fieldDataFields, err := json.Get("fielddata_fields").StringArray() + So(err, ShouldBeNil) + So(fieldDataFields, ShouldHaveLength, 1) + So(fieldDataFields[0], ShouldEqual, timeField) + }) + }) + }) + }) + }) +} + +func TestMultiSearchRequest(t *testing.T) { + Convey("Test elasticsearch multi search request", t, func() { + Convey("Given new multi search request builder", func() { + b := NewMultiSearchRequestBuilder(0) + + Convey("When adding one search request", func() { + b.Search() + + Convey("When building search request should contain one search request", func() { + mr, err := b.Build() + So(err, ShouldBeNil) + So(mr.Requests, ShouldHaveLength, 1) + }) + }) + + Convey("When adding two search requests", func() { + b.Search() + b.Search() + + Convey("When building search request should contain two search requests", func() { + mr, err := b.Build() + So(err, ShouldBeNil) + So(mr.Requests, ShouldHaveLength, 2) + }) + }) + }) + }) +} From 4840adff00019b3372ec5840eb34b08822d1f82f Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 23 May 2018 15:09:58 +0200 Subject: [PATCH 11/22] elasticsearch: refactor query handling and use new es simple client Removes moment dependency. Adds response parser tests (based on frontend tests). Adds time series query tests (based on frontend tests). Fixes various issues related to response parsing and building search request queries. Added support for extended stats metrics and geo hash grid aggregations. --- Gopkg.lock | 8 +- Gopkg.toml | 4 - pkg/tsdb/elasticsearch/elasticsearch.go | 34 +- pkg/tsdb/elasticsearch/elasticsearch_test.go | 121 -- pkg/tsdb/elasticsearch/models.go | 150 +-- pkg/tsdb/elasticsearch/query.go | 296 ---- pkg/tsdb/elasticsearch/query_def.go | 43 - pkg/tsdb/elasticsearch/query_test.go | 297 ----- pkg/tsdb/elasticsearch/response_parser.go | 381 +++++- .../elasticsearch/response_parser_test.go | 960 +++++++++++-- pkg/tsdb/elasticsearch/time_series_query.go | 300 +++-- .../elasticsearch/time_series_query_test.go | 518 ++++++- vendor/github.com/leibowitz/moment/diff.go | 75 -- vendor/github.com/leibowitz/moment/moment.go | 1185 ----------------- .../leibowitz/moment/moment_parser.go | 100 -- .../github.com/leibowitz/moment/parse_day.go | 32 - .../leibowitz/moment/strftime_parser.go | 68 - 17 files changed, 1966 insertions(+), 2606 deletions(-) delete mode 100644 pkg/tsdb/elasticsearch/elasticsearch_test.go delete mode 100644 pkg/tsdb/elasticsearch/query.go delete mode 100644 pkg/tsdb/elasticsearch/query_def.go delete mode 100644 pkg/tsdb/elasticsearch/query_test.go delete mode 100644 vendor/github.com/leibowitz/moment/diff.go delete mode 100644 vendor/github.com/leibowitz/moment/moment.go delete mode 100644 vendor/github.com/leibowitz/moment/moment_parser.go delete mode 100644 vendor/github.com/leibowitz/moment/parse_day.go delete mode 100644 vendor/github.com/leibowitz/moment/strftime_parser.go diff --git a/Gopkg.lock b/Gopkg.lock index 65548818ca3..41fc92313d1 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -308,12 +308,6 @@ packages = ["."] revision = "7cafcd837844e784b526369c9bce262804aebc60" -[[projects]] - branch = "master" - name = "github.com/leibowitz/moment" - packages = ["."] - revision = "8548108dcca204a1110b99e5fec966817499fe84" - [[projects]] branch = "master" name = "github.com/lib/pq" @@ -667,6 +661,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "4039f122ac5dd045948e003eb7a74c8864df1759b25147f1b2e2e8ad7a8414d6" + inputs-digest = "bd54a1a836599d90b36d4ac1af56d716ef9ca5be4865e217bddd49e3d32a1997" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 98665ab7310..a9f79c402df 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -199,7 +199,3 @@ ignored = [ [[constraint]] name = "github.com/denisenkom/go-mssqldb" revision = "270bc3860bb94dd3a3ffd047377d746c5e276726" - -[[constraint]] - branch = "master" - name = "github.com/leibowitz/moment" diff --git a/pkg/tsdb/elasticsearch/elasticsearch.go b/pkg/tsdb/elasticsearch/elasticsearch.go index abf25feac06..857b847f0f9 100644 --- a/pkg/tsdb/elasticsearch/elasticsearch.go +++ b/pkg/tsdb/elasticsearch/elasticsearch.go @@ -3,17 +3,14 @@ package elasticsearch import ( "context" "fmt" - "net/http" - "net/url" - "path" - "strings" - "time" "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/tsdb" + "github.com/grafana/grafana/pkg/tsdb/elasticsearch/client" ) +// ElasticsearchExecutor represents a handler for handling elasticsearch datasource request type ElasticsearchExecutor struct{} var ( @@ -21,43 +18,28 @@ var ( intervalCalculator tsdb.IntervalCalculator ) +// NewElasticsearchExecutor creates a new elasticsearch executor func NewElasticsearchExecutor(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) { return &ElasticsearchExecutor{}, nil } func init() { glog = log.New("tsdb.elasticsearch") + intervalCalculator = tsdb.NewIntervalCalculator(nil) tsdb.RegisterTsdbQueryEndpoint("elasticsearch", NewElasticsearchExecutor) - intervalCalculator = tsdb.NewIntervalCalculator(&tsdb.IntervalOptions{MinInterval: time.Millisecond * 1}) } +// Query handles an elasticsearch datasource request func (e *ElasticsearchExecutor) Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) { if len(tsdbQuery.Queries) == 0 { return nil, fmt.Errorf("query contains no queries") } - return e.executeTimeSeriesQuery(ctx, dsInfo, tsdbQuery) -} - -func (e *ElasticsearchExecutor) createRequest(dsInfo *models.DataSource, query string) (*http.Request, error) { - u, _ := url.Parse(dsInfo.Url) - u.Path = path.Join(u.Path, "_msearch") - req, err := http.NewRequest(http.MethodPost, u.String(), strings.NewReader(query)) + client, err := es.NewClient(ctx, dsInfo, tsdbQuery.TimeRange) if err != nil { return nil, err } - req.Header.Set("User-Agent", "Grafana") - req.Header.Set("Content-Type", "application/json") - if dsInfo.BasicAuth { - req.SetBasicAuth(dsInfo.BasicAuthUser, dsInfo.BasicAuthPassword) - } - - if !dsInfo.BasicAuth && dsInfo.User != "" { - req.SetBasicAuth(dsInfo.User, dsInfo.Password) - } - - glog.Debug("Elasticsearch request", "url", req.URL.String()) - glog.Debug("Elasticsearch request", "body", query) - return req, nil + query := newTimeSeriesQuery(client, tsdbQuery, intervalCalculator) + return query.execute() } diff --git a/pkg/tsdb/elasticsearch/elasticsearch_test.go b/pkg/tsdb/elasticsearch/elasticsearch_test.go deleted file mode 100644 index ad905299166..00000000000 --- a/pkg/tsdb/elasticsearch/elasticsearch_test.go +++ /dev/null @@ -1,121 +0,0 @@ -package elasticsearch - -import ( - "github.com/grafana/grafana/pkg/components/simplejson" - "time" -) - -var avgWithMovingAvg = Query{ - TimeField: "timestamp", - RawQuery: "(test:query) AND (name:sample)", - Interval: time.Millisecond, - BucketAggs: []*BucketAgg{{ - Field: "timestamp", - ID: "2", - Type: "date_histogram", - Settings: simplejson.NewFromAny(map[string]interface{}{ - "interval": "auto", - "min_doc_count": 0, - "trimEdges": 0, - }), - }}, - Metrics: []*Metric{{ - Field: "value", - ID: "1", - Type: "avg", - Settings: simplejson.NewFromAny(map[string]interface{}{ - "script": map[string]string{ - "inline": "_value * 2", - }, - }), - }, { - Field: "1", - ID: "3", - Type: "moving_avg", - PipelineAggregate: "1", - Settings: simplejson.NewFromAny(map[string]interface{}{ - "minimize": false, - "model": "simple", - "window": 5, - }), - }}, -} - -var wildcardsAndQuotes = Query{ - TimeField: "timestamp", - RawQuery: "scope:$location.leagueconnect.api AND name:*CreateRegistration AND name:\"*.201-responses.rate\"", - Interval: time.Millisecond, - BucketAggs: []*BucketAgg{{ - Field: "timestamp", - ID: "2", - Type: "date_histogram", - Settings: simplejson.NewFromAny(map[string]interface{}{}), - }}, - Metrics: []*Metric{{ - Field: "value", - ID: "1", - Type: "sum", - Settings: simplejson.NewFromAny(map[string]interface{}{}), - }}, -} -var termAggs = Query{ - TimeField: "timestamp", - RawQuery: "(scope:*.hmp.metricsd) AND (name_raw:builtin.general.*_instance_count)", - Interval: time.Millisecond, - BucketAggs: []*BucketAgg{{ - Field: "name_raw", - ID: "4", - Type: "terms", - Settings: simplejson.NewFromAny(map[string]interface{}{ - "order": "desc", - "orderBy": "_term", - "size": "10", - }), - }, { - Field: "timestamp", - ID: "2", - Type: "date_histogram", - Settings: simplejson.NewFromAny(map[string]interface{}{ - "interval": "auto", - "min_doc_count": 0, - "trimEdges": 0, - }), - }}, - Metrics: []*Metric{{ - Field: "value", - ID: "1", - Type: "sum", - Settings: simplejson.NewFromAny(map[string]interface{}{}), - }}, -} - -var filtersAggs = Query{ - TimeField: "time", - RawQuery: "*", - Interval: time.Millisecond, - BucketAggs: []*BucketAgg{{ - ID: "3", - Type: "filters", - Settings: simplejson.NewFromAny(map[string]interface{}{ - "filters": []interface{}{ - map[string]interface{}{"label": "hello", "query": "host:\"67.65.185.232\""}, - }, - }), - }, { - Field: "timestamp", - ID: "2", - Type: "date_histogram", - Settings: simplejson.NewFromAny(map[string]interface{}{ - "interval": "auto", - "min_doc_count": 0, - "trimEdges": 0, - }), - }}, - Metrics: []*Metric{{ - Field: "bytesSent", - ID: "1", - Type: "count", - PipelineAggregate: "select metric", - Settings: simplejson.NewFromAny(map[string]interface{}{}), - }}, -} diff --git a/pkg/tsdb/elasticsearch/models.go b/pkg/tsdb/elasticsearch/models.go index 9cf295cbd0e..b3fdee95b91 100644 --- a/pkg/tsdb/elasticsearch/models.go +++ b/pkg/tsdb/elasticsearch/models.go @@ -1,12 +1,21 @@ package elasticsearch import ( - "bytes" - "encoding/json" - "fmt" "github.com/grafana/grafana/pkg/components/simplejson" ) +// Query represents the time series query model of the datasource +type Query struct { + TimeField string `json:"timeField"` + RawQuery string `json:"query"` + BucketAggs []*BucketAgg `json:"bucketAggs"` + Metrics []*MetricAgg `json:"metrics"` + Alias string `json:"alias"` + Interval string + RefID string +} + +// BucketAgg represents a bucket aggregation of the time series query model of the datasource type BucketAgg struct { Field string `json:"field"` ID string `json:"id"` @@ -14,120 +23,55 @@ type BucketAgg struct { Type string `jsons:"type"` } -type Metric struct { +// MetricAgg represents a metric aggregation of the time series query model of the datasource +type MetricAgg struct { Field string `json:"field"` Hide bool `json:"hide"` ID string `json:"id"` PipelineAggregate string `json:"pipelineAgg"` Settings *simplejson.Json `json:"settings"` + Meta *simplejson.Json `json:"meta"` Type string `json:"type"` } -type QueryHeader struct { - SearchType string `json:"search_type"` - IgnoreUnavailable bool `json:"ignore_unavailable"` - Index interface{} `json:"index"` - MaxConcurrentShardRequests int `json:"max_concurrent_shard_requests,omitempty"` +var metricAggType = map[string]string{ + "count": "Count", + "avg": "Average", + "sum": "Sum", + "max": "Max", + "min": "Min", + "extended_stats": "Extended Stats", + "percentiles": "Percentiles", + "cardinality": "Unique Count", + "moving_avg": "Moving Average", + "derivative": "Derivative", + "raw_document": "Raw Document", } -func (q *QueryHeader) String() string { - r, _ := json.Marshal(q) - return string(r) +var extendedStats = map[string]string{ + "avg": "Avg", + "min": "Min", + "max": "Max", + "sum": "Sum", + "count": "Count", + "std_deviation": "Std Dev", + "std_deviation_bounds_upper": "Std Dev Upper", + "std_deviation_bounds_lower": "Std Dev Lower", } -type Request struct { - Query map[string]interface{} `json:"query"` - Aggs Aggs `json:"aggs"` - Size int `json:"size"` +var pipelineAggType = map[string]string{ + "moving_avg": "moving_avg", + "derivative": "derivative", } -type Aggs map[string]interface{} - -type HistogramAgg struct { - Interval string `json:"interval,omitempty"` - Field string `json:"field"` - MinDocCount int `json:"min_doc_count"` - Missing string `json:"missing,omitempty"` -} - -type DateHistogramAgg struct { - HistogramAgg - ExtendedBounds ExtendedBounds `json:"extended_bounds"` - Format string `json:"format"` -} - -type FiltersAgg struct { - Filters map[string]interface{} `json:"filters"` -} - -type TermsAgg struct { - Field string `json:"field"` - Size int `json:"size"` - Order map[string]interface{} `json:"order"` - Missing string `json:"missing,omitempty"` -} - -type TermsAggWrap struct { - Terms TermsAgg `json:"terms"` - Aggs Aggs `json:"aggs"` -} - -type ExtendedBounds struct { - Min string `json:"min"` - Max string `json:"max"` -} - -type RangeFilter struct { - Range map[string]RangeFilterSetting `json:"range"` -} -type RangeFilterSetting struct { - Gte string `json:"gte"` - Lte string `json:"lte"` - Format string `json:"format"` -} - -func newRangeFilter(field string, rangeFilterSetting RangeFilterSetting) *RangeFilter { - return &RangeFilter{ - map[string]RangeFilterSetting{field: rangeFilterSetting}} -} - -type QueryStringFilter struct { - QueryString QueryStringFilterSetting `json:"query_string"` -} -type QueryStringFilterSetting struct { - AnalyzeWildcard bool `json:"analyze_wildcard"` - Query string `json:"query"` -} - -func newQueryStringFilter(analyzeWildcard bool, query string) *QueryStringFilter { - return &QueryStringFilter{QueryStringFilterSetting{AnalyzeWildcard: analyzeWildcard, Query: query}} -} - -type BoolQuery struct { - Filter []interface{} `json:"filter"` -} - -type Responses struct { - Responses []Response `json:"responses"` -} - -type Response struct { - Status int `json:"status"` - Err map[string]interface{} `json:"error"` - Aggregations map[string]interface{} `json:"aggregations"` -} - -func (r *Response) getErrMsg() string { - var msg bytes.Buffer - errJson := simplejson.NewFromAny(r.Err) - errType, err := errJson.Get("type").String() - if err == nil { - msg.WriteString(fmt.Sprintf("type:%s", errType)) +func isPipelineAgg(metricType string) bool { + if _, ok := pipelineAggType[metricType]; ok { + return true } + return false +} - reason, err := errJson.Get("type").String() - if err == nil { - msg.WriteString(fmt.Sprintf("reason:%s", reason)) - } - return msg.String() +func describeMetric(metricType, field string) string { + text := metricAggType[metricType] + return text + " " + field } diff --git a/pkg/tsdb/elasticsearch/query.go b/pkg/tsdb/elasticsearch/query.go deleted file mode 100644 index 123f8dd5667..00000000000 --- a/pkg/tsdb/elasticsearch/query.go +++ /dev/null @@ -1,296 +0,0 @@ -package elasticsearch - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "strconv" - "strings" - "time" - - "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/tsdb" - "github.com/leibowitz/moment" -) - -var rangeFilterSetting = RangeFilterSetting{Gte: "$timeFrom", - Lte: "$timeTo", - Format: "epoch_millis"} - -type Query struct { - TimeField string `json:"timeField"` - RawQuery string `json:"query"` - BucketAggs []*BucketAgg `json:"bucketAggs"` - Metrics []*Metric `json:"metrics"` - Alias string `json:"alias"` - Interval time.Duration -} - -func (q *Query) Build(queryContext *tsdb.TsdbQuery, dsInfo *models.DataSource) (string, error) { - var req Request - req.Size = 0 - q.renderReqQuery(&req) - - // handle document query - if q.isRawDocumentQuery() { - return "", errors.New("alert not support Raw_Document") - } - - err := q.parseAggs(&req) - if err != nil { - return "", err - } - - reqBytes, err := json.Marshal(req) - reqHeader := getRequestHeader(queryContext.TimeRange, dsInfo) - payload := bytes.Buffer{} - payload.WriteString(reqHeader.String() + "\n") - payload.WriteString(string(reqBytes) + "\n") - return q.renderTemplate(payload.String(), queryContext) -} - -func (q *Query) isRawDocumentQuery() bool { - if len(q.BucketAggs) == 0 { - if len(q.Metrics) > 0 { - metric := simplejson.NewFromAny(q.Metrics[0]) - if metric.Get("type").MustString("") == "raw_document" { - return true - } - } - } - return false -} - -func (q *Query) renderReqQuery(req *Request) { - req.Query = make(map[string]interface{}) - boolQuery := BoolQuery{} - boolQuery.Filter = append(boolQuery.Filter, newRangeFilter(q.TimeField, rangeFilterSetting)) - boolQuery.Filter = append(boolQuery.Filter, newQueryStringFilter(true, q.RawQuery)) - req.Query["bool"] = boolQuery -} - -func (q *Query) parseAggs(req *Request) error { - aggs := make(Aggs) - nestedAggs := aggs - for _, agg := range q.BucketAggs { - esAggs := make(Aggs) - switch agg.Type { - case "date_histogram": - esAggs["date_histogram"] = q.getDateHistogramAgg(agg) - case "histogram": - esAggs["histogram"] = q.getHistogramAgg(agg) - case "filters": - esAggs["filters"] = q.getFilters(agg) - case "terms": - terms := q.getTerms(agg) - esAggs["terms"] = terms.Terms - esAggs["aggs"] = terms.Aggs - case "geohash_grid": - return errors.New("alert not support Geo_Hash_Grid") - } - - if _, ok := nestedAggs["aggs"]; !ok { - nestedAggs["aggs"] = make(Aggs) - } - - if aggs, ok := (nestedAggs["aggs"]).(Aggs); ok { - aggs[agg.ID] = esAggs - } - nestedAggs = esAggs - - } - nestedAggs["aggs"] = make(Aggs) - - for _, metric := range q.Metrics { - subAgg := make(Aggs) - - if metric.Type == "count" { - continue - } - settings := metric.Settings.MustMap(make(map[string]interface{})) - - if isPipelineAgg(metric.Type) { - if _, err := strconv.Atoi(metric.PipelineAggregate); err == nil { - settings["buckets_path"] = metric.PipelineAggregate - } else { - continue - } - - } else { - settings["field"] = metric.Field - } - - subAgg[metric.Type] = settings - nestedAggs["aggs"].(Aggs)[metric.ID] = subAgg - } - req.Aggs = aggs["aggs"].(Aggs) - return nil -} - -func (q *Query) getDateHistogramAgg(target *BucketAgg) *DateHistogramAgg { - agg := &DateHistogramAgg{} - interval, err := target.Settings.Get("interval").String() - if err == nil { - agg.Interval = interval - } - agg.Field = q.TimeField - agg.MinDocCount = target.Settings.Get("min_doc_count").MustInt(0) - agg.ExtendedBounds = ExtendedBounds{"$timeFrom", "$timeTo"} - agg.Format = "epoch_millis" - - if agg.Interval == "auto" { - agg.Interval = "$__interval" - } - - missing, err := target.Settings.Get("missing").String() - if err == nil { - agg.Missing = missing - } - return agg -} - -func (q *Query) getHistogramAgg(target *BucketAgg) *HistogramAgg { - agg := &HistogramAgg{} - interval, err := target.Settings.Get("interval").String() - if err == nil { - agg.Interval = interval - } - - if target.Field != "" { - agg.Field = target.Field - } - agg.MinDocCount = target.Settings.Get("min_doc_count").MustInt(0) - missing, err := target.Settings.Get("missing").String() - if err == nil { - agg.Missing = missing - } - return agg -} - -func (q *Query) getFilters(target *BucketAgg) *FiltersAgg { - agg := &FiltersAgg{} - agg.Filters = map[string]interface{}{} - for _, filter := range target.Settings.Get("filters").MustArray() { - filterJson := simplejson.NewFromAny(filter) - query := filterJson.Get("query").MustString("") - label := filterJson.Get("label").MustString("") - if label == "" { - label = query - } - - agg.Filters[label] = newQueryStringFilter(true, query) - } - return agg -} - -func (q *Query) getTerms(target *BucketAgg) *TermsAggWrap { - agg := &TermsAggWrap{Aggs: make(Aggs)} - agg.Terms.Field = target.Field - if len(target.Settings.MustMap()) == 0 { - return agg - } - sizeStr := target.Settings.Get("size").MustString("") - size, err := strconv.Atoi(sizeStr) - if err != nil { - size = 500 - } - agg.Terms.Size = size - orderBy, err := target.Settings.Get("orderBy").String() - if err == nil { - agg.Terms.Order = make(map[string]interface{}) - agg.Terms.Order[orderBy] = target.Settings.Get("order").MustString("") - if _, err := strconv.Atoi(orderBy); err != nil { - for _, metricI := range q.Metrics { - metric := simplejson.NewFromAny(metricI) - metricId := metric.Get("id").MustString() - if metricId == orderBy { - subAggs := make(Aggs) - metricField := metric.Get("field").MustString() - metricType := metric.Get("type").MustString() - subAggs[metricType] = map[string]string{"field": metricField} - agg.Aggs = make(Aggs) - agg.Aggs[metricId] = subAggs - break - } - } - } - } - - missing, err := target.Settings.Get("missing").String() - if err == nil { - agg.Terms.Missing = missing - } - - return agg -} - -func (q *Query) renderTemplate(payload string, queryContext *tsdb.TsdbQuery) (string, error) { - timeRange := queryContext.TimeRange - interval := intervalCalculator.Calculate(timeRange, q.Interval) - payload = strings.Replace(payload, "$timeFrom", fmt.Sprintf("%d", timeRange.GetFromAsMsEpoch()), -1) - payload = strings.Replace(payload, "$timeTo", fmt.Sprintf("%d", timeRange.GetToAsMsEpoch()), -1) - payload = strings.Replace(payload, "$interval", interval.Text, -1) - payload = strings.Replace(payload, "$__interval_ms", strconv.FormatInt(interval.Value.Nanoseconds()/int64(time.Millisecond), 10), -1) - payload = strings.Replace(payload, "$__interval", interval.Text, -1) - return payload, nil -} - -func getRequestHeader(timeRange *tsdb.TimeRange, dsInfo *models.DataSource) *QueryHeader { - var header QueryHeader - esVersion := dsInfo.JsonData.Get("esVersion").MustInt() - - searchType := "query_then_fetch" - if esVersion < 5 { - searchType = "count" - } - header.SearchType = searchType - header.IgnoreUnavailable = true - header.Index = getIndexList(dsInfo.Database, dsInfo.JsonData.Get("interval").MustString(), timeRange) - - if esVersion >= 56 { - header.MaxConcurrentShardRequests = dsInfo.JsonData.Get("maxConcurrentShardRequests").MustInt() - } - return &header -} - -func getIndexList(pattern string, interval string, timeRange *tsdb.TimeRange) string { - if interval == "" { - return pattern - } - - var indexes []string - indexParts := strings.Split(strings.TrimLeft(pattern, "["), "]") - indexBase := indexParts[0] - if len(indexParts) <= 1 { - return pattern - } - - indexDateFormat := indexParts[1] - - start := moment.NewMoment(timeRange.MustGetFrom()) - end := moment.NewMoment(timeRange.MustGetTo()) - - indexes = append(indexes, fmt.Sprintf("%s%s", indexBase, start.Format(indexDateFormat))) - for start.IsBefore(*end) { - switch interval { - case "Hourly": - start = start.AddHours(1) - - case "Daily": - start = start.AddDay() - - case "Weekly": - start = start.AddWeeks(1) - - case "Monthly": - start = start.AddMonths(1) - - case "Yearly": - start = start.AddYears(1) - } - indexes = append(indexes, fmt.Sprintf("%s%s", indexBase, start.Format(indexDateFormat))) - } - return strings.Join(indexes, ",") -} diff --git a/pkg/tsdb/elasticsearch/query_def.go b/pkg/tsdb/elasticsearch/query_def.go deleted file mode 100644 index 128e752d97a..00000000000 --- a/pkg/tsdb/elasticsearch/query_def.go +++ /dev/null @@ -1,43 +0,0 @@ -package elasticsearch - -var metricAggType = map[string]string{ - "count": "Count", - "avg": "Average", - "sum": "Sum", - "max": "Max", - "min": "Min", - "extended_stats": "Extended Stats", - "percentiles": "Percentiles", - "cardinality": "Unique Count", - "moving_avg": "Moving Average", - "derivative": "Derivative", - "raw_document": "Raw Document", -} - -var extendedStats = map[string]string{ - "avg": "Avg", - "min": "Min", - "max": "Max", - "sum": "Sum", - "count": "Count", - "std_deviation": "Std Dev", - "std_deviation_bounds_upper": "Std Dev Upper", - "std_deviation_bounds_lower": "Std Dev Lower", -} - -var pipelineOptions = map[string]string{ - "moving_avg": "moving_avg", - "derivative": "derivative", -} - -func isPipelineAgg(metricType string) bool { - if _, ok := pipelineOptions[metricType]; ok { - return true - } - return false -} - -func describeMetric(metricType, field string) string { - text := metricAggType[metricType] - return text + " " + field -} diff --git a/pkg/tsdb/elasticsearch/query_test.go b/pkg/tsdb/elasticsearch/query_test.go deleted file mode 100644 index 1ce6e5ac7bb..00000000000 --- a/pkg/tsdb/elasticsearch/query_test.go +++ /dev/null @@ -1,297 +0,0 @@ -package elasticsearch - -import ( - "encoding/json" - "fmt" - "reflect" - "strconv" - "strings" - "testing" - - "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/tsdb" - . "github.com/smartystreets/goconvey/convey" -) - -func testElasticSearchResponse(query Query, expectedElasticSearchRequestJSON string) { - var queryExpectedJSONInterface, queryJSONInterface interface{} - jsonDate, _ := simplejson.NewJson([]byte(`{"esVersion":2}`)) - dsInfo := &models.DataSource{ - Database: "grafana-test", - JsonData: jsonDate, - } - - testTimeRange := tsdb.NewTimeRange("5m", "now") - - s, err := query.Build(&tsdb.TsdbQuery{TimeRange: testTimeRange}, dsInfo) - So(err, ShouldBeNil) - queryJSON := strings.Split(s, "\n")[1] - err = json.Unmarshal([]byte(queryJSON), &queryJSONInterface) - So(err, ShouldBeNil) - - expectedElasticSearchRequestJSON = strings.Replace( - expectedElasticSearchRequestJSON, - "", - strconv.FormatInt(testTimeRange.GetFromAsMsEpoch(), 10), - -1, - ) - - expectedElasticSearchRequestJSON = strings.Replace( - expectedElasticSearchRequestJSON, - "", - strconv.FormatInt(testTimeRange.GetToAsMsEpoch(), 10), - -1, - ) - - err = json.Unmarshal([]byte(expectedElasticSearchRequestJSON), &queryExpectedJSONInterface) - So(err, ShouldBeNil) - - result := reflect.DeepEqual(queryExpectedJSONInterface, queryJSONInterface) - if !result { - fmt.Printf("ERROR: %s \n != \n %s", expectedElasticSearchRequestJSON, queryJSON) - } - So(result, ShouldBeTrue) -} -func TestElasticSearchQueryBuilder(t *testing.T) { - Convey("Elasticsearch QueryBuilder query testing", t, func() { - Convey("Build test average metric with moving average", func() { - var expectedElasticsearchQueryJSON = ` - { - "size": 0, - "query": { - "bool": { - "filter": [ - { - "range": { - "timestamp": { - "gte": "", - "lte": "", - "format": "epoch_millis" - } - } - }, - { - "query_string": { - "analyze_wildcard": true, - "query": "(test:query) AND (name:sample)" - } - } - ] - } - }, - "aggs": { - "2": { - "date_histogram": { - "interval": "200ms", - "field": "timestamp", - "min_doc_count": 0, - "extended_bounds": { - "min": "", - "max": "" - }, - "format": "epoch_millis" - }, - "aggs": { - "1": { - "avg": { - "field": "value", - "script": { - "inline": "_value * 2" - } - } - }, - "3": { - "moving_avg": { - "buckets_path": "1", - "window": 5, - "model": "simple", - "minimize": false - } - } - } - } - } - }` - - testElasticSearchResponse(avgWithMovingAvg, expectedElasticsearchQueryJSON) - }) - Convey("Test Wildcards and Quotes", func() { - expectedElasticsearchQueryJSON := ` - { - "size": 0, - "query": { - "bool": { - "filter": [ - { - "range": { - "timestamp": { - "gte": "", - "lte": "", - "format": "epoch_millis" - } - } - }, - { - "query_string": { - "analyze_wildcard": true, - "query": "scope:$location.leagueconnect.api AND name:*CreateRegistration AND name:\"*.201-responses.rate\"" - } - } - ] - } - }, - "aggs": { - "2": { - "aggs": { - "1": { - "sum": { - "field": "value" - } - } - }, - "date_histogram": { - "extended_bounds": { - "max": "", - "min": "" - }, - "field": "timestamp", - "format": "epoch_millis", - "min_doc_count": 0 - } - } - } - }` - - testElasticSearchResponse(wildcardsAndQuotes, expectedElasticsearchQueryJSON) - }) - Convey("Test Term Aggregates", func() { - expectedElasticsearchQueryJSON := ` - { - "size": 0, - "query": { - "bool": { - "filter": [ - { - "range": { - "timestamp": { - "gte": "", - "lte": "", - "format": "epoch_millis" - } - } - }, - { - "query_string": { - "analyze_wildcard": true, - "query": "(scope:*.hmp.metricsd) AND (name_raw:builtin.general.*_instance_count)" - } - } - ] - } - }, - "aggs": {"4":{"aggs":{"2":{"aggs":{"1":{"sum":{"field":"value"}}},"date_histogram":{"extended_bounds":{"max":"","min":""},"field":"timestamp","format":"epoch_millis","interval":"200ms","min_doc_count":0}}},"terms":{"field":"name_raw","order":{"_term":"desc"},"size":10}}} - }` - - testElasticSearchResponse(termAggs, expectedElasticsearchQueryJSON) - }) - Convey("Test Filters Aggregates", func() { - expectedElasticsearchQueryJSON := `{ - "size": 0, - "query": { - "bool": { - "filter": [ - { - "range": { - "time": { - "gte": "", - "lte": "", - "format": "epoch_millis" - } - } - }, - { - "query_string": { - "analyze_wildcard": true, - "query": "*" - } - } - ] - } - }, - "aggs": { - "3": { - "filters": { - "filters": { - "hello": { - "query_string": { - "query": "host:\"67.65.185.232\"", - "analyze_wildcard": true - } - } - } - }, - "aggs": { - "2": { - "date_histogram": { - "interval": "200ms", - "field": "time", - "min_doc_count": 0, - "extended_bounds": { - "min": "", - "max": "" - }, - "format": "epoch_millis" - }, - "aggs": {} - } - } - } - } - } - ` - - testElasticSearchResponse(filtersAggs, expectedElasticsearchQueryJSON) - }) - }) -} - -func makeTime(hour int) string { - //unixtime 1500000000 == 2017-07-14T02:40:00+00:00 - return strconv.Itoa((1500000000 + hour*60*60) * 1000) -} - -func getIndexListByTime(pattern string, interval string, hour int) string { - timeRange := &tsdb.TimeRange{ - From: makeTime(0), - To: makeTime(hour), - } - return getIndexList(pattern, interval, timeRange) -} - -func TestElasticsearchGetIndexList(t *testing.T) { - Convey("Test Elasticsearch getIndex ", t, func() { - - Convey("Parse Interval Formats", func() { - So(getIndexListByTime("[logstash-]YYYY.MM.DD", "Daily", 48), - ShouldEqual, "logstash-2017.07.14,logstash-2017.07.15,logstash-2017.07.16") - - So(len(strings.Split(getIndexListByTime("[logstash-]YYYY.MM.DD.HH", "Hourly", 3), ",")), - ShouldEqual, 4) - - So(getIndexListByTime("[logstash-]YYYY.W", "Weekly", 100), - ShouldEqual, "logstash-2017.28,logstash-2017.29") - - So(getIndexListByTime("[logstash-]YYYY.MM", "Monthly", 700), - ShouldEqual, "logstash-2017.07,logstash-2017.08") - - So(getIndexListByTime("[logstash-]YYYY", "Yearly", 10000), - ShouldEqual, "logstash-2017,logstash-2018,logstash-2019") - }) - - Convey("No Interval", func() { - index := getIndexListByTime("logstash-test", "", 1) - So(index, ShouldEqual, "logstash-test") - }) - }) -} diff --git a/pkg/tsdb/elasticsearch/response_parser.go b/pkg/tsdb/elasticsearch/response_parser.go index ec7d2f9eb08..029b2e02142 100644 --- a/pkg/tsdb/elasticsearch/response_parser.go +++ b/pkg/tsdb/elasticsearch/response_parser.go @@ -2,39 +2,79 @@ package elasticsearch import ( "errors" - "fmt" + "regexp" + "sort" + "strconv" + "strings" + "github.com/grafana/grafana/pkg/components/null" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/tsdb" - "regexp" - "strconv" - "strings" + "github.com/grafana/grafana/pkg/tsdb/elasticsearch/client" ) -type ElasticsearchResponseParser struct { - Responses []Response +type responseParser struct { + Responses []*es.SearchResponse Targets []*Query } -func (rp *ElasticsearchResponseParser) getTimeSeries() *tsdb.QueryResult { - queryRes := tsdb.NewQueryResult() - for i, res := range rp.Responses { - target := rp.Targets[i] - props := make(map[string]string) - series := make([]*tsdb.TimeSeries, 0) - rp.processBuckets(res.Aggregations, target, &series, props, 0) - rp.nameSeries(&series, target) - queryRes.Series = append(queryRes.Series, series...) +var newResponseParser = func(responses []*es.SearchResponse, targets []*Query) *responseParser { + return &responseParser{ + Responses: responses, + Targets: targets, } - return queryRes } -func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{}, target *Query, series *[]*tsdb.TimeSeries, props map[string]string, depth int) error { +func (rp *responseParser) getTimeSeries() (*tsdb.Response, error) { + result := &tsdb.Response{} + result.Results = make(map[string]*tsdb.QueryResult) + if rp.Responses == nil { + return result, nil + } + + for i, res := range rp.Responses { + target := rp.Targets[i] + + if res.Error != nil { + result.Results[target.RefID] = getErrorFromElasticResponse(res) + continue + } + + queryRes := tsdb.NewQueryResult() + props := make(map[string]string) + table := tsdb.Table{ + Columns: make([]tsdb.TableColumn, 0), + Rows: make([]tsdb.RowValues, 0), + } + err := rp.processBuckets(res.Aggregations, target, &queryRes.Series, &table, props, 0) + if err != nil { + return nil, err + } + rp.nameSeries(&queryRes.Series, target) + rp.trimDatapoints(&queryRes.Series, target) + + if len(table.Rows) > 0 { + queryRes.Tables = append(queryRes.Tables, &table) + } + + result.Results[target.RefID] = queryRes + } + return result, nil +} + +func (rp *responseParser) processBuckets(aggs map[string]interface{}, target *Query, series *tsdb.TimeSeriesSlice, table *tsdb.Table, props map[string]string, depth int) error { var err error maxDepth := len(target.BucketAggs) - 1 - for aggId, v := range aggs { - aggDef, _ := findAgg(target, aggId) + + aggIDs := make([]string, 0) + for k := range aggs { + aggIDs = append(aggIDs, k) + } + sort.Strings(aggIDs) + for _, aggID := range aggIDs { + v := aggs[aggID] + aggDef, _ := findAgg(target, aggID) esAgg := simplejson.NewFromAny(v) if aggDef == nil { continue @@ -43,26 +83,50 @@ func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{ if depth == maxDepth { if aggDef.Type == "date_histogram" { err = rp.processMetrics(esAgg, target, series, props) - if err != nil { - return err - } } else { - return fmt.Errorf("not support type:%s", aggDef.Type) + err = rp.processAggregationDocs(esAgg, aggDef, target, table, props) + } + if err != nil { + return err } } else { - for i, b := range esAgg.Get("buckets").MustArray() { + for _, b := range esAgg.Get("buckets").MustArray() { bucket := simplejson.NewFromAny(b) - newProps := props + newProps := make(map[string]string, 0) + + for k, v := range props { + newProps[k] = v + } + if key, err := bucket.Get("key").String(); err == nil { newProps[aggDef.Field] = key - } else { - props["filter"] = strconv.Itoa(i) + } else if key, err := bucket.Get("key").Int64(); err == nil { + newProps[aggDef.Field] = strconv.FormatInt(key, 10) } if key, err := bucket.Get("key_as_string").String(); err == nil { - props[aggDef.Field] = key + newProps[aggDef.Field] = key + } + err = rp.processBuckets(bucket.MustMap(), target, series, table, newProps, depth+1) + if err != nil { + return err + } + } + + for k, v := range esAgg.Get("buckets").MustMap() { + bucket := simplejson.NewFromAny(v) + newProps := make(map[string]string, 0) + + for k, v := range props { + newProps[k] = v + } + + newProps["filter"] = k + + err = rp.processBuckets(bucket.MustMap(), target, series, table, newProps, depth+1) + if err != nil { + return err } - rp.processBuckets(bucket.MustMap(), target, series, newProps, depth+1) } } @@ -71,7 +135,7 @@ func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{ } -func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, target *Query, series *[]*tsdb.TimeSeries, props map[string]string) error { +func (rp *responseParser) processMetrics(esAgg *simplejson.Json, target *Query, series *tsdb.TimeSeriesSlice, props map[string]string) error { for _, metric := range target.Metrics { if metric.Hide { continue @@ -79,14 +143,20 @@ func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, ta switch metric.Type { case "count": - newSeries := tsdb.TimeSeries{} + newSeries := tsdb.TimeSeries{ + Tags: make(map[string]string), + } + for _, v := range esAgg.Get("buckets").MustArray() { bucket := simplejson.NewFromAny(v) value := castToNullFloat(bucket.Get("doc_count")) key := castToNullFloat(bucket.Get("key")) newSeries.Points = append(newSeries.Points, tsdb.TimePoint{value, key}) } - newSeries.Tags = props + + for k, v := range props { + newSeries.Tags[k] = v + } newSeries.Tags["metric"] = "count" *series = append(*series, &newSeries) @@ -99,9 +169,18 @@ func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, ta firstBucket := simplejson.NewFromAny(buckets[0]) percentiles := firstBucket.GetPath(metric.ID, "values").MustMap() - for percentileName := range percentiles { - newSeries := tsdb.TimeSeries{} - newSeries.Tags = props + percentileKeys := make([]string, 0) + for k := range percentiles { + percentileKeys = append(percentileKeys, k) + } + sort.Strings(percentileKeys) + for _, percentileName := range percentileKeys { + newSeries := tsdb.TimeSeries{ + Tags: make(map[string]string), + } + for k, v := range props { + newSeries.Tags[k] = v + } newSeries.Tags["metric"] = "p" + percentileName newSeries.Tags["field"] = metric.Field for _, v := range buckets { @@ -112,9 +191,49 @@ func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, ta } *series = append(*series, &newSeries) } + case "extended_stats": + buckets := esAgg.Get("buckets").MustArray() + + metaKeys := make([]string, 0) + meta := metric.Meta.MustMap() + for k := range meta { + metaKeys = append(metaKeys, k) + } + sort.Strings(metaKeys) + for _, statName := range metaKeys { + v := meta[statName] + if enabled, ok := v.(bool); !ok || !enabled { + continue + } + + newSeries := tsdb.TimeSeries{ + Tags: make(map[string]string), + } + for k, v := range props { + newSeries.Tags[k] = v + } + newSeries.Tags["metric"] = statName + newSeries.Tags["field"] = metric.Field + + for _, v := range buckets { + bucket := simplejson.NewFromAny(v) + key := castToNullFloat(bucket.Get("key")) + var value null.Float + if statName == "std_deviation_bounds_upper" { + value = castToNullFloat(bucket.GetPath(metric.ID, "std_deviation_bounds", "upper")) + } else if statName == "std_deviation_bounds_lower" { + value = castToNullFloat(bucket.GetPath(metric.ID, "std_deviation_bounds", "lower")) + } else { + value = castToNullFloat(bucket.GetPath(metric.ID, statName)) + } + newSeries.Points = append(newSeries.Points, tsdb.TimePoint{value, key}) + } + *series = append(*series, &newSeries) + } default: - newSeries := tsdb.TimeSeries{} - newSeries.Tags = map[string]string{} + newSeries := tsdb.TimeSeries{ + Tags: make(map[string]string), + } for k, v := range props { newSeries.Tags[k] = v } @@ -142,7 +261,129 @@ func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, ta return nil } -func (rp *ElasticsearchResponseParser) nameSeries(seriesList *[]*tsdb.TimeSeries, target *Query) { +func (rp *responseParser) processAggregationDocs(esAgg *simplejson.Json, aggDef *BucketAgg, target *Query, table *tsdb.Table, props map[string]string) error { + propKeys := make([]string, 0) + for k := range props { + propKeys = append(propKeys, k) + } + sort.Strings(propKeys) + + if len(table.Columns) == 0 { + for _, propKey := range propKeys { + table.Columns = append(table.Columns, tsdb.TableColumn{Text: propKey}) + } + table.Columns = append(table.Columns, tsdb.TableColumn{Text: aggDef.Field}) + } + + addMetricValue := func(values *tsdb.RowValues, metricName string, value null.Float) { + found := false + for _, c := range table.Columns { + if c.Text == metricName { + found = true + break + } + } + if !found { + table.Columns = append(table.Columns, tsdb.TableColumn{Text: metricName}) + } + *values = append(*values, value) + } + + for _, v := range esAgg.Get("buckets").MustArray() { + bucket := simplejson.NewFromAny(v) + values := make(tsdb.RowValues, 0) + + for _, propKey := range propKeys { + values = append(values, props[propKey]) + } + + if key, err := bucket.Get("key").String(); err == nil { + values = append(values, key) + } else { + values = append(values, castToNullFloat(bucket.Get("key"))) + } + + for _, metric := range target.Metrics { + switch metric.Type { + case "count": + addMetricValue(&values, rp.getMetricName(metric.Type), castToNullFloat(bucket.Get("doc_count"))) + break + case "extended_stats": + metaKeys := make([]string, 0) + meta := metric.Meta.MustMap() + for k := range meta { + metaKeys = append(metaKeys, k) + } + sort.Strings(metaKeys) + for _, statName := range metaKeys { + v := meta[statName] + if enabled, ok := v.(bool); !ok || !enabled { + continue + } + + var value null.Float + if statName == "std_deviation_bounds_upper" { + value = castToNullFloat(bucket.GetPath(metric.ID, "std_deviation_bounds", "upper")) + } else if statName == "std_deviation_bounds_lower" { + value = castToNullFloat(bucket.GetPath(metric.ID, "std_deviation_bounds", "lower")) + } else { + value = castToNullFloat(bucket.GetPath(metric.ID, statName)) + } + + addMetricValue(&values, rp.getMetricName(metric.Type), value) + break + } + default: + metricName := rp.getMetricName(metric.Type) + otherMetrics := make([]*MetricAgg, 0) + + for _, m := range target.Metrics { + if m.Type == metric.Type { + otherMetrics = append(otherMetrics, m) + } + } + + if len(otherMetrics) > 1 { + metricName += " " + metric.Field + } + + addMetricValue(&values, metricName, castToNullFloat(bucket.GetPath(metric.ID, "value"))) + break + } + } + + table.Rows = append(table.Rows, values) + } + + return nil +} + +func (rp *responseParser) trimDatapoints(series *tsdb.TimeSeriesSlice, target *Query) { + var histogram *BucketAgg + for _, bucketAgg := range target.BucketAggs { + if bucketAgg.Type == "date_histogram" { + histogram = bucketAgg + break + } + } + + if histogram == nil { + return + } + + trimEdges, err := histogram.Settings.Get("trimEdges").Int() + if err != nil { + return + } + + for _, s := range *series { + if len(s.Points) > trimEdges*2 { + s.Points = s.Points[trimEdges : len(s.Points)-trimEdges] + } + } +} + +func (rp *responseParser) nameSeries(seriesList *tsdb.TimeSeriesSlice, target *Query) { set := make(map[string]string) for _, v := range *seriesList { if metricType, exists := v.Tags["metric"]; exists { @@ -158,7 +399,9 @@ func (rp *ElasticsearchResponseParser) nameSeries(seriesList *[]*tsdb.TimeSeries } -func (rp *ElasticsearchResponseParser) getSeriesName(series *tsdb.TimeSeries, target *Query, metricTypeCount int) string { +var aliasPatternRegex = regexp.MustCompile(`\{\{([\s\S]+?)\}\}`) + +func (rp *responseParser) getSeriesName(series *tsdb.TimeSeries, target *Query, metricTypeCount int) string { metricType := series.Tags["metric"] metricName := rp.getMetricName(metricType) delete(series.Tags, "metric") @@ -170,27 +413,31 @@ func (rp *ElasticsearchResponseParser) getSeriesName(series *tsdb.TimeSeries, ta } if target.Alias != "" { - var re = regexp.MustCompile(`{{([\s\S]+?)}}`) - for _, match := range re.FindAllString(target.Alias, -1) { - group := match[2 : len(match)-2] + seriesName := target.Alias - if strings.HasPrefix(group, "term ") { - if term, ok := series.Tags["term "]; ok { - strings.Replace(target.Alias, match, term, 1) - } + subMatches := aliasPatternRegex.FindAllStringSubmatch(target.Alias, -1) + for _, subMatch := range subMatches { + group := subMatch[0] + + if len(subMatch) > 1 { + group = subMatch[1] + } + + if strings.Index(group, "term ") == 0 { + seriesName = strings.Replace(seriesName, subMatch[0], series.Tags[group[5:]], 1) } if v, ok := series.Tags[group]; ok { - strings.Replace(target.Alias, match, v, 1) + seriesName = strings.Replace(seriesName, subMatch[0], v, 1) } - - switch group { - case "metric": - strings.Replace(target.Alias, match, metricName, 1) - case "field": - strings.Replace(target.Alias, match, field, 1) + if group == "metric" { + seriesName = strings.Replace(seriesName, subMatch[0], metricName, 1) + } + if group == "field" { + seriesName = strings.Replace(seriesName, subMatch[0], field, 1) } - } + + return seriesName } // todo, if field and pipelineAgg if field != "" && isPipelineAgg(metricType) { @@ -204,7 +451,6 @@ func (rp *ElasticsearchResponseParser) getSeriesName(series *tsdb.TimeSeries, ta if !found { metricName = "Unset" } - } else if field != "" { metricName += " " + field } @@ -226,7 +472,7 @@ func (rp *ElasticsearchResponseParser) getSeriesName(series *tsdb.TimeSeries, ta } -func (rp *ElasticsearchResponseParser) getMetricName(metric string) string { +func (rp *responseParser) getMetricName(metric string) string { if text, ok := metricAggType[metric]; ok { return text } @@ -253,11 +499,28 @@ func castToNullFloat(j *simplejson.Json) null.Float { return null.NewFloat(0, false) } -func findAgg(target *Query, aggId string) (*BucketAgg, error) { +func findAgg(target *Query, aggID string) (*BucketAgg, error) { for _, v := range target.BucketAggs { - if aggId == v.ID { + if aggID == v.ID { return v, nil } } - return nil, errors.New("can't found aggDef, aggID:" + aggId) + return nil, errors.New("can't found aggDef, aggID:" + aggID) +} + +func getErrorFromElasticResponse(response *es.SearchResponse) *tsdb.QueryResult { + result := tsdb.NewQueryResult() + json := simplejson.NewFromAny(response.Error) + reason := json.Get("reason").MustString() + rootCauseReason := json.Get("root_cause").GetIndex(0).Get("reason").MustString() + + if rootCauseReason != "" { + result.ErrorString = rootCauseReason + } else if reason != "" { + result.ErrorString = reason + } else { + result.ErrorString = "Unkown elasticsearch error response" + } + + return result } diff --git a/pkg/tsdb/elasticsearch/response_parser_test.go b/pkg/tsdb/elasticsearch/response_parser_test.go index 1df2c4551ae..b00c14cf946 100644 --- a/pkg/tsdb/elasticsearch/response_parser_test.go +++ b/pkg/tsdb/elasticsearch/response_parser_test.go @@ -2,109 +2,879 @@ package elasticsearch import ( "encoding/json" + "fmt" "testing" + "time" + + "github.com/grafana/grafana/pkg/components/null" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/tsdb/elasticsearch/client" "github.com/grafana/grafana/pkg/tsdb" . "github.com/smartystreets/goconvey/convey" ) -func testElasticsearchResponse(body string, target Query) *tsdb.QueryResult { - var responses Responses - err := json.Unmarshal([]byte(body), &responses) - So(err, ShouldBeNil) - - responseParser := ElasticsearchResponseParser{responses.Responses, []*Query{&target}} - return responseParser.getTimeSeries() -} - -func TestElasticSearchResponseParser(t *testing.T) { - Convey("Elasticsearch Response query testing", t, func() { - Convey("Build test average metric with moving average", func() { - responses := `{ - "responses": [ - { - "took": 1, - "timed_out": false, - "_shards": { - "total": 5, - "successful": 5, - "skipped": 0, - "failed": 0 - }, - "hits": { - "total": 4500, - "max_score": 0, - "hits": [] - }, - "aggregations": { - "2": { - "buckets": [ - { - "1": { - "value": null - }, - "key_as_string": "1522205880000", - "key": 1522205880000, - "doc_count": 0 - }, - { - "1": { - "value": 10 - }, - "key_as_string": "1522205940000", - "key": 1522205940000, - "doc_count": 300 - }, - { - "1": { - "value": 10 - }, - "3": { - "value": 20 - }, - "key_as_string": "1522206000000", - "key": 1522206000000, - "doc_count": 300 - }, - { - "1": { - "value": 10 - }, - "3": { - "value": 20 - }, - "key_as_string": "1522206060000", - "key": 1522206060000, - "doc_count": 300 +func TestResponseParser(t *testing.T) { + Convey("Elasticsearch response parser test", t, func() { + Convey("Simple query and count", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "count", "id": "1" }], + "bucketAggs": [{ "type": "date_histogram", "field": "@timestamp", "id": "2" }] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "2": { + "buckets": [ + { + "doc_count": 10, + "key": 1000 + }, + { + "doc_count": 15, + "key": 2000 + } + ] + } } - ] - } - }, - "status": 200 - } - ] -} -` - res := testElasticsearchResponse(responses, avgWithMovingAvg) - So(len(res.Series), ShouldEqual, 2) - So(res.Series[0].Name, ShouldEqual, "Average value") - So(len(res.Series[0].Points), ShouldEqual, 4) - for i, p := range res.Series[0].Points { - if i == 0 { - So(p[0].Valid, ShouldBeFalse) - } else { - So(p[0].Float64, ShouldEqual, 10) - } - So(p[1].Float64, ShouldEqual, 1522205880000+60000*i) - } - - So(res.Series[1].Name, ShouldEqual, "Moving Average Average 1") - So(len(res.Series[1].Points), ShouldEqual, 2) - - for _, p := range res.Series[1].Points { - So(p[0].Float64, ShouldEqual, 20) - } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Series, ShouldHaveLength, 1) + series := queryRes.Series[0] + So(series.Name, ShouldEqual, "Count") + So(series.Points, ShouldHaveLength, 2) + So(series.Points[0][0].Float64, ShouldEqual, 10) + So(series.Points[0][1].Float64, ShouldEqual, 1000) + So(series.Points[1][0].Float64, ShouldEqual, 15) + So(series.Points[1][1].Float64, ShouldEqual, 2000) }) + + Convey("Simple query count & avg aggregation", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "count", "id": "1" }, {"type": "avg", "field": "value", "id": "2" }], + "bucketAggs": [{ "type": "date_histogram", "field": "@timestamp", "id": "3" }] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "3": { + "buckets": [ + { + "2": { "value": 88 }, + "doc_count": 10, + "key": 1000 + }, + { + "2": { "value": 99 }, + "doc_count": 15, + "key": 2000 + } + ] + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Series, ShouldHaveLength, 2) + seriesOne := queryRes.Series[0] + So(seriesOne.Name, ShouldEqual, "Count") + So(seriesOne.Points, ShouldHaveLength, 2) + So(seriesOne.Points[0][0].Float64, ShouldEqual, 10) + So(seriesOne.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesOne.Points[1][0].Float64, ShouldEqual, 15) + So(seriesOne.Points[1][1].Float64, ShouldEqual, 2000) + + seriesTwo := queryRes.Series[1] + So(seriesTwo.Name, ShouldEqual, "Average value") + So(seriesTwo.Points, ShouldHaveLength, 2) + So(seriesTwo.Points[0][0].Float64, ShouldEqual, 88) + So(seriesTwo.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesTwo.Points[1][0].Float64, ShouldEqual, 99) + So(seriesTwo.Points[1][1].Float64, ShouldEqual, 2000) + }) + + Convey("Single group by query one metric", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "count", "id": "1" }], + "bucketAggs": [ + { "type": "terms", "field": "host", "id": "2" }, + { "type": "date_histogram", "field": "@timestamp", "id": "3" } + ] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "2": { + "buckets": [ + { + "3": { + "buckets": [{ "doc_count": 1, "key": 1000 }, { "doc_count": 3, "key": 2000 }] + }, + "doc_count": 4, + "key": "server1" + }, + { + "3": { + "buckets": [{ "doc_count": 2, "key": 1000 }, { "doc_count": 8, "key": 2000 }] + }, + "doc_count": 10, + "key": "server2" + } + ] + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Series, ShouldHaveLength, 2) + seriesOne := queryRes.Series[0] + So(seriesOne.Name, ShouldEqual, "server1") + So(seriesOne.Points, ShouldHaveLength, 2) + So(seriesOne.Points[0][0].Float64, ShouldEqual, 1) + So(seriesOne.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesOne.Points[1][0].Float64, ShouldEqual, 3) + So(seriesOne.Points[1][1].Float64, ShouldEqual, 2000) + + seriesTwo := queryRes.Series[1] + So(seriesTwo.Name, ShouldEqual, "server2") + So(seriesTwo.Points, ShouldHaveLength, 2) + So(seriesTwo.Points[0][0].Float64, ShouldEqual, 2) + So(seriesTwo.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesTwo.Points[1][0].Float64, ShouldEqual, 8) + So(seriesTwo.Points[1][1].Float64, ShouldEqual, 2000) + }) + + Convey("Single group by query two metrics", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "count", "id": "1" }, { "type": "avg", "field": "@value", "id": "4" }], + "bucketAggs": [ + { "type": "terms", "field": "host", "id": "2" }, + { "type": "date_histogram", "field": "@timestamp", "id": "3" } + ] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "2": { + "buckets": [ + { + "3": { + "buckets": [ + { "4": { "value": 10 }, "doc_count": 1, "key": 1000 }, + { "4": { "value": 12 }, "doc_count": 3, "key": 2000 } + ] + }, + "doc_count": 4, + "key": "server1" + }, + { + "3": { + "buckets": [ + { "4": { "value": 20 }, "doc_count": 1, "key": 1000 }, + { "4": { "value": 32 }, "doc_count": 3, "key": 2000 } + ] + }, + "doc_count": 10, + "key": "server2" + } + ] + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Series, ShouldHaveLength, 4) + seriesOne := queryRes.Series[0] + So(seriesOne.Name, ShouldEqual, "server1 Count") + So(seriesOne.Points, ShouldHaveLength, 2) + So(seriesOne.Points[0][0].Float64, ShouldEqual, 1) + So(seriesOne.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesOne.Points[1][0].Float64, ShouldEqual, 3) + So(seriesOne.Points[1][1].Float64, ShouldEqual, 2000) + + seriesTwo := queryRes.Series[1] + So(seriesTwo.Name, ShouldEqual, "server1 Average @value") + So(seriesTwo.Points, ShouldHaveLength, 2) + So(seriesTwo.Points[0][0].Float64, ShouldEqual, 10) + So(seriesTwo.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesTwo.Points[1][0].Float64, ShouldEqual, 12) + So(seriesTwo.Points[1][1].Float64, ShouldEqual, 2000) + + seriesThree := queryRes.Series[2] + So(seriesThree.Name, ShouldEqual, "server2 Count") + So(seriesThree.Points, ShouldHaveLength, 2) + So(seriesThree.Points[0][0].Float64, ShouldEqual, 1) + So(seriesThree.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesThree.Points[1][0].Float64, ShouldEqual, 3) + So(seriesThree.Points[1][1].Float64, ShouldEqual, 2000) + + seriesFour := queryRes.Series[3] + So(seriesFour.Name, ShouldEqual, "server2 Average @value") + So(seriesFour.Points, ShouldHaveLength, 2) + So(seriesFour.Points[0][0].Float64, ShouldEqual, 20) + So(seriesFour.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesFour.Points[1][0].Float64, ShouldEqual, 32) + So(seriesFour.Points[1][1].Float64, ShouldEqual, 2000) + }) + + Convey("With percentiles", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "percentiles", "settings": { "percents": [75, 90] }, "id": "1" }], + "bucketAggs": [{ "type": "date_histogram", "field": "@timestamp", "id": "3" }] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "3": { + "buckets": [ + { + "1": { "values": { "75": 3.3, "90": 5.5 } }, + "doc_count": 10, + "key": 1000 + }, + { + "1": { "values": { "75": 2.3, "90": 4.5 } }, + "doc_count": 15, + "key": 2000 + } + ] + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Series, ShouldHaveLength, 2) + seriesOne := queryRes.Series[0] + So(seriesOne.Name, ShouldEqual, "p75") + So(seriesOne.Points, ShouldHaveLength, 2) + So(seriesOne.Points[0][0].Float64, ShouldEqual, 3.3) + So(seriesOne.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesOne.Points[1][0].Float64, ShouldEqual, 2.3) + So(seriesOne.Points[1][1].Float64, ShouldEqual, 2000) + + seriesTwo := queryRes.Series[1] + So(seriesTwo.Name, ShouldEqual, "p90") + So(seriesTwo.Points, ShouldHaveLength, 2) + So(seriesTwo.Points[0][0].Float64, ShouldEqual, 5.5) + So(seriesTwo.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesTwo.Points[1][0].Float64, ShouldEqual, 4.5) + So(seriesTwo.Points[1][1].Float64, ShouldEqual, 2000) + }) + + Convey("With extended stats", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "extended_stats", "meta": { "max": true, "std_deviation_bounds_upper": true, "std_deviation_bounds_lower": true }, "id": "1" }], + "bucketAggs": [ + { "type": "terms", "field": "host", "id": "3" }, + { "type": "date_histogram", "field": "@timestamp", "id": "4" } + ] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "3": { + "buckets": [ + { + "key": "server1", + "4": { + "buckets": [ + { + "1": { + "max": 10.2, + "min": 5.5, + "std_deviation_bounds": { "upper": 3, "lower": -2 } + }, + "doc_count": 10, + "key": 1000 + } + ] + } + }, + { + "key": "server2", + "4": { + "buckets": [ + { + "1": { + "max": 15.5, + "min": 3.4, + "std_deviation_bounds": { "upper": 4, "lower": -1 } + }, + "doc_count": 10, + "key": 1000 + } + ] + } + } + ] + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Series, ShouldHaveLength, 6) + + seriesOne := queryRes.Series[0] + So(seriesOne.Name, ShouldEqual, "server1 Max") + So(seriesOne.Points, ShouldHaveLength, 1) + So(seriesOne.Points[0][0].Float64, ShouldEqual, 10.2) + So(seriesOne.Points[0][1].Float64, ShouldEqual, 1000) + + seriesTwo := queryRes.Series[1] + So(seriesTwo.Name, ShouldEqual, "server1 Std Dev Lower") + So(seriesTwo.Points, ShouldHaveLength, 1) + So(seriesTwo.Points[0][0].Float64, ShouldEqual, -2) + So(seriesTwo.Points[0][1].Float64, ShouldEqual, 1000) + + seriesThree := queryRes.Series[2] + So(seriesThree.Name, ShouldEqual, "server1 Std Dev Upper") + So(seriesThree.Points, ShouldHaveLength, 1) + So(seriesThree.Points[0][0].Float64, ShouldEqual, 3) + So(seriesThree.Points[0][1].Float64, ShouldEqual, 1000) + + seriesFour := queryRes.Series[3] + So(seriesFour.Name, ShouldEqual, "server2 Max") + So(seriesFour.Points, ShouldHaveLength, 1) + So(seriesFour.Points[0][0].Float64, ShouldEqual, 15.5) + So(seriesFour.Points[0][1].Float64, ShouldEqual, 1000) + + seriesFive := queryRes.Series[4] + So(seriesFive.Name, ShouldEqual, "server2 Std Dev Lower") + So(seriesFive.Points, ShouldHaveLength, 1) + So(seriesFive.Points[0][0].Float64, ShouldEqual, -1) + So(seriesFive.Points[0][1].Float64, ShouldEqual, 1000) + + seriesSix := queryRes.Series[5] + So(seriesSix.Name, ShouldEqual, "server2 Std Dev Upper") + So(seriesSix.Points, ShouldHaveLength, 1) + So(seriesSix.Points[0][0].Float64, ShouldEqual, 4) + So(seriesSix.Points[0][1].Float64, ShouldEqual, 1000) + }) + + Convey("Single group by with alias pattern", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "alias": "{{term @host}} {{metric}} and {{not_exist}} {{@host}}", + "metrics": [{ "type": "count", "id": "1" }], + "bucketAggs": [ + { "type": "terms", "field": "@host", "id": "2" }, + { "type": "date_histogram", "field": "@timestamp", "id": "3" } + ] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "2": { + "buckets": [ + { + "3": { + "buckets": [{ "doc_count": 1, "key": 1000 }, { "doc_count": 3, "key": 2000 }] + }, + "doc_count": 4, + "key": "server1" + }, + { + "3": { + "buckets": [{ "doc_count": 2, "key": 1000 }, { "doc_count": 8, "key": 2000 }] + }, + "doc_count": 10, + "key": "server2" + }, + { + "3": { + "buckets": [{ "doc_count": 2, "key": 1000 }, { "doc_count": 8, "key": 2000 }] + }, + "doc_count": 10, + "key": 0 + } + ] + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Series, ShouldHaveLength, 3) + + seriesOne := queryRes.Series[0] + So(seriesOne.Name, ShouldEqual, "server1 Count and {{not_exist}} server1") + So(seriesOne.Points, ShouldHaveLength, 2) + So(seriesOne.Points[0][0].Float64, ShouldEqual, 1) + So(seriesOne.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesOne.Points[1][0].Float64, ShouldEqual, 3) + So(seriesOne.Points[1][1].Float64, ShouldEqual, 2000) + + seriesTwo := queryRes.Series[1] + So(seriesTwo.Name, ShouldEqual, "server2 Count and {{not_exist}} server2") + So(seriesTwo.Points, ShouldHaveLength, 2) + So(seriesTwo.Points[0][0].Float64, ShouldEqual, 2) + So(seriesTwo.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesTwo.Points[1][0].Float64, ShouldEqual, 8) + So(seriesTwo.Points[1][1].Float64, ShouldEqual, 2000) + + seriesThree := queryRes.Series[2] + So(seriesThree.Name, ShouldEqual, "0 Count and {{not_exist}} 0") + So(seriesThree.Points, ShouldHaveLength, 2) + So(seriesThree.Points[0][0].Float64, ShouldEqual, 2) + So(seriesThree.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesThree.Points[1][0].Float64, ShouldEqual, 8) + So(seriesThree.Points[1][1].Float64, ShouldEqual, 2000) + }) + + Convey("Histogram response", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "count", "id": "1" }], + "bucketAggs": [{ "type": "histogram", "field": "bytes", "id": "3" }] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "3": { + "buckets": [{ "doc_count": 1, "key": 1000 }, { "doc_count": 3, "key": 2000 }, { "doc_count": 2, "key": 3000 }] + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Tables, ShouldHaveLength, 1) + + rows := queryRes.Tables[0].Rows + So(rows, ShouldHaveLength, 3) + cols := queryRes.Tables[0].Columns + So(cols, ShouldHaveLength, 2) + + So(cols[0].Text, ShouldEqual, "bytes") + So(cols[1].Text, ShouldEqual, "Count") + + So(rows[0][0].(null.Float).Float64, ShouldEqual, 1000) + So(rows[0][1].(null.Float).Float64, ShouldEqual, 1) + So(rows[1][0].(null.Float).Float64, ShouldEqual, 2000) + So(rows[1][1].(null.Float).Float64, ShouldEqual, 3) + So(rows[2][0].(null.Float).Float64, ShouldEqual, 3000) + So(rows[2][1].(null.Float).Float64, ShouldEqual, 2) + }) + + Convey("With two filters agg", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "count", "id": "1" }], + "bucketAggs": [ + { + "type": "filters", + "id": "2", + "settings": { + "filters": [{ "query": "@metric:cpu" }, { "query": "@metric:logins.count" }] + } + }, + { "type": "date_histogram", "field": "@timestamp", "id": "3" } + ] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "2": { + "buckets": { + "@metric:cpu": { + "3": { + "buckets": [{ "doc_count": 1, "key": 1000 }, { "doc_count": 3, "key": 2000 }] + } + }, + "@metric:logins.count": { + "3": { + "buckets": [{ "doc_count": 2, "key": 1000 }, { "doc_count": 8, "key": 2000 }] + } + } + } + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Series, ShouldHaveLength, 2) + + seriesOne := queryRes.Series[0] + So(seriesOne.Name, ShouldEqual, "@metric:cpu") + So(seriesOne.Points, ShouldHaveLength, 2) + So(seriesOne.Points[0][0].Float64, ShouldEqual, 1) + So(seriesOne.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesOne.Points[1][0].Float64, ShouldEqual, 3) + So(seriesOne.Points[1][1].Float64, ShouldEqual, 2000) + + seriesTwo := queryRes.Series[1] + So(seriesTwo.Name, ShouldEqual, "@metric:logins.count") + So(seriesTwo.Points, ShouldHaveLength, 2) + So(seriesTwo.Points[0][0].Float64, ShouldEqual, 2) + So(seriesTwo.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesTwo.Points[1][0].Float64, ShouldEqual, 8) + So(seriesTwo.Points[1][1].Float64, ShouldEqual, 2000) + }) + + Convey("With dropfirst and last aggregation", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "avg", "id": "1" }, { "type": "count" }], + "bucketAggs": [ + { + "type": "date_histogram", + "field": "@timestamp", + "id": "2", + "settings": { "trimEdges": 1 } + } + ] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "2": { + "buckets": [ + { + "1": { "value": 1000 }, + "key": 1, + "doc_count": 369 + }, + { + "1": { "value": 2000 }, + "key": 2, + "doc_count": 200 + }, + { + "1": { "value": 2000 }, + "key": 3, + "doc_count": 200 + } + ] + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Series, ShouldHaveLength, 2) + + seriesOne := queryRes.Series[0] + So(seriesOne.Name, ShouldEqual, "Average") + So(seriesOne.Points, ShouldHaveLength, 1) + So(seriesOne.Points[0][0].Float64, ShouldEqual, 2000) + So(seriesOne.Points[0][1].Float64, ShouldEqual, 2) + + seriesTwo := queryRes.Series[1] + So(seriesTwo.Name, ShouldEqual, "Count") + So(seriesTwo.Points, ShouldHaveLength, 1) + So(seriesTwo.Points[0][0].Float64, ShouldEqual, 200) + So(seriesTwo.Points[0][1].Float64, ShouldEqual, 2) + }) + + Convey("No group by time", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "avg", "id": "1" }, { "type": "count" }], + "bucketAggs": [{ "type": "terms", "field": "host", "id": "2" }] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "2": { + "buckets": [ + { + "1": { "value": 1000 }, + "key": "server-1", + "doc_count": 369 + }, + { + "1": { "value": 2000 }, + "key": "server-2", + "doc_count": 200 + } + ] + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Tables, ShouldHaveLength, 1) + + rows := queryRes.Tables[0].Rows + So(rows, ShouldHaveLength, 2) + cols := queryRes.Tables[0].Columns + So(cols, ShouldHaveLength, 3) + + So(cols[0].Text, ShouldEqual, "host") + So(cols[1].Text, ShouldEqual, "Average") + So(cols[2].Text, ShouldEqual, "Count") + + So(rows[0][0].(string), ShouldEqual, "server-1") + So(rows[0][1].(null.Float).Float64, ShouldEqual, 1000) + So(rows[0][2].(null.Float).Float64, ShouldEqual, 369) + So(rows[1][0].(string), ShouldEqual, "server-2") + So(rows[1][1].(null.Float).Float64, ShouldEqual, 2000) + So(rows[1][2].(null.Float).Float64, ShouldEqual, 200) + }) + + Convey("Multiple metrics of same type", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "avg", "field": "test", "id": "1" }, { "type": "avg", "field": "test2", "id": "2" }], + "bucketAggs": [{ "type": "terms", "field": "host", "id": "2" }] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "2": { + "buckets": [ + { + "1": { "value": 1000 }, + "2": { "value": 3000 }, + "key": "server-1", + "doc_count": 369 + } + ] + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Tables, ShouldHaveLength, 1) + + rows := queryRes.Tables[0].Rows + So(rows, ShouldHaveLength, 1) + cols := queryRes.Tables[0].Columns + So(cols, ShouldHaveLength, 3) + + So(cols[0].Text, ShouldEqual, "host") + So(cols[1].Text, ShouldEqual, "Average test") + So(cols[2].Text, ShouldEqual, "Average test2") + + So(rows[0][0].(string), ShouldEqual, "server-1") + So(rows[0][1].(null.Float).Float64, ShouldEqual, 1000) + So(rows[0][2].(null.Float).Float64, ShouldEqual, 3000) + }) + + // Convey("Raw documents query", func() { + // targets := map[string]string{ + // "A": `{ + // "timeField": "@timestamp", + // "metrics": [{ "type": "raw_document", "id": "1" }] + // }`, + // } + // response := `{ + // "responses": [ + // { + // "hits": { + // "total": 100, + // "hits": [ + // { + // "_id": "1", + // "_type": "type", + // "_index": "index", + // "_source": { "sourceProp": "asd" }, + // "fields": { "fieldProp": "field" } + // }, + // { + // "_source": { "sourceProp": "asd2" }, + // "fields": { "fieldProp": "field2" } + // } + // ] + // } + // } + // ] + // }` + // rp, err := newResponseParserForTest(targets, response) + // So(err, ShouldBeNil) + // result, err := rp.getTimeSeries() + // So(err, ShouldBeNil) + // So(result.Results, ShouldHaveLength, 1) + + // queryRes := result.Results["A"] + // So(queryRes, ShouldNotBeNil) + // So(queryRes.Tables, ShouldHaveLength, 1) + + // rows := queryRes.Tables[0].Rows + // So(rows, ShouldHaveLength, 1) + // cols := queryRes.Tables[0].Columns + // So(cols, ShouldHaveLength, 3) + + // So(cols[0].Text, ShouldEqual, "host") + // So(cols[1].Text, ShouldEqual, "Average test") + // So(cols[2].Text, ShouldEqual, "Average test2") + + // So(rows[0][0].(string), ShouldEqual, "server-1") + // So(rows[0][1].(null.Float).Float64, ShouldEqual, 1000) + // So(rows[0][2].(null.Float).Float64, ShouldEqual, 3000) + // }) }) } + +func newResponseParserForTest(tsdbQueries map[string]string, responseBody string) (*responseParser, error) { + from := time.Date(2018, 5, 15, 17, 50, 0, 0, time.UTC) + to := time.Date(2018, 5, 15, 17, 55, 0, 0, time.UTC) + fromStr := fmt.Sprintf("%d", from.UnixNano()/int64(time.Millisecond)) + toStr := fmt.Sprintf("%d", to.UnixNano()/int64(time.Millisecond)) + tsdbQuery := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{}, + TimeRange: tsdb.NewTimeRange(fromStr, toStr), + } + + for refID, tsdbQueryBody := range tsdbQueries { + tsdbQueryJSON, err := simplejson.NewJson([]byte(tsdbQueryBody)) + if err != nil { + return nil, err + } + + tsdbQuery.Queries = append(tsdbQuery.Queries, &tsdb.Query{ + Model: tsdbQueryJSON, + RefId: refID, + }) + } + + var response es.MultiSearchResponse + err := json.Unmarshal([]byte(responseBody), &response) + if err != nil { + return nil, err + } + + tsQueryParser := newTimeSeriesQueryParser() + queries, err := tsQueryParser.parse(tsdbQuery) + if err != nil { + return nil, err + } + + return newResponseParser(response.Responses, queries), nil +} diff --git a/pkg/tsdb/elasticsearch/time_series_query.go b/pkg/tsdb/elasticsearch/time_series_query.go index af8f61eb144..ae4af7704fb 100644 --- a/pkg/tsdb/elasticsearch/time_series_query.go +++ b/pkg/tsdb/elasticsearch/time_series_query.go @@ -1,99 +1,246 @@ package elasticsearch import ( - "bytes" - "context" - "encoding/json" - "errors" "fmt" + "strconv" + "strings" "time" "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/tsdb" - "golang.org/x/net/context/ctxhttp" + "github.com/grafana/grafana/pkg/tsdb/elasticsearch/client" ) type timeSeriesQuery struct { - queries []*Query + client es.Client + tsdbQuery *tsdb.TsdbQuery + intervalCalculator tsdb.IntervalCalculator } -func (e *ElasticsearchExecutor) executeTimeSeriesQuery(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) { +var newTimeSeriesQuery = func(client es.Client, tsdbQuery *tsdb.TsdbQuery, intervalCalculator tsdb.IntervalCalculator) *timeSeriesQuery { + return &timeSeriesQuery{ + client: client, + tsdbQuery: tsdbQuery, + intervalCalculator: intervalCalculator, + } +} + +func (e *timeSeriesQuery) execute() (*tsdb.Response, error) { result := &tsdb.Response{} result.Results = make(map[string]*tsdb.QueryResult) - tsQueryParser := newTimeSeriesQueryParser(dsInfo) - query, err := tsQueryParser.parse(tsdbQuery) + tsQueryParser := newTimeSeriesQueryParser() + queries, err := tsQueryParser.parse(e.tsdbQuery) if err != nil { return nil, err } - buff := bytes.Buffer{} - for _, q := range query.queries { - s, err := q.Build(tsdbQuery, dsInfo) + ms := e.client.MultiSearch() + + from := fmt.Sprintf("%d", e.tsdbQuery.TimeRange.GetFromAsMsEpoch()) + to := fmt.Sprintf("%d", e.tsdbQuery.TimeRange.GetToAsMsEpoch()) + + for _, q := range queries { + minInterval, err := e.client.GetMinInterval(q.Interval) if err != nil { return nil, err } - buff.WriteString(s) - } - payload := buff.String() + interval := e.intervalCalculator.Calculate(e.tsdbQuery.TimeRange, minInterval) - if setting.Env == setting.DEV { - glog.Debug("Elasticsearch playload", "raw playload", payload) - } - glog.Info("Elasticsearch playload", "raw playload", payload) + b := ms.Search() + b.Size(0) + filters := b.Query().Bool().Filter() + filters.AddDateRangeFilter(e.client.GetTimeField(), to, from, es.DateFormatEpochMS) - req, err := e.createRequest(dsInfo, payload) - if err != nil { - return nil, err - } + if q.RawQuery != "" { + filters.AddQueryStringFilter(q.RawQuery, true) + } - httpClient, err := dsInfo.GetHttpClient() - if err != nil { - return nil, err - } + if len(q.BucketAggs) == 0 { + if len(q.Metrics) == 0 || q.Metrics[0].Type != "raw_document" { + result.Results[q.RefID] = &tsdb.QueryResult{ + RefId: q.RefID, + Error: fmt.Errorf("invalid query, missing metrics and aggregations"), + ErrorString: "invalid query, missing metrics and aggregations", + } + continue + } + metric := q.Metrics[0] + b.Size(metric.Settings.Get("size").MustInt(500)) + b.SortDesc("@timestamp", "boolean") + b.AddDocValueField("@timestamp") + continue + } - resp, err := ctxhttp.Do(ctx, httpClient, req) - if err != nil { - return nil, err - } + aggBuilder := b.Agg() - if resp.StatusCode/100 != 2 { - return nil, fmt.Errorf("elasticsearch returned statuscode invalid status code: %v", resp.Status) - } + // iterate backwards to create aggregations bottom-down + for _, bucketAgg := range q.BucketAggs { + switch bucketAgg.Type { + case "date_histogram": + aggBuilder = addDateHistogramAgg(aggBuilder, bucketAgg, from, to, interval) + case "histogram": + aggBuilder = addHistogramAgg(aggBuilder, bucketAgg) + case "filters": + aggBuilder = addFiltersAgg(aggBuilder, bucketAgg) + case "terms": + aggBuilder = addTermsAgg(aggBuilder, bucketAgg, q.Metrics) + case "geohash_grid": + aggBuilder = addGeoHashGridAgg(aggBuilder, bucketAgg) + } + } - var responses Responses - defer resp.Body.Close() - dec := json.NewDecoder(resp.Body) - dec.UseNumber() - err = dec.Decode(&responses) - if err != nil { - return nil, err - } + for _, m := range q.Metrics { + if m.Type == "count" { + continue + } - for _, res := range responses.Responses { - if res.Err != nil { - return nil, errors.New(res.getErrMsg()) + if isPipelineAgg(m.Type) { + if _, err := strconv.Atoi(m.PipelineAggregate); err == nil { + aggBuilder.Pipeline(m.ID, m.Type, m.PipelineAggregate, func(a *es.PipelineAggregation) { + a.Settings = m.Settings.MustMap() + }) + } else { + continue + } + } else { + aggBuilder.Metric(m.ID, m.Type, m.Field, func(a *es.MetricAggregation) { + a.Settings = m.Settings.MustMap() + }) + } } } - responseParser := ElasticsearchResponseParser{responses.Responses, query.queries} - queryRes := responseParser.getTimeSeries() - result.Results["A"] = queryRes - return result, nil -} -type timeSeriesQueryParser struct { - ds *models.DataSource -} - -func newTimeSeriesQueryParser(ds *models.DataSource) *timeSeriesQueryParser { - return &timeSeriesQueryParser{ - ds: ds, + req, err := ms.Build() + if err != nil { + return nil, err } + + res, err := e.client.ExecuteMultisearch(req) + if err != nil { + return nil, err + } + + rp := newResponseParser(res.Responses, queries) + return rp.getTimeSeries() } -func (p *timeSeriesQueryParser) parse(tsdbQuery *tsdb.TsdbQuery) (*timeSeriesQuery, error) { +func addDateHistogramAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg, timeFrom, timeTo string, interval tsdb.Interval) es.AggBuilder { + aggBuilder.DateHistogram(bucketAgg.ID, bucketAgg.Field, func(a *es.DateHistogramAgg, b es.AggBuilder) { + a.Interval = bucketAgg.Settings.Get("interval").MustString("auto") + a.MinDocCount = bucketAgg.Settings.Get("min_doc_count").MustInt(0) + a.ExtendedBounds = &es.ExtendedBounds{Min: timeFrom, Max: timeTo} + a.Format = bucketAgg.Settings.Get("format").MustString(es.DateFormatEpochMS) + + if a.Interval == "auto" { + a.Interval = "$__interval" + } + + a.Interval = strings.Replace(a.Interval, "$interval", interval.Text, -1) + a.Interval = strings.Replace(a.Interval, "$__interval_ms", strconv.FormatInt(interval.Value.Nanoseconds()/int64(time.Millisecond), 10), -1) + a.Interval = strings.Replace(a.Interval, "$__interval", interval.Text, -1) + + if missing, err := bucketAgg.Settings.Get("missing").String(); err == nil { + a.Missing = &missing + } + + aggBuilder = b + }) + + return aggBuilder +} + +func addHistogramAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg) es.AggBuilder { + aggBuilder.Histogram(bucketAgg.ID, bucketAgg.Field, func(a *es.HistogramAgg, b es.AggBuilder) { + a.Interval = bucketAgg.Settings.Get("interval").MustInt(1000) + a.MinDocCount = bucketAgg.Settings.Get("min_doc_count").MustInt(0) + + if missing, err := bucketAgg.Settings.Get("missing").Int(); err == nil { + a.Missing = &missing + } + + aggBuilder = b + }) + + return aggBuilder +} + +func addTermsAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg, metrics []*MetricAgg) es.AggBuilder { + aggBuilder.Terms(bucketAgg.ID, bucketAgg.Field, func(a *es.TermsAggregation, b es.AggBuilder) { + if size, err := bucketAgg.Settings.Get("size").Int(); err == nil { + a.Size = size + } else if size, err := bucketAgg.Settings.Get("size").String(); err == nil { + a.Size, err = strconv.Atoi(size) + if err != nil { + a.Size = 500 + } + } else { + a.Size = 500 + } + if minDocCount, err := bucketAgg.Settings.Get("min_doc_count").Int(); err == nil { + a.MinDocCount = &minDocCount + } + if missing, err := bucketAgg.Settings.Get("missing").String(); err == nil { + a.Missing = &missing + } + + if orderBy, err := bucketAgg.Settings.Get("orderBy").String(); err == nil { + a.Order[orderBy] = bucketAgg.Settings.Get("order").MustString("desc") + + if _, err := strconv.Atoi(orderBy); err == nil { + for _, m := range metrics { + if m.ID == orderBy { + b.Metric(m.ID, m.Type, m.Field, nil) + break + } + } + } + } + + aggBuilder = b + }) + + return aggBuilder +} + +func addFiltersAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg) es.AggBuilder { + filters := make(map[string]interface{}) + for _, filter := range bucketAgg.Settings.Get("filters").MustArray() { + json := simplejson.NewFromAny(filter) + query := json.Get("query").MustString() + label := json.Get("label").MustString() + if label == "" { + label = query + } + filters[label] = &es.QueryStringFilter{Query: query, AnalyzeWildcard: true} + } + + if len(filters) > 0 { + aggBuilder.Filters(bucketAgg.ID, func(a *es.FiltersAggregation, b es.AggBuilder) { + a.Filters = filters + aggBuilder = b + }) + } + + return aggBuilder +} + +func addGeoHashGridAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg) es.AggBuilder { + aggBuilder.GeoHashGrid(bucketAgg.ID, bucketAgg.Field, func(a *es.GeoHashGridAggregation, b es.AggBuilder) { + a.Precision = bucketAgg.Settings.Get("precision").MustInt(3) + aggBuilder = b + }) + + return aggBuilder +} + +type timeSeriesQueryParser struct{} + +func newTimeSeriesQueryParser() *timeSeriesQueryParser { + return &timeSeriesQueryParser{} +} + +func (p *timeSeriesQueryParser) parse(tsdbQuery *tsdb.TsdbQuery) ([]*Query, error) { queries := make([]*Query, 0) for _, q := range tsdbQuery.Queries { model := q.Model @@ -111,10 +258,7 @@ func (p *timeSeriesQueryParser) parse(tsdbQuery *tsdb.TsdbQuery) (*timeSeriesQue return nil, err } alias := model.Get("alias").MustString("") - parsedInterval, err := tsdb.GetIntervalFrom(p.ds, model, time.Millisecond) - if err != nil { - return nil, err - } + interval := model.Get("interval").MustString() queries = append(queries, &Query{ TimeField: timeField, @@ -122,54 +266,52 @@ func (p *timeSeriesQueryParser) parse(tsdbQuery *tsdb.TsdbQuery) (*timeSeriesQue BucketAggs: bucketAggs, Metrics: metrics, Alias: alias, - Interval: parsedInterval, + Interval: interval, + RefID: q.RefId, }) } - return &timeSeriesQuery{queries: queries}, nil + return queries, nil } func (p *timeSeriesQueryParser) parseBucketAggs(model *simplejson.Json) ([]*BucketAgg, error) { var err error var result []*BucketAgg for _, t := range model.Get("bucketAggs").MustArray() { - aggJson := simplejson.NewFromAny(t) + aggJSON := simplejson.NewFromAny(t) agg := &BucketAgg{} - agg.Type, err = aggJson.Get("type").String() + agg.Type, err = aggJSON.Get("type").String() if err != nil { return nil, err } - agg.ID, err = aggJson.Get("id").String() + agg.ID, err = aggJSON.Get("id").String() if err != nil { return nil, err } - agg.Field = aggJson.Get("field").MustString() - agg.Settings = simplejson.NewFromAny(aggJson.Get("settings").MustMap()) + agg.Field = aggJSON.Get("field").MustString() + agg.Settings = simplejson.NewFromAny(aggJSON.Get("settings").MustMap()) result = append(result, agg) } return result, nil } -func (p *timeSeriesQueryParser) parseMetrics(model *simplejson.Json) ([]*Metric, error) { +func (p *timeSeriesQueryParser) parseMetrics(model *simplejson.Json) ([]*MetricAgg, error) { var err error - var result []*Metric + var result []*MetricAgg for _, t := range model.Get("metrics").MustArray() { metricJSON := simplejson.NewFromAny(t) - metric := &Metric{} + metric := &MetricAgg{} metric.Field = metricJSON.Get("field").MustString() metric.Hide = metricJSON.Get("hide").MustBool(false) - metric.ID, err = metricJSON.Get("id").String() - if err != nil { - return nil, err - } - + metric.ID = metricJSON.Get("id").MustString() metric.PipelineAggregate = metricJSON.Get("pipelineAgg").MustString() metric.Settings = simplejson.NewFromAny(metricJSON.Get("settings").MustMap()) + metric.Meta = simplejson.NewFromAny(metricJSON.Get("meta").MustMap()) metric.Type, err = metricJSON.Get("type").String() if err != nil { diff --git a/pkg/tsdb/elasticsearch/time_series_query_test.go b/pkg/tsdb/elasticsearch/time_series_query_test.go index 4950bc811de..e2af4de749a 100644 --- a/pkg/tsdb/elasticsearch/time_series_query_test.go +++ b/pkg/tsdb/elasticsearch/time_series_query_test.go @@ -1,21 +1,514 @@ package elasticsearch import ( + "fmt" "testing" + "time" + + "github.com/grafana/grafana/pkg/tsdb/elasticsearch/client" "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/tsdb" . "github.com/smartystreets/goconvey/convey" ) +func TestExecuteTimeSeriesQuery(t *testing.T) { + from := time.Date(2018, 5, 15, 17, 50, 0, 0, time.UTC) + to := time.Date(2018, 5, 15, 17, 55, 0, 0, time.UTC) + fromStr := fmt.Sprintf("%d", from.UnixNano()/int64(time.Millisecond)) + toStr := fmt.Sprintf("%d", to.UnixNano()/int64(time.Millisecond)) + + Convey("Test execute time series query", t, func() { + Convey("With defaults on es 2", func() { + c := newFakeClient(2) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [{ "type": "date_histogram", "field": "@timestamp", "id": "2" }], + "metrics": [{"type": "count", "id": "0" }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + rangeFilter := sr.Query.Bool.Filters[0].(*es.RangeFilter) + So(rangeFilter.Key, ShouldEqual, c.timeField) + So(rangeFilter.Lte, ShouldEqual, toStr) + So(rangeFilter.Gte, ShouldEqual, fromStr) + So(rangeFilter.Format, ShouldEqual, es.DateFormatEpochMS) + So(sr.Aggs[0].Key, ShouldEqual, "2") + dateHistogramAgg := sr.Aggs[0].Aggregation.Aggregation.(*es.DateHistogramAgg) + So(dateHistogramAgg.Field, ShouldEqual, "@timestamp") + So(dateHistogramAgg.ExtendedBounds.Min, ShouldEqual, fromStr) + So(dateHistogramAgg.ExtendedBounds.Max, ShouldEqual, toStr) + }) + + Convey("With defaults on es 5", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [{ "type": "date_histogram", "field": "@timestamp", "id": "2" }], + "metrics": [{"type": "count", "id": "0" }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + So(sr.Query.Bool.Filters[0].(*es.RangeFilter).Key, ShouldEqual, c.timeField) + So(sr.Aggs[0].Key, ShouldEqual, "2") + So(sr.Aggs[0].Aggregation.Aggregation.(*es.DateHistogramAgg).ExtendedBounds.Min, ShouldEqual, fromStr) + So(sr.Aggs[0].Aggregation.Aggregation.(*es.DateHistogramAgg).ExtendedBounds.Max, ShouldEqual, toStr) + }) + + Convey("With multiple bucket aggs", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { "type": "terms", "field": "@host", "id": "2" }, + { "type": "date_histogram", "field": "@timestamp", "id": "3" } + ], + "metrics": [{"type": "count", "id": "1" }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + firstLevel := sr.Aggs[0] + So(firstLevel.Key, ShouldEqual, "2") + So(firstLevel.Aggregation.Aggregation.(*es.TermsAggregation).Field, ShouldEqual, "@host") + secondLevel := firstLevel.Aggregation.Aggs[0] + So(secondLevel.Key, ShouldEqual, "3") + So(secondLevel.Aggregation.Aggregation.(*es.DateHistogramAgg).Field, ShouldEqual, "@timestamp") + }) + + Convey("With select field", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { "type": "date_histogram", "field": "@timestamp", "id": "2" } + ], + "metrics": [{"type": "avg", "field": "@value", "id": "1" }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + firstLevel := sr.Aggs[0] + So(firstLevel.Key, ShouldEqual, "2") + So(firstLevel.Aggregation.Aggregation.(*es.DateHistogramAgg).Field, ShouldEqual, "@timestamp") + secondLevel := firstLevel.Aggregation.Aggs[0] + So(secondLevel.Key, ShouldEqual, "1") + So(secondLevel.Aggregation.Type, ShouldEqual, "avg") + So(secondLevel.Aggregation.Aggregation.(*es.MetricAggregation).Field, ShouldEqual, "@value") + }) + + Convey("With term agg and order by metric agg", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { + "type": "terms", + "field": "@host", + "id": "2", + "settings": { "size": "5", "order": "asc", "orderBy": "5" } + }, + { "type": "date_histogram", "field": "@timestamp", "id": "3" } + ], + "metrics": [ + {"type": "count", "id": "1" }, + {"type": "avg", "field": "@value", "id": "5" } + ] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + avgAggOrderBy := sr.Aggs[0].Aggregation.Aggs[0] + So(avgAggOrderBy.Key, ShouldEqual, "5") + So(avgAggOrderBy.Aggregation.Type, ShouldEqual, "avg") + + avgAgg := sr.Aggs[0].Aggregation.Aggs[1].Aggregation.Aggs[0] + So(avgAgg.Key, ShouldEqual, "5") + So(avgAgg.Aggregation.Type, ShouldEqual, "avg") + }) + + Convey("With metric percentiles", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { "type": "date_histogram", "field": "@timestamp", "id": "3" } + ], + "metrics": [ + { + "id": "1", + "type": "percentiles", + "field": "@load_time", + "settings": { + "percents": [ "1", "2", "3", "4" ] + } + } + ] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + percentilesAgg := sr.Aggs[0].Aggregation.Aggs[0] + So(percentilesAgg.Key, ShouldEqual, "1") + So(percentilesAgg.Aggregation.Type, ShouldEqual, "percentiles") + metricAgg := percentilesAgg.Aggregation.Aggregation.(*es.MetricAggregation) + percents := metricAgg.Settings["percents"].([]interface{}) + So(percents, ShouldHaveLength, 4) + So(percents[0], ShouldEqual, "1") + So(percents[1], ShouldEqual, "2") + So(percents[2], ShouldEqual, "3") + So(percents[3], ShouldEqual, "4") + }) + + Convey("With filters aggs on es 2", func() { + c := newFakeClient(2) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { + "id": "2", + "type": "filters", + "settings": { + "filters": [ { "query": "@metric:cpu" }, { "query": "@metric:logins.count" } ] + } + }, + { "type": "date_histogram", "field": "@timestamp", "id": "4" } + ], + "metrics": [{"type": "count", "id": "1" }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + filtersAgg := sr.Aggs[0] + So(filtersAgg.Key, ShouldEqual, "2") + So(filtersAgg.Aggregation.Type, ShouldEqual, "filters") + fAgg := filtersAgg.Aggregation.Aggregation.(*es.FiltersAggregation) + So(fAgg.Filters["@metric:cpu"].(*es.QueryStringFilter).Query, ShouldEqual, "@metric:cpu") + So(fAgg.Filters["@metric:logins.count"].(*es.QueryStringFilter).Query, ShouldEqual, "@metric:logins.count") + + dateHistogramAgg := sr.Aggs[0].Aggregation.Aggs[0] + So(dateHistogramAgg.Key, ShouldEqual, "4") + So(dateHistogramAgg.Aggregation.Aggregation.(*es.DateHistogramAgg).Field, ShouldEqual, "@timestamp") + }) + + Convey("With filters aggs on es 5", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { + "id": "2", + "type": "filters", + "settings": { + "filters": [ { "query": "@metric:cpu" }, { "query": "@metric:logins.count" } ] + } + }, + { "type": "date_histogram", "field": "@timestamp", "id": "4" } + ], + "metrics": [{"type": "count", "id": "1" }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + filtersAgg := sr.Aggs[0] + So(filtersAgg.Key, ShouldEqual, "2") + So(filtersAgg.Aggregation.Type, ShouldEqual, "filters") + fAgg := filtersAgg.Aggregation.Aggregation.(*es.FiltersAggregation) + So(fAgg.Filters["@metric:cpu"].(*es.QueryStringFilter).Query, ShouldEqual, "@metric:cpu") + So(fAgg.Filters["@metric:logins.count"].(*es.QueryStringFilter).Query, ShouldEqual, "@metric:logins.count") + + dateHistogramAgg := sr.Aggs[0].Aggregation.Aggs[0] + So(dateHistogramAgg.Key, ShouldEqual, "4") + So(dateHistogramAgg.Aggregation.Aggregation.(*es.DateHistogramAgg).Field, ShouldEqual, "@timestamp") + }) + + Convey("With raw document metric", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [], + "metrics": [{ "id": "1", "type": "raw_document", "settings": {} }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + So(sr.Size, ShouldEqual, 500) + }) + + Convey("With raw document metric size set", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [], + "metrics": [{ "id": "1", "type": "raw_document", "settings": { "size": 1337 } }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + So(sr.Size, ShouldEqual, 1337) + }) + + Convey("With date histogram agg", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { + "id": "2", + "type": "date_histogram", + "field": "@timestamp", + "settings": { "interval": "auto", "min_doc_count": 2 } + } + ], + "metrics": [{"type": "count", "id": "1" }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + firstLevel := sr.Aggs[0] + So(firstLevel.Key, ShouldEqual, "2") + So(firstLevel.Aggregation.Type, ShouldEqual, "date_histogram") + hAgg := firstLevel.Aggregation.Aggregation.(*es.DateHistogramAgg) + So(hAgg.Field, ShouldEqual, "@timestamp") + So(hAgg.Interval, ShouldEqual, "15s") + So(hAgg.MinDocCount, ShouldEqual, 2) + }) + + Convey("With histogram agg", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { + "id": "3", + "type": "histogram", + "field": "bytes", + "settings": { "interval": 10, "min_doc_count": 2, "missing": 5 } + } + ], + "metrics": [{"type": "count", "id": "1" }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + firstLevel := sr.Aggs[0] + So(firstLevel.Key, ShouldEqual, "3") + So(firstLevel.Aggregation.Type, ShouldEqual, "histogram") + hAgg := firstLevel.Aggregation.Aggregation.(*es.HistogramAgg) + So(hAgg.Field, ShouldEqual, "bytes") + So(hAgg.Interval, ShouldEqual, 10) + So(hAgg.MinDocCount, ShouldEqual, 2) + So(*hAgg.Missing, ShouldEqual, 5) + }) + + Convey("With geo hash grid agg", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { + "id": "3", + "type": "geohash_grid", + "field": "@location", + "settings": { "precision": 3 } + } + ], + "metrics": [{"type": "count", "id": "1" }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + firstLevel := sr.Aggs[0] + So(firstLevel.Key, ShouldEqual, "3") + So(firstLevel.Aggregation.Type, ShouldEqual, "geohash_grid") + ghGridAgg := firstLevel.Aggregation.Aggregation.(*es.GeoHashGridAggregation) + So(ghGridAgg.Field, ShouldEqual, "@location") + So(ghGridAgg.Precision, ShouldEqual, 3) + }) + + Convey("With moving average", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { "type": "date_histogram", "field": "@timestamp", "id": "4" } + ], + "metrics": [ + { "id": "3", "type": "sum", "field": "@value" }, + { + "id": "2", + "type": "moving_avg", + "field": "3", + "pipelineAgg": "3" + } + ] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + firstLevel := sr.Aggs[0] + So(firstLevel.Key, ShouldEqual, "4") + So(firstLevel.Aggregation.Type, ShouldEqual, "date_histogram") + So(firstLevel.Aggregation.Aggs, ShouldHaveLength, 2) + + sumAgg := firstLevel.Aggregation.Aggs[0] + So(sumAgg.Key, ShouldEqual, "3") + So(sumAgg.Aggregation.Type, ShouldEqual, "sum") + mAgg := sumAgg.Aggregation.Aggregation.(*es.MetricAggregation) + So(mAgg.Field, ShouldEqual, "@value") + + movingAvgAgg := firstLevel.Aggregation.Aggs[1] + So(movingAvgAgg.Key, ShouldEqual, "2") + So(movingAvgAgg.Aggregation.Type, ShouldEqual, "moving_avg") + pl := movingAvgAgg.Aggregation.Aggregation.(*es.PipelineAggregation) + So(pl.BucketPath, ShouldEqual, "3") + }) + + Convey("With broken moving average", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { "type": "date_histogram", "field": "@timestamp", "id": "5" } + ], + "metrics": [ + { "id": "3", "type": "sum", "field": "@value" }, + { + "id": "2", + "type": "moving_avg", + "pipelineAgg": "3" + }, + { + "id": "4", + "type": "moving_avg", + "pipelineAgg": "Metric to apply moving average" + } + ] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + firstLevel := sr.Aggs[0] + So(firstLevel.Key, ShouldEqual, "5") + So(firstLevel.Aggregation.Type, ShouldEqual, "date_histogram") + + So(firstLevel.Aggregation.Aggs, ShouldHaveLength, 2) + + movingAvgAgg := firstLevel.Aggregation.Aggs[1] + So(movingAvgAgg.Key, ShouldEqual, "2") + plAgg := movingAvgAgg.Aggregation.Aggregation.(*es.PipelineAggregation) + So(plAgg.BucketPath, ShouldEqual, "3") + }) + + Convey("With derivative", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { "type": "date_histogram", "field": "@timestamp", "id": "4" } + ], + "metrics": [ + { "id": "3", "type": "sum", "field": "@value" }, + { + "id": "2", + "type": "derivative", + "pipelineAgg": "3" + } + ] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + firstLevel := sr.Aggs[0] + So(firstLevel.Key, ShouldEqual, "4") + So(firstLevel.Aggregation.Type, ShouldEqual, "date_histogram") + + derivativeAgg := firstLevel.Aggregation.Aggs[1] + So(derivativeAgg.Key, ShouldEqual, "2") + plAgg := derivativeAgg.Aggregation.Aggregation.(*es.PipelineAggregation) + So(plAgg.BucketPath, ShouldEqual, "3") + }) + + }) +} + +type fakeClient struct { + version int + timeField string + multiSearchResponse *es.MultiSearchResponse + multiSearchError error + builder *es.MultiSearchRequestBuilder + multisearchRequests []*es.MultiSearchRequest +} + +func newFakeClient(version int) *fakeClient { + return &fakeClient{ + version: version, + timeField: "@timestamp", + multisearchRequests: make([]*es.MultiSearchRequest, 0), + multiSearchResponse: &es.MultiSearchResponse{}, + } +} + +func (c *fakeClient) GetVersion() int { + return c.version +} + +func (c *fakeClient) GetTimeField() string { + return c.timeField +} + +func (c *fakeClient) GetMinInterval(queryInterval string) (time.Duration, error) { + return 15 * time.Second, nil +} + +func (c *fakeClient) ExecuteMultisearch(r *es.MultiSearchRequest) (*es.MultiSearchResponse, error) { + c.multisearchRequests = append(c.multisearchRequests, r) + return c.multiSearchResponse, c.multiSearchError +} + +func (c *fakeClient) MultiSearch() *es.MultiSearchRequestBuilder { + c.builder = es.NewMultiSearchRequestBuilder(c.version) + return c.builder +} + +func newTsdbQuery(body string) (*tsdb.TsdbQuery, error) { + json, err := simplejson.NewJson([]byte(body)) + if err != nil { + return nil, err + } + return &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: json, + }, + }, + }, nil +} + +func executeTsdbQuery(c es.Client, body string, from, to time.Time, minInterval time.Duration) (*tsdb.Response, error) { + json, err := simplejson.NewJson([]byte(body)) + if err != nil { + return nil, err + } + fromStr := fmt.Sprintf("%d", from.UnixNano()/int64(time.Millisecond)) + toStr := fmt.Sprintf("%d", to.UnixNano()/int64(time.Millisecond)) + tsdbQuery := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: json, + }, + }, + TimeRange: tsdb.NewTimeRange(fromStr, toStr), + } + query := newTimeSeriesQuery(c, tsdbQuery, tsdb.NewIntervalCalculator(&tsdb.IntervalOptions{MinInterval: minInterval})) + return query.execute() +} + func TestTimeSeriesQueryParser(t *testing.T) { Convey("Test time series query parser", t, func() { - ds := &models.DataSource{} - p := newTimeSeriesQueryParser(ds) + p := newTimeSeriesQueryParser() Convey("Should be able to parse query", func() { - json, err := simplejson.NewJson([]byte(`{ + body := `{ "timeField": "@timestamp", "query": "@metric:cpu", "alias": "{{@hostname}} {{metric}}", @@ -63,21 +556,14 @@ func TestTimeSeriesQueryParser(t *testing.T) { "type": "date_histogram" } ] - }`)) + }` + tsdbQuery, err := newTsdbQuery(body) So(err, ShouldBeNil) - tsdbQuery := &tsdb.TsdbQuery{ - Queries: []*tsdb.Query{ - { - DataSource: ds, - Model: json, - }, - }, - } - tsQuery, err := p.parse(tsdbQuery) + queries, err := p.parse(tsdbQuery) So(err, ShouldBeNil) - So(tsQuery.queries, ShouldHaveLength, 1) + So(queries, ShouldHaveLength, 1) - q := tsQuery.queries[0] + q := queries[0] So(q.TimeField, ShouldEqual, "@timestamp") So(q.RawQuery, ShouldEqual, "@metric:cpu") diff --git a/vendor/github.com/leibowitz/moment/diff.go b/vendor/github.com/leibowitz/moment/diff.go deleted file mode 100644 index 0d6b3935adf..00000000000 --- a/vendor/github.com/leibowitz/moment/diff.go +++ /dev/null @@ -1,75 +0,0 @@ -package moment - -import ( - "fmt" - "math" - "time" -) - -// @todo In months/years requires the old and new to calculate correctly, right? -// @todo decide how to handle rounding (i.e. always floor?) -type Diff struct { - duration time.Duration -} - -func (d *Diff) InSeconds() int { - return int(d.duration.Seconds()) -} - -func (d *Diff) InMinutes() int { - return int(d.duration.Minutes()) -} - -func (d *Diff) InHours() int { - return int(d.duration.Hours()) -} - -func (d *Diff) InDays() int { - return int(math.Floor(float64(d.InSeconds()) / 86400)) -} - -// This depends on where the weeks fall? -func (d *Diff) InWeeks() int { - return int(math.Floor(float64(d.InDays() / 7))) -} - -func (d *Diff) InMonths() int { - return 0 -} - -func (d *Diff) InYears() int { - return 0 -} - -// http://momentjs.com/docs/#/durations/humanize/ -func (d *Diff) Humanize() string { - diffInSeconds := d.InSeconds() - - if diffInSeconds <= 45 { - return fmt.Sprintf("%d seconds ago", diffInSeconds) - } else if diffInSeconds <= 90 { - return "a minute ago" - } - - diffInMinutes := d.InMinutes() - - if diffInMinutes <= 45 { - return fmt.Sprintf("%d minutes ago", diffInMinutes) - } else if diffInMinutes <= 90 { - return "an hour ago" - } - - diffInHours := d.InHours() - - if diffInHours <= 22 { - return fmt.Sprintf("%d hours ago", diffInHours) - } else if diffInHours <= 36 { - return "a day ago" - } - - return "diff is in days" -} - -// In Months - -// In years diff --git a/vendor/github.com/leibowitz/moment/moment.go b/vendor/github.com/leibowitz/moment/moment.go deleted file mode 100644 index 13c8ef7dbef..00000000000 --- a/vendor/github.com/leibowitz/moment/moment.go +++ /dev/null @@ -1,1185 +0,0 @@ -package moment - -import ( - "fmt" - "regexp" - "strconv" - "strings" - "time" -) - -// links -// http://en.wikipedia.org/wiki/ISO_week_date -// http://golang.org/src/pkg/time/format.go -// http://www.php.net/manual/en/class.datetime.php#datetime.constants.rfc822 -// http://php.net/manual/en/function.date.php -// http://www.php.net/manual/en/datetime.formats.relative.php - -// @todo are these constants needed if they are in the time package? -// There are a lot of extras here, and RFC822 doesn't match up. Why? -// Also, is timezone usage wrong? Double-check -const ( - ATOM = "2006-01-02T15:04:05Z07:00" - COOKIE = "Monday, 02-Jan-06 15:04:05 MST" - ISO8601 = "2006-01-02T15:04:05Z0700" - RFC822 = "Mon, 02 Jan 06 15:04:05 Z0700" - RFC850 = "Monday, 02-Jan-06 15:04:05 MST" - RFC1036 = "Mon, 02 Jan 06 15:04:05 Z0700" - RFC1123 = "Mon, 02 Jan 2006 15:04:05 Z0700" - RFC2822 = "Mon, 02 Jan 2006 15:04:05 Z0700" - RFC3339 = "2006-01-02T15:04:05Z07:00" - RSS = "Mon, 02 Jan 2006 15:04:05 Z0700" - W3C = "2006-01-02T15:04:05Z07:00" -) - -var ( - regex_days = "monday|mon|tuesday|tues|wednesday|wed|thursday|thurs|friday|fri|saturday|sat|sunday|sun" - regex_period = "second|minute|hour|day|week|month|year" - regex_numbers = "one|two|three|four|five|six|seven|eight|nine|ten" -) - -// regexp -var ( - compiled = regexp.MustCompile(`\s{2,}`) - relativeday = regexp.MustCompile(`(yesterday|today|tomorrow)`) - //relative1 = regexp.MustCompile(`(first|last) day of (this|next|last|previous) (week|month|year)`) - //relative2 = regexp.MustCompile(`(first|last) day of (` + "jan|january|feb|february|mar|march|apr|april|may|jun|june|jul|july|aug|august|sep|september|oct|october|nov|november|dec|december" + `)(?:\s(\d{4,4}))?`) - relative3 = regexp.MustCompile(`((?Pthis|next|last|previous) )?(` + regex_days + `)`) - //relativeval = regexp.MustCompile(`([0-9]+) (day|week|month|year)s? ago`) - ago = regexp.MustCompile(`([0-9]+) (` + regex_period + `)s? ago`) - ordinal = regexp.MustCompile("([0-9]+)(st|nd|rd|th)") - written = regexp.MustCompile(regex_numbers) - relativediff = regexp.MustCompile(`([\+\-])?([0-9]+),? ?(` + regex_period + `)s?`) - relativetime = regexp.MustCompile(`(?P\d\d?):(?P\d\d?)(:(?P\d\d?))?\s?(?Pam|pm)?\s?(?P[a-z]{3,3})?|(?Pnoon|midnight)`) - yearmonthday = regexp.MustCompile(`(?P\d{4})-(?P\d{1,2})-(?P\d{1,2})`) - relativeperiod = regexp.MustCompile(`(?Pthis|next|last) (week|month|year)`) - numberRegex = regexp.MustCompile("([0-9]+)(?:)") -) - -// http://golang.org/src/pkg/time/format.go?s=12686:12728#L404 - -// Timezone implementation -// https://groups.google.com/forum/#!topic/golang-nuts/XEVN4QwTvHw -// http://en.wikipedia.org/wiki/Zone.tab - -// Support ISO8601 Duration Parsing? -// http://en.wikipedia.org/wiki/ISO_8601 - -// Differences -// Months are NOT zero-index, MOmentJS they are -// Weeks are 0 indexed -// -- Sunday being the last day of the week ISO-8601 - is that diff from Moment? -// From/FromNow Return a Diff object rather than strings - -// Support for locale and languages with English as default - -// Support for strftime -// https://github.com/benjaminoakes/moment-strftime -// Format: https://php.net/strftime - -type Moment struct { - time time.Time - - Parser -} - -type Parser interface { - Convert(string) string -} - -func New() *Moment { - m := &Moment{time.Now(), new(MomentParser)} - - return m -} - -func NewMoment(t time.Time) *Moment { - m := &Moment{t, new(MomentParser)} - - return m -} - -func (m *Moment) GetTime() time.Time { - return m.time -} - -func (m *Moment) Now() *Moment { - m.time = time.Now().In(m.GetTime().Location()) - - return m -} - -func (m *Moment) Moment(layout string, datetime string) *Moment { - return m.MomentGo(m.Convert(layout), datetime) -} - -func (m *Moment) MomentGo(layout string, datetime string) *Moment { - time, _ := time.Parse(layout, datetime) - - m.time = time - - return m -} - -// This method is nowhere near done - requires lots of work. -func (m *Moment) Strtotime(str string) *Moment { - str = strings.ToLower(strings.TrimSpace(str)) - str = compiled.ReplaceAllString(str, " ") - - // Replace written numbers (i.e. nine, ten) with actual numbers (9, 10) - str = written.ReplaceAllStringFunc(str, func(n string) string { - switch n { - case "one": - return "1" - case "two": - return "2" - case "three": - return "3" - case "four": - return "4" - case "five": - return "5" - case "six": - return "6" - case "seven": - return "7" - case "eight": - return "8" - case "nine": - return "9" - case "ten": - return "10" - } - - return "" - }) - - // Remove ordinal suffixes st, nd, rd, th - str = ordinal.ReplaceAllString(str, "$1") - - // Replace n second|minute|hour... ago to -n second|minute|hour... to consolidate parsing - str = ago.ReplaceAllString(str, "-$1 $2") - - // Look for relative +1day, +3 days 5 hours 15 minutes - if match := relativediff.FindAllStringSubmatch(str, -1); match != nil { - for i := range match { - switch match[i][1] { - case "-": - number, _ := strconv.Atoi(match[i][2]) - m.Subtract(match[i][3], number) - default: - number, _ := strconv.Atoi(match[i][2]) - m.Add(match[i][3], number) - } - - str = strings.Replace(str, match[i][0], "", 1) - } - } - - // Remove any words that aren't needed for consistency - str = strings.Replace(str, " at ", " ", -1) - str = strings.Replace(str, " on ", " ", -1) - - // Support for interchangeable previous/last - str = strings.Replace(str, "previous", "last", -1) - - var dateDefaults = map[string]int{ - "year": 0, - "month": 0, - "day": 0, - } - - dateMatches := dateDefaults - if match := yearmonthday.FindStringSubmatch(str); match != nil { - for i, name := range yearmonthday.SubexpNames() { - if i == 0 { - str = strings.Replace(str, match[i], "", 1) - continue - } - - if match[i] == "" { - continue - } - - if name == "year" || name == "month" || name == "day" { - dateMatches[name], _ = strconv.Atoi(match[i]) - } - - } - - defer m.strtotimeSetDate(dateMatches) - if str == "" { - // Nothing left to parse - return m - } - - str = strings.TrimSpace(str) - } - - // Try to parse out time from the string - var timeDefaults = map[string]int{ - "hour": 0, - "minutes": 0, - "seconds": 0, - } - - timeMatches := timeDefaults - var zone string - if match := relativetime.FindStringSubmatch(str); match != nil { - for i, name := range relativetime.SubexpNames() { - if i == 0 { - str = strings.Replace(str, match[i], "", 1) - continue - } - - if match[i] == "" { - continue - } - - // Midnight is all zero's so nothing to do - if name == "relativetime" && match[i] == "noon" { - timeDefaults["hour"] = 12 - } - - if name == "zone" { - zone = match[i] - } - - if name == "meridiem" && match[i] == "pm" && timeMatches["hour"] < 12 { - timeMatches["hour"] += 12 - } - - if name == "hour" || name == "minutes" || name == "seconds" { - timeMatches[name], _ = strconv.Atoi(match[i]) - } - } - - // Processing time is always last - defer m.strtotimeSetTime(timeMatches, zone) - - if str == "" { - // Nothing left to parse - return m - } - - str = strings.TrimSpace(str) - } - - // m.StartOf("month", "January").GoTo(time.Sunday) - - if match := relativeperiod.FindStringSubmatch(str); match != nil { - period := match[1] - unit := match[2] - - str = strings.Replace(str, match[0], "", 1) - - switch period { - case "next": - if unit == "year" { - m.AddYears(1) - } - if unit == "month" { - m.AddMonths(1) - } - if unit == "week" { - m.AddWeeks(1) - } - case "last": - if unit == "year" { - m.SubYears(1) - } - if unit == "month" { - m.SubMonths(1) - } - if unit == "week" { - m.SubWeeks(1) - } - } - - str = strings.TrimSpace(str) - - // first := regexp.MustCompile("(?Pfirst|last)?") - } - - /* - - relativeday: first day of - relativeperiod: this, last, next - relativeperiodunit week, month, year - day: monday, tues, wednesday - month: january, feb - - - YYYY-MM-DD (HH:MM:SS MST)? - MM-DD-YYYY (HH:MM:SS MST) - 10 September 2015 (HH:MM:SS MST)? - September, 10 2015 (HH:MM:SS MST)? - September 10 2015 (HH:MM:SS M - - this year 2014 - next year 2015 - last year 2013 - - this month April - next month May - last month Mar - - first day of April - last day of April - - - DONE 3PM - DONE 3:00 PM - DONE 3:00:05 MST - 3PM on January 5th - January 5th at 3:00PM - first saturday _of_ next month - first saturday _of_ next month _at_ 3:00PM - saturday of next week - saturday of last week - saturday next week - monday next week - saturday of this week - saturday at 3:00pm - saturday at 4:00PM - saturday at midn - first of january - last of january - january of next year - first day of january - last day of january - first day of February - - DONE midnight - DONE noon - DONE 3 days ago - DONE ten days - DONE 9 weeks ago // Convert to -9 weeks - DONE -9 weeks - - */ - - if match := relativeday.FindStringSubmatch(str); match != nil && len(match) > 1 { - day := match[1] - - str = strings.Replace(str, match[0], "", 1) - - switch day { - case "today": - m.Today() - case "yesterday": - m.Yesterday() - case "tomorrow": - m.Tomorrow() - } - } - - if match := relative3.FindStringSubmatch(str); match != nil { - var when string - for i, name := range relative3.SubexpNames() { - if name == "relperiod" { - when = match[i] - } - } - weekDay := match[len(match)-1] - - str = strings.Replace(str, match[0], "", 1) - - wDay, err := ParseWeekDay(weekDay) - if err == nil { - switch when { - case "last", "previous": - m.GoBackTo(wDay, true) - - case "next": - m.GoTo(wDay, true) - - case "", "this": - m.GoTo(wDay, false) - default: - m.GoTo(wDay, false) - } - } - } - - /* - - - yesterday 11:00 - today 11:00 - tomorrow 11:00 - midnight - noon - DONE +n (second|day|week|month|year)s? - DONE -n (second|day|week|month|year)s? - next (monday|tuesday|wednesday|thursday|friday|saturday|sunday) 11:00 - last (monday|tuesday|wednesday|thursday|friday|saturday|sunday) 11:00 - next (month|year) - last (month|year) - first day of (january|february|march...|december) 2014 - last day of (january|february|march...|december) 2014 - first day of (this|next|last) (week|month|year) - last day of (this|next|last) (week|month|year) - first (monday|tuesday|wednesday) of July 2014 - last (monday|tuesday|wednesday) of July 2014 - n (day|week|month|year)s? ago - Monday|Tuesday|Wednesday|Thursday|Friday - Monday (last|this|next) week - - DONE +1 week 2 days 3 hours 4 minutes 5 seconds - */ - - return m -} - -// @todo deal with timezone -func (m *Moment) strtotimeSetTime(time map[string]int, zone string) { - m.SetHour(time["hour"]).SetMinute(time["minutes"]).SetSecond(time["seconds"]) -} - -func (m *Moment) strtotimeSetDate(date map[string]int) { - m.SetYear(date["year"]).SetMonth(time.Month(date["month"])).SetDay(date["day"]) -} - -func (m Moment) Clone() *Moment { - copy := New() - copy.time = m.GetTime() - - return copy -} - -/** - * Getters - * - */ -// https://groups.google.com/forum/#!topic/golang-nuts/pret7hjDc70 -func (m *Moment) Millisecond() { - -} - -func (m *Moment) Second() int { - return m.GetTime().Second() -} - -func (m *Moment) Minute() int { - return m.GetTime().Minute() -} - -func (m *Moment) Hour() int { - return m.GetTime().Hour() -} - -// Day of month -func (m *Moment) Date() int { - return m.DayOfMonth() -} - -// Carbon convenience method -func (m *Moment) DayOfMonth() int { - return m.GetTime().Day() -} - -// Day of week (int or string) -func (m *Moment) Day() time.Weekday { - return m.DayOfWeek() -} - -// Carbon convenience method -func (m *Moment) DayOfWeek() time.Weekday { - return m.GetTime().Weekday() -} - -func (m *Moment) DayOfWeekISO() int { - day := m.GetTime().Weekday() - - if day == time.Sunday { - return 7 - } - - return int(day) -} - -func (m *Moment) DayOfYear() int { - return m.GetTime().YearDay() -} - -// Day of Year with zero padding -func (m *Moment) dayOfYearZero() string { - day := m.GetTime().YearDay() - - if day < 10 { - return fmt.Sprintf("00%d", day) - } - - if day < 100 { - return fmt.Sprintf("0%d", day) - } - - return fmt.Sprintf("%d", day) -} - -// todo panic? -func (m *Moment) Weekday(index int) string { - if index > 6 { - panic("Weekday index must be between 0 and 6") - } - - return time.Weekday(index).String() -} - -func (m *Moment) Week() int { - return 0 -} - -// Is this the week number where as ISOWeekYear is the number of weeks in the year? -// @see http://stackoverflow.com/questions/18478741/get-weeks-in-year -func (m *Moment) ISOWeek() int { - _, week := m.GetTime().ISOWeek() - - return week -} - -// @todo Consider language support -func (m *Moment) Month() time.Month { - return m.GetTime().Month() -} - -func (m *Moment) Quarter() (quarter int) { - quarter = 4 - - switch m.Month() { - case time.January, time.February, time.March: - quarter = 1 - case time.April, time.May, time.June: - quarter = 2 - case time.July, time.August, time.September: - quarter = 3 - } - - return -} - -func (m *Moment) Year() int { - return m.GetTime().Year() -} - -// @see comments for ISOWeek -func (m *Moment) WeekYear() { - -} - -func (m *Moment) ISOWeekYear() { - -} - -/** - * Manipulate - * - */ -func (m *Moment) Add(key string, value int) *Moment { - switch key { - case "years", "year", "y": - m.AddYears(value) - case "months", "month", "M": - m.AddMonths(value) - case "weeks", "week", "w": - m.AddWeeks(value) - case "days", "day", "d": - m.AddDays(value) - case "hours", "hour", "h": - m.AddHours(value) - case "minutes", "minute", "m": - m.AddMinutes(value) - case "seconds", "second", "s": - m.AddSeconds(value) - case "milliseconds", "millisecond", "ms": - - } - - return m -} - -// Carbon -func (m *Moment) AddSeconds(seconds int) *Moment { - return m.addTime(time.Second * time.Duration(seconds)) -} - -// Carbon -func (m *Moment) AddMinutes(minutes int) *Moment { - return m.addTime(time.Minute * time.Duration(minutes)) -} - -// Carbon -func (m *Moment) AddHours(hours int) *Moment { - return m.addTime(time.Hour * time.Duration(hours)) -} - -// Carbon -func (m *Moment) AddDay() *Moment { - return m.AddDays(1) -} - -// Carbon -func (m *Moment) AddDays(days int) *Moment { - m.time = m.GetTime().AddDate(0, 0, days) - - return m -} - -// Carbon -func (m *Moment) AddWeeks(weeks int) *Moment { - return m.AddDays(weeks * 7) -} - -// Carbon -func (m *Moment) AddMonths(months int) *Moment { - m.time = m.GetTime().AddDate(0, months, 0) - - return m -} - -// Carbon -func (m *Moment) AddYears(years int) *Moment { - m.time = m.GetTime().AddDate(years, 0, 0) - - return m -} - -func (m *Moment) addTime(d time.Duration) *Moment { - m.time = m.GetTime().Add(d) - - return m -} - -func (m *Moment) Subtract(key string, value int) *Moment { - switch key { - case "years", "year", "y": - m.SubYears(value) - case "months", "month", "M": - m.SubMonths(value) - case "weeks", "week", "w": - m.SubWeeks(value) - case "days", "day", "d": - m.SubDays(value) - case "hours", "hour", "h": - m.SubHours(value) - case "minutes", "minute", "m": - m.SubMinutes(value) - case "seconds", "second", "s": - m.SubSeconds(value) - case "milliseconds", "millisecond", "ms": - - } - - return m -} - -// Carbon -func (m *Moment) SubSeconds(seconds int) *Moment { - return m.addTime(time.Second * time.Duration(seconds*-1)) -} - -// Carbon -func (m *Moment) SubMinutes(minutes int) *Moment { - return m.addTime(time.Minute * time.Duration(minutes*-1)) -} - -// Carbon -func (m *Moment) SubHours(hours int) *Moment { - return m.addTime(time.Hour * time.Duration(hours*-1)) -} - -// Carbon -func (m *Moment) SubDay() *Moment { - return m.SubDays(1) -} - -// Carbon -func (m *Moment) SubDays(days int) *Moment { - return m.AddDays(days * -1) -} - -func (m *Moment) SubWeeks(weeks int) *Moment { - return m.SubDays(weeks * 7) -} - -// Carbon -func (m *Moment) SubMonths(months int) *Moment { - return m.AddMonths(months * -1) -} - -// Carbon -func (m *Moment) SubYears(years int) *Moment { - return m.AddYears(years * -1) -} - -// Carbon -func (m *Moment) Today() *Moment { - return m.Now() -} - -// Carbon -func (m *Moment) Tomorrow() *Moment { - return m.Today().AddDay() -} - -// Carbon -func (m *Moment) Yesterday() *Moment { - return m.Today().SubDay() -} - -func (m *Moment) StartOf(key string) *Moment { - switch key { - case "year", "y": - m.StartOfYear() - case "month", "M": - m.StartOfMonth() - case "week", "w": - m.StartOfWeek() - case "day", "d": - m.StartOfDay() - case "hour", "h": - if m.Minute() > 0 { - m.SubMinutes(m.Minute()) - } - - if m.Second() > 0 { - m.SubSeconds(m.Second()) - } - case "minute", "m": - if m.Second() > 0 { - m.SubSeconds(m.Second()) - } - case "second", "s": - - } - - return m -} - -// Carbon -func (m *Moment) StartOfDay() *Moment { - if m.Hour() > 0 { - _, timeOffset := m.GetTime().Zone() - m.SubHours(m.Hour()) - - _, newTimeOffset := m.GetTime().Zone() - diffOffset := timeOffset - newTimeOffset - if diffOffset != 0 { - // we need to adjust for time zone difference - m.AddSeconds(diffOffset) - } - } - - return m.StartOf("hour") -} - -// @todo ISO8601 Starts on Monday -func (m *Moment) StartOfWeek() *Moment { - return m.GoBackTo(time.Monday, false).StartOfDay() -} - -// Carbon -func (m *Moment) StartOfMonth() *Moment { - return m.SetDay(1).StartOfDay() -} - -// Carbon -func (m *Moment) StartOfYear() *Moment { - return m.SetMonth(time.January).SetDay(1).StartOfDay() -} - -// Carbon -func (m *Moment) EndOf(key string) *Moment { - switch key { - case "year", "y": - m.EndOfYear() - case "month", "M": - m.EndOfMonth() - case "week", "w": - m.EndOfWeek() - case "day", "d": - m.EndOfDay() - case "hour", "h": - if m.Minute() < 59 { - m.AddMinutes(59 - m.Minute()) - } - case "minute", "m": - if m.Second() < 59 { - m.AddSeconds(59 - m.Second()) - } - case "second", "s": - - } - - return m -} - -// Carbon -func (m *Moment) EndOfDay() *Moment { - if m.Hour() < 23 { - _, timeOffset := m.GetTime().Zone() - m.AddHours(23 - m.Hour()) - - _, newTimeOffset := m.GetTime().Zone() - diffOffset := newTimeOffset - timeOffset - if diffOffset != 0 { - // we need to adjust for time zone difference - m.SubSeconds(diffOffset) - } - } - - return m.EndOf("hour") -} - -// @todo ISO8601 Ends on Sunday -func (m *Moment) EndOfWeek() *Moment { - return m.GoTo(time.Sunday, false).EndOfDay() -} - -// Carbon -func (m *Moment) EndOfMonth() *Moment { - return m.SetDay(m.DaysInMonth()).EndOfDay() -} - -// Carbon -func (m *Moment) EndOfYear() *Moment { - return m.GoToMonth(time.December, false).EndOfMonth() -} - -// Custom -func (m *Moment) GoTo(day time.Weekday, next bool) *Moment { - if m.Day() == day { - if !next { - return m - } else { - m.AddDay() - } - } - - var diff int - if diff = int(day) - int(m.Day()); diff > 0 { - return m.AddDays(diff) - } - - return m.AddDays(7 + diff) -} - -// Custom -func (m *Moment) GoBackTo(day time.Weekday, previous bool) *Moment { - if m.Day() == day { - if !previous { - return m - } else { - m.SubDay() - } - } - - var diff int - if diff = int(day) - int(m.Day()); diff > 0 { - return m.SubDays(7 - diff) - } - - return m.SubDays(diff * -1) -} - -// Custom -func (m *Moment) GoToMonth(month time.Month, next bool) *Moment { - if m.Month() == month { - if !next { - return m - } else { - m.AddMonths(1) - } - } - - var diff int - if diff = int(month - m.Month()); diff > 0 { - return m.AddMonths(diff) - } - - return m.AddMonths(12 + diff) -} - -// Custom -func (m *Moment) GoBackToMonth(month time.Month, previous bool) *Moment { - if m.Month() == month { - if !previous { - return m - } else { - m.SubMonths(1) - } - } - - var diff int - if diff = int(month) - int(m.Month()); diff > 0 { - return m.SubMonths(12 - diff) - } - - return m.SubMonths(diff * -1) -} - -func (m *Moment) SetSecond(seconds int) *Moment { - if seconds >= 0 && seconds <= 60 { - return m.AddSeconds(seconds - m.Second()) - } - - return m -} - -func (m *Moment) SetMinute(minute int) *Moment { - if minute >= 0 && minute <= 60 { - return m.AddMinutes(minute - m.Minute()) - } - - return m -} - -func (m *Moment) SetHour(hour int) *Moment { - if hour >= 0 && hour <= 23 { - return m.AddHours(hour - m.Hour()) - } - - return m -} - -// Custom -func (m *Moment) SetDay(day int) *Moment { - if m.DayOfMonth() == day { - return m - } - - return m.AddDays(day - m.DayOfMonth()) -} - -// Custom -func (m *Moment) SetMonth(month time.Month) *Moment { - if m.Month() > month { - return m.GoBackToMonth(month, false) - } - - return m.GoToMonth(month, false) -} - -// Custom -func (m *Moment) SetYear(year int) *Moment { - if m.Year() == year { - return m - } - - return m.AddYears(year - m.Year()) -} - -// UTC Mode. @see http://momentjs.com/docs/#/parsing/utc/ -func (m *Moment) UTC() *Moment { - return m -} - -// http://momentjs.com/docs/#/manipulating/timezone-offset/ -func (m *Moment) Zone() int { - _, offset := m.GetTime().Zone() - - return (offset / 60) * -1 -} - -/** - * Display - * - */ -func (m *Moment) Format(layout string) string { - format := m.Convert(layout) - hasCustom := false - - formatted := m.GetTime().Format(format) - - if strings.Contains(formatted, "", fmt.Sprintf("%d", m.Unix()), -1) - formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", m.ISOWeek()), -1) - formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", m.DayOfWeek()), -1) - formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", m.DayOfWeekISO()), -1) - formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", m.DayOfYear()), -1) - formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", m.Quarter()), -1) - formatted = strings.Replace(formatted, "", m.dayOfYearZero(), -1) - formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", m.Hour()), -1) - } - - // This has to happen after time.Format - if hasCustom && strings.Contains(formatted, "") { - formatted = numberRegex.ReplaceAllStringFunc(formatted, func(n string) string { - ordinal, _ := strconv.Atoi(strings.Replace(n, "", "", 1)) - return m.ordinal(ordinal) - }) - } - - return formatted -} - -func (m *Moment) FormatGo(layout string) string { - return m.GetTime().Format(layout) -} - -// From Dmytro Shteflyuk @https://groups.google.com/forum/#!topic/golang-nuts/l8NhI74jl-4 -func (m *Moment) ordinal(x int) string { - suffix := "th" - switch x % 10 { - case 1: - if x%100 != 11 { - suffix = "st" - } - case 2: - if x%100 != 12 { - suffix = "nd" - } - case 3: - if x%100 != 13 { - suffix = "rd" - } - } - - return strconv.Itoa(x) + suffix -} - -func (m *Moment) FromNow() Diff { - now := new(Moment) - now.Now() - - return m.From(now) -} - -// Carbon -func (m *Moment) From(f *Moment) Diff { - return m.GetDiff(f) -} - -/** - * Difference - * - */ -func (m *Moment) Diff(t *Moment, unit string) int { - diff := m.GetDiff(t) - - switch unit { - case "years": - return diff.InYears() - case "months": - return diff.InMonths() - case "weeks": - return diff.InWeeks() - case "days": - return diff.InDays() - case "hours": - return diff.InHours() - case "minutes": - return diff.InMinutes() - case "seconds": - return diff.InSeconds() - } - - return 0 -} - -// Custom -func (m *Moment) GetDiff(t *Moment) Diff { - duration := m.GetTime().Sub(t.GetTime()) - - return Diff{duration} -} - -/** - * Display - * - */ -func (m *Moment) ValueOf() int64 { - return m.Unix() * 1000 -} - -func (m *Moment) Unix() int64 { - return m.GetTime().Unix() -} - -func (m *Moment) DaysInMonth() int { - days := 31 - switch m.Month() { - case time.April, time.June, time.September, time.November: - days = 30 - break - case time.February: - days = 28 - if m.IsLeapYear() { - days = 29 - } - break - } - - return days -} - -// or ToSlice? -func (m *Moment) ToArray() []int { - return []int{ - m.Year(), - int(m.Month()), - m.DayOfMonth(), - m.Hour(), - m.Minute(), - m.Second(), - } -} - -/** - * Query - * - */ -func (m *Moment) IsBefore(t Moment) bool { - return m.GetTime().Before(t.GetTime()) -} - -func (m *Moment) IsSame(t *Moment, layout string) bool { - return m.Format(layout) == t.Format(layout) -} - -func (m *Moment) IsAfter(t Moment) bool { - return m.GetTime().After(t.GetTime()) -} - -// Carbon -func (m *Moment) IsToday() bool { - today := m.Clone().Today() - - return m.Year() == today.Year() && m.Month() == today.Month() && m.Day() == today.Day() -} - -// Carbon -func (m *Moment) IsTomorrow() bool { - tomorrow := m.Clone().Tomorrow() - - return m.Year() == tomorrow.Year() && m.Month() == tomorrow.Month() && m.Day() == tomorrow.Day() -} - -// Carbon -func (m *Moment) IsYesterday() bool { - yesterday := m.Clone().Yesterday() - - return m.Year() == yesterday.Year() && m.Month() == yesterday.Month() && m.Day() == yesterday.Day() -} - -// Carbon -func (m *Moment) IsWeekday() bool { - return !m.IsWeekend() -} - -// Carbon -func (m *Moment) IsWeekend() bool { - return m.DayOfWeek() == time.Sunday || m.DayOfWeek() == time.Saturday -} - -func (m *Moment) IsLeapYear() bool { - year := m.Year() - return year%4 == 0 && (year%100 != 0 || year%400 == 0) -} - -// Custom -func (m *Moment) Range(start Moment, end Moment) bool { - return m.IsAfter(start) && m.IsBefore(end) -} diff --git a/vendor/github.com/leibowitz/moment/moment_parser.go b/vendor/github.com/leibowitz/moment/moment_parser.go deleted file mode 100644 index 3361cfba113..00000000000 --- a/vendor/github.com/leibowitz/moment/moment_parser.go +++ /dev/null @@ -1,100 +0,0 @@ -package moment - -import ( - "regexp" - "strings" -) - -type MomentParser struct{} - -var ( - date_pattern = regexp.MustCompile("(LT|LL?L?L?|l{1,4}|Mo|MM?M?M?|Do|DDDo|DD?D?D?|ddd?d?|do?|w[o|w]?|W[o|W]?|YYYYY|YYYY|YY|gg(ggg?)?|GG(GGG?)?|e|E|a|A|hh?|HH?|mm?|ss?|SS?S?|X|zz?|ZZ?|Q)") -) - -/* - + S (makes any number before it ordinal) - + stdDayOfYear 1,2,365 - + stdDayOfYearZero 001, 002, 365 - + stdDayOfWeek w 0, 1, 2 numeric day of the week (0 = sunday) - + stdDayOfWeekISO N 1 = Monday - + stdWeekOfYear W Iso week number of year - + stdUnix U - + stdQuarter -*/ - -// Thanks to https://github.com/fightbulc/moment.php for replacement keys and regex -var moment_replacements = map[string]string{ - "M": "1", // stdNumMonth 1 2 ... 11 12 - "Mo": "1", // stdNumMonth 1st 2nd ... 11th 12th - "MM": "01", // stdZeroMonth 01 02 ... 11 12 - "MMM": "Jan", // stdMonth Jan Feb ... Nov Dec - "MMMM": "January", // stdLongMonth January February ... November December - "D": "2", // stdDay 1 2 ... 30 30 - "Do": "2", // stdDay 1st 2nd ... 30th 31st @todo support st nd th etch - "DD": "02", // stdZeroDay 01 02 ... 30 31 - "DDD": "", // Day of the year 1 2 ... 364 365 - "DDDo": "", // Day of the year 1st 2nd ... 364th 365th - "DDDD": "", // Day of the year 001 002 ... 364 365 @todo**** - "d": "", // Numeric representation of day of the week 0 1 ... 5 6 - "do": "", // 0th 1st ... 5th 6th - "dd": "Mon", // ***Su Mo ... Fr Sa @todo - "ddd": "Mon", // Sun Mon ... Fri Sat - "dddd": "Monday", // stdLongWeekDay Sunday Monday ... Friday Saturday - "e": "", // Numeric representation of day of the week 0 1 ... 5 6 @todo - "E": "", // ISO-8601 numeric representation of the day of the week (added in PHP 5.1.0) 1 2 ... 6 7 @todo - "w": "", // 1 2 ... 52 53 - "wo": "", // 1st 2nd ... 52nd 53rd - "ww": "", // ***01 02 ... 52 53 @todo - "W": "", // 1 2 ... 52 53 - "Wo": "", // 1st 2nd ... 52nd 53rd - "WW": "", // ***01 02 ... 52 53 @todo - "YY": "06", // stdYear 70 71 ... 29 30 - "YYYY": "2006", // stdLongYear 1970 1971 ... 2029 2030 - // "gg" : "o", // ISO-8601 year number 70 71 ... 29 30 @todo - // "gggg" : "o", // ***1970 1971 ... 2029 2030 @todo - // "GG" : "o", //70 71 ... 29 30 @todo - // "GGGG" : "o", // ***1970 1971 ... 2029 2030 @todo - "Q": "", - "A": "PM", // stdPM AM PM - "a": "pm", // stdpm am pm - "H": "", // stdHour 0 1 ... 22 23 - "HH": "15", // 00 01 ... 22 23 - "h": "3", // stdHour12 1 2 ... 11 12 - "hh": "03", // stdZeroHour12 01 02 ... 11 12 - "m": "4", // stdZeroMinute 0 1 ... 58 59 - "mm": "04", // stdZeroMinute 00 01 ... 58 59 - "s": "5", // stdSecond 0 1 ... 58 59 - "ss": "05", // stdZeroSecond ***00 01 ... 58 59 - // "S" : "", //0 1 ... 8 9 - // "SS" : "", //0 1 ... 98 99 - // "SSS" : "", //0 1 ... 998 999 - "z": "MST", //EST CST ... MST PST - "zz": "MST", //EST CST ... MST PST - "Z": "Z07:00", // stdNumColonTZ -07:00 -06:00 ... +06:00 +07:00 - "ZZ": "-0700", // stdNumTZ -0700 -0600 ... +0600 +0700 - "X": "", // Seconds since unix epoch 1360013296 - "LT": "3:04 PM", // 8:30 PM - "L": "01/02/2006", //09/04/1986 - "l": "1/2/2006", //9/4/1986 - "LL": "January 2 2006", //September 4th 1986 the php s flag isn't supported - "ll": "Jan 2 2006", //Sep 4 1986 - "LLL": "January 2 2006 3:04 PM", //September 4th 1986 8:30 PM @todo the php s flag isn't supported - "lll": "Jan 2 2006 3:04 PM", //Sep 4 1986 8:30 PM - "LLLL": "Monday, January 2 2006 3:04 PM", //Thursday, September 4th 1986 8:30 PM the php s flag isn't supported - "llll": "Mon, Jan 2 2006 3:04 PM", //Thu, Sep 4 1986 8:30 PM -} - -func (p *MomentParser) Convert(layout string) string { - var match [][]string - if match = date_pattern.FindAllStringSubmatch(layout, -1); match == nil { - return layout - } - - for i := range match { - if replace, ok := moment_replacements[match[i][0]]; ok { - layout = strings.Replace(layout, match[i][0], replace, 1) - } - } - - return layout -} diff --git a/vendor/github.com/leibowitz/moment/parse_day.go b/vendor/github.com/leibowitz/moment/parse_day.go deleted file mode 100644 index e8e890a462e..00000000000 --- a/vendor/github.com/leibowitz/moment/parse_day.go +++ /dev/null @@ -1,32 +0,0 @@ -package moment - -import ( - "fmt" - "strings" - "time" -) - -var ( - days = []time.Weekday{ - time.Sunday, - time.Monday, - time.Tuesday, - time.Wednesday, - time.Thursday, - time.Friday, - time.Saturday, - } -) - -func ParseWeekDay(day string) (time.Weekday, error) { - - day = strings.ToLower(day) - - for _, d := range days { - if day == strings.ToLower(d.String()) { - return d, nil - } - } - - return -1, fmt.Errorf("Unable to parse %s as week day", day) -} diff --git a/vendor/github.com/leibowitz/moment/strftime_parser.go b/vendor/github.com/leibowitz/moment/strftime_parser.go deleted file mode 100644 index 3c024376535..00000000000 --- a/vendor/github.com/leibowitz/moment/strftime_parser.go +++ /dev/null @@ -1,68 +0,0 @@ -package moment - -import ( - "regexp" - "strings" -) - -type StrftimeParser struct{} - -var ( - replacements_pattern = regexp.MustCompile("%[mbhBedjwuaAVgyGYpPkHlIMSZzsTrRTDFXx]") -) - -// Not implemented -// U -// C - -var strftime_replacements = map[string]string{ - "%m": "01", // stdZeroMonth 01 02 ... 11 12 - "%b": "Jan", // stdMonth Jan Feb ... Nov Dec - "%h": "Jan", - "%B": "January", // stdLongMonth January February ... November December - "%e": "2", // stdDay 1 2 ... 30 30 - "%d": "02", // stdZeroDay 01 02 ... 30 31 - "%j": "", // Day of the year ***001 002 ... 364 365 @todo**** - "%w": "", // Numeric representation of day of the week 0 1 ... 5 6 - "%u": "", // ISO-8601 numeric representation of the day of the week (added in PHP 5.1.0) 1 2 ... 6 7 @todo - "%a": "Mon", // Sun Mon ... Fri Sat - "%A": "Monday", // stdLongWeekDay Sunday Monday ... Friday Saturday - "%V": "", // ***01 02 ... 52 53 @todo begin with zeros - "%g": "06", // stdYear 70 71 ... 29 30 - "%y": "06", - "%G": "2006", // stdLongYear 1970 1971 ... 2029 2030 - "%Y": "2006", - "%p": "PM", // stdPM AM PM - "%P": "pm", // stdpm am pm - "%k": "15", // stdHour 0 1 ... 22 23 - "%H": "15", // 00 01 ... 22 23 - "%l": "3", // stdHour12 1 2 ... 11 12 - "%I": "03", // stdZeroHour12 01 02 ... 11 12 - "%M": "04", // stdZeroMinute 00 01 ... 58 59 - "%S": "05", // stdZeroSecond ***00 01 ... 58 59 - "%Z": "MST", //EST CST ... MST PST - "%z": "-0700", // stdNumTZ -0700 -0600 ... +0600 +0700 - "%s": "", // Seconds since unix epoch 1360013296 - "%r": "03:04:05 PM", - "%R": "15:04", - "%T": "15:04:05", - "%D": "01/02/06", - "%F": "2006-01-02", - "%X": "15:04:05", - "%x": "01/02/06", -} - -func (p *StrftimeParser) Convert(layout string) string { - var match [][]string - if match = replacements_pattern.FindAllStringSubmatch(layout, -1); match == nil { - return layout - } - - for i := range match { - if replace, ok := strftime_replacements[match[i][0]]; ok { - layout = strings.Replace(layout, match[i][0], replace, 1) - } - } - - return layout -} From dce59ccff2f13bbefafb645cbffdcf8b07f94f20 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 23 May 2018 15:28:36 +0200 Subject: [PATCH 12/22] fix: remove deadcode to make gometalinter happy --- pkg/tsdb/elasticsearch/client/client.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/tsdb/elasticsearch/client/client.go b/pkg/tsdb/elasticsearch/client/client.go index 3dae343bd37..3762a58317a 100644 --- a/pkg/tsdb/elasticsearch/client/client.go +++ b/pkg/tsdb/elasticsearch/client/client.go @@ -22,8 +22,7 @@ import ( const loggerName = "tsdb.elasticsearch.client" var ( - clientLog = log.New(loggerName) - intervalCalculator = tsdb.NewIntervalCalculator(&tsdb.IntervalOptions{MinInterval: 15 * time.Second}) + clientLog = log.New(loggerName) ) // Client represents a client which can interact with elasticsearch api From 242689abe242d30bfdd4ab9b8a6be6347eb37931 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 23 May 2018 16:46:26 +0200 Subject: [PATCH 13/22] elasticsearch: pipeline aggregation fix for json encoding --- pkg/tsdb/elasticsearch/client/models.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/tsdb/elasticsearch/client/models.go b/pkg/tsdb/elasticsearch/client/models.go index 3c86dcce825..2d9839dfd53 100644 --- a/pkg/tsdb/elasticsearch/client/models.go +++ b/pkg/tsdb/elasticsearch/client/models.go @@ -293,7 +293,7 @@ type PipelineAggregation struct { // MarshalJSON returns the JSON encoding of the pipeline aggregation func (a *PipelineAggregation) MarshalJSON() ([]byte, error) { root := map[string]interface{}{ - "bucket_path": a.BucketPath, + "buckets_path": a.BucketPath, } for k, v := range a.Settings { From 8870e3e85b84a5adf371faeb3b833e90385fd006 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 23 May 2018 21:44:09 +0200 Subject: [PATCH 14/22] elasticsearch: default interval fix 5s instead of 15s --- pkg/tsdb/elasticsearch/client/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/tsdb/elasticsearch/client/client.go b/pkg/tsdb/elasticsearch/client/client.go index 3762a58317a..e3583070d38 100644 --- a/pkg/tsdb/elasticsearch/client/client.go +++ b/pkg/tsdb/elasticsearch/client/client.go @@ -106,7 +106,7 @@ func (c *baseClientImpl) GetTimeField() string { func (c *baseClientImpl) GetMinInterval(queryInterval string) (time.Duration, error) { return tsdb.GetIntervalFrom(c.ds, simplejson.NewFromAny(map[string]string{ "interval": queryInterval, - }), 15*time.Second) + }), 5*time.Second) } func (c *baseClientImpl) getSettings() *simplejson.Json { From 4436b8da12c092df4f0cc583a3b8c9e36de6023d Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 23 May 2018 22:07:52 +0200 Subject: [PATCH 15/22] elasticsearch: query interval override fix --- pkg/tsdb/elasticsearch/client/client.go | 2 +- pkg/tsdb/elasticsearch/time_series_query.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/tsdb/elasticsearch/client/client.go b/pkg/tsdb/elasticsearch/client/client.go index e3583070d38..4fe5c40b127 100644 --- a/pkg/tsdb/elasticsearch/client/client.go +++ b/pkg/tsdb/elasticsearch/client/client.go @@ -104,7 +104,7 @@ func (c *baseClientImpl) GetTimeField() string { } func (c *baseClientImpl) GetMinInterval(queryInterval string) (time.Duration, error) { - return tsdb.GetIntervalFrom(c.ds, simplejson.NewFromAny(map[string]string{ + return tsdb.GetIntervalFrom(c.ds, simplejson.NewFromAny(map[string]interface{}{ "interval": queryInterval, }), 5*time.Second) } diff --git a/pkg/tsdb/elasticsearch/time_series_query.go b/pkg/tsdb/elasticsearch/time_series_query.go index ae4af7704fb..ef59c62c1dc 100644 --- a/pkg/tsdb/elasticsearch/time_series_query.go +++ b/pkg/tsdb/elasticsearch/time_series_query.go @@ -258,7 +258,7 @@ func (p *timeSeriesQueryParser) parse(tsdbQuery *tsdb.TsdbQuery) ([]*Query, erro return nil, err } alias := model.Get("alias").MustString("") - interval := model.Get("interval").MustString() + interval := strconv.FormatInt(q.IntervalMs, 10) + "ms" queries = append(queries, &Query{ TimeField: timeField, From 688f5b830ce728f99551b6032a40abd2958a2144 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 23 May 2018 22:21:41 +0200 Subject: [PATCH 16/22] elasticsearch: metric and pipeline agg setting json encoding fix --- pkg/tsdb/elasticsearch/client/models.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/tsdb/elasticsearch/client/models.go b/pkg/tsdb/elasticsearch/client/models.go index 2d9839dfd53..2f4f5dcd162 100644 --- a/pkg/tsdb/elasticsearch/client/models.go +++ b/pkg/tsdb/elasticsearch/client/models.go @@ -278,7 +278,9 @@ func (a *MetricAggregation) MarshalJSON() ([]byte, error) { } for k, v := range a.Settings { - root[k] = v + if k != "" && v != nil { + root[k] = v + } } return json.Marshal(root) @@ -297,7 +299,9 @@ func (a *PipelineAggregation) MarshalJSON() ([]byte, error) { } for k, v := range a.Settings { - root[k] = v + if k != "" && v != nil { + root[k] = v + } } return json.Marshal(root) From 16c3566a873ed7327604f09f88bff7d6a8d90bcb Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 23 May 2018 22:57:46 +0200 Subject: [PATCH 17/22] elasticsearch: handle NaN values --- pkg/tsdb/elasticsearch/response_parser.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/tsdb/elasticsearch/response_parser.go b/pkg/tsdb/elasticsearch/response_parser.go index 029b2e02142..4a45d6271b9 100644 --- a/pkg/tsdb/elasticsearch/response_parser.go +++ b/pkg/tsdb/elasticsearch/response_parser.go @@ -490,10 +490,14 @@ func castToNullFloat(j *simplejson.Json) null.Float { return null.FloatFrom(f) } - s, err := j.String() - if err == nil { - v, _ := strconv.ParseFloat(s, 64) - return null.FloatFromPtr(&v) + if s, err := j.String(); err == nil { + if strings.ToLower(s) == "nan" { + return null.NewFloat(0, false) + } + + if v, err := strconv.ParseFloat(s, 64); err == nil { + return null.FloatFromPtr(&v) + } } return null.NewFloat(0, false) From 4752d7884ab44323fcb425955c135b6182b673be Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Fri, 25 May 2018 10:31:56 +0200 Subject: [PATCH 18/22] elasticsearch: adds some more/better debug logging to client --- pkg/tsdb/elasticsearch/client/client.go | 30 +++++++++++++++++++------ 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/pkg/tsdb/elasticsearch/client/client.go b/pkg/tsdb/elasticsearch/client/client.go index 4fe5c40b127..3a2d31b42c6 100644 --- a/pkg/tsdb/elasticsearch/client/client.go +++ b/pkg/tsdb/elasticsearch/client/client.go @@ -119,6 +119,9 @@ type multiRequest struct { } func (c *baseClientImpl) executeBatchRequest(uriPath string, requests []*multiRequest) (*http.Response, error) { + clientLog.Debug("Encoding batch requests to json", "batch requests", len(requests)) + start := time.Now() + payload := bytes.Buffer{} for _, r := range requests { reqHeader, err := json.Marshal(r.header) @@ -134,6 +137,9 @@ func (c *baseClientImpl) executeBatchRequest(uriPath string, requests []*multiRe payload.WriteString(string(reqBody) + "\n") } + elapsed := time.Now().Sub(start) + clientLog.Debug("Encoded batch requests to json", "took", elapsed) + return c.executeRequest(http.MethodPost, uriPath, payload.Bytes()) } @@ -151,6 +157,9 @@ func (c *baseClientImpl) executeRequest(method, uriPath string, body []byte) (*h if err != nil { return nil, err } + + clientLog.Debug("Executing request", "url", req.URL.String(), "method", method) + req.Header.Set("User-Agent", "Grafana") req.Header.Set("Content-Type", "application/json") @@ -169,22 +178,28 @@ func (c *baseClientImpl) executeRequest(method, uriPath string, body []byte) (*h return nil, err } - if method == http.MethodPost { - clientLog.Debug("Executing request", "url", req.URL.String(), "method", method) - } else { - clientLog.Debug("Executing request", "url", req.URL.String(), "method", method) - } - + start := time.Now() + defer func() { + elapsed := time.Now().Sub(start) + clientLog.Debug("Executed request", "took", elapsed) + }() return ctxhttp.Do(c.ctx, httpClient, req) } func (c *baseClientImpl) ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearchResponse, error) { + clientLog.Debug("Executing multisearch", "search requests", len(r.Requests)) + multiRequests := c.createMultiSearchRequests(r.Requests) res, err := c.executeBatchRequest("_msearch", multiRequests) if err != nil { return nil, err } + clientLog.Debug("Received multisearch response", "code", res.StatusCode, "status", res.Status, "content-length", res.ContentLength) + + start := time.Now() + clientLog.Debug("Decoding multisearch json response") + var msr MultiSearchResponse defer res.Body.Close() dec := json.NewDecoder(res.Body) @@ -193,7 +208,8 @@ func (c *baseClientImpl) ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearch return nil, err } - clientLog.Debug("Received multisearch response", "code", res.StatusCode, "status", res.Status, "content-length", res.ContentLength) + elapsed := time.Now().Sub(start) + clientLog.Debug("Decoded multisearch json response", "took", elapsed) msr.status = res.StatusCode From 7cb0403faa07ea66c35e1cafae0661c4726bc3a5 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Mon, 28 May 2018 19:45:18 +0200 Subject: [PATCH 19/22] elasticsearch: handle if alert query contains template variable If datasource handles targetContainsTemplate function it can evaluate if a certain query contains template variables and this is used for show an error message that template variables not is supported in alert queries. --- .../datasource/elasticsearch/datasource.ts | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/public/app/plugins/datasource/elasticsearch/datasource.ts b/public/app/plugins/datasource/elasticsearch/datasource.ts index e3eccfb8029..5a8e83a16cb 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.ts +++ b/public/app/plugins/datasource/elasticsearch/datasource.ts @@ -408,4 +408,65 @@ export class ElasticDatasource { getTagValues(options) { return this.getTerms({ field: options.key, query: '*' }); } + + targetContainsTemplate(target) { + if (this.templateSrv.variableExists(target.query) || this.templateSrv.variableExists(target.alias)) { + return true; + } + + for (let bucketAgg of target.bucketAggs) { + if (this.templateSrv.variableExists(bucketAgg.field) || this.objectContainsTemplate(bucketAgg.settings)) { + return true; + } + } + + for (let metric of target.metrics) { + if ( + this.templateSrv.variableExists(metric.field) || + this.objectContainsTemplate(metric.settings) || + this.objectContainsTemplate(metric.meta) + ) { + return true; + } + } + + return false; + } + + private isPrimitive(obj) { + if (obj === null || obj === undefined) { + return true; + } + if (['string', 'number', 'boolean'].some(type => type === typeof true)) { + return true; + } + + return false; + } + + private objectContainsTemplate(obj) { + if (!obj) { + return false; + } + + for (let key of Object.keys(obj)) { + if (this.isPrimitive(obj[key])) { + if (this.templateSrv.variableExists(obj[key])) { + return true; + } + } else if (Array.isArray(obj[key])) { + for (let item of obj[key]) { + if (this.objectContainsTemplate(item)) { + return true; + } + } + } else { + if (this.objectContainsTemplate(obj[key])) { + return true; + } + } + } + + return false; + } } From dcac63936bbff64fb0e9277b216a0e1e27955d51 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Thu, 31 May 2018 19:02:20 +0200 Subject: [PATCH 20/22] elasticsearch: minor refactor Handle all replacements if interval template variables in the client. Fix issue with client and different versions. Adds better tests of the client --- pkg/tsdb/elasticsearch/client/client.go | 128 +++----- pkg/tsdb/elasticsearch/client/client_test.go | 279 ++++++++++++------ pkg/tsdb/elasticsearch/client/models.go | 3 + .../elasticsearch/client/search_request.go | 11 +- .../client/search_request_test.go | 12 +- pkg/tsdb/elasticsearch/time_series_query.go | 12 +- .../elasticsearch/time_series_query_test.go | 2 +- 7 files changed, 252 insertions(+), 195 deletions(-) diff --git a/pkg/tsdb/elasticsearch/client/client.go b/pkg/tsdb/elasticsearch/client/client.go index 3a2d31b42c6..8ae45599484 100644 --- a/pkg/tsdb/elasticsearch/client/client.go +++ b/pkg/tsdb/elasticsearch/client/client.go @@ -8,6 +8,7 @@ import ( "net/http" "net/url" "path" + "strconv" "strings" "time" @@ -25,6 +26,10 @@ var ( clientLog = log.New(loggerName) ) +var newDatasourceHttpClient = func(ds *models.DataSource) (*http.Client, error) { + return ds.GetHttpClient() +} + // Client represents a client which can interact with elasticsearch api type Client interface { GetVersion() int @@ -57,23 +62,18 @@ var NewClient = func(ctx context.Context, ds *models.DataSource, timeRange *tsdb return nil, err } - bc := &baseClientImpl{ - ctx: ctx, - ds: ds, - version: version, - timeField: timeField, - indices: indices, - } - clientLog.Debug("Creating new client", "version", version, "timeField", timeField, "indices", strings.Join(indices, ", ")) switch version { - case 2: - return newV2Client(bc) - case 5: - return newV5Client(bc) - case 56: - return newV56Client(bc) + case 2, 5, 56: + return &baseClientImpl{ + ctx: ctx, + ds: ds, + version: version, + timeField: timeField, + indices: indices, + timeRange: timeRange, + }, nil } return nil, fmt.Errorf("elasticsearch version=%d is not supported", version) @@ -93,6 +93,7 @@ type baseClientImpl struct { version int timeField string indices []string + timeRange *tsdb.TimeRange } func (c *baseClientImpl) GetVersion() int { @@ -114,11 +115,20 @@ func (c *baseClientImpl) getSettings() *simplejson.Json { } type multiRequest struct { - header map[string]interface{} - body interface{} + header map[string]interface{} + body interface{} + interval tsdb.Interval } func (c *baseClientImpl) executeBatchRequest(uriPath string, requests []*multiRequest) (*http.Response, error) { + bytes, err := c.encodeBatchRequests(requests) + if err != nil { + return nil, err + } + return c.executeRequest(http.MethodPost, uriPath, bytes) +} + +func (c *baseClientImpl) encodeBatchRequests(requests []*multiRequest) ([]byte, error) { clientLog.Debug("Encoding batch requests to json", "batch requests", len(requests)) start := time.Now() @@ -134,13 +144,18 @@ func (c *baseClientImpl) executeBatchRequest(uriPath string, requests []*multiRe if err != nil { return nil, err } - payload.WriteString(string(reqBody) + "\n") + + body := string(reqBody) + body = strings.Replace(body, "$__interval_ms", strconv.FormatInt(r.interval.Value.Nanoseconds()/int64(time.Millisecond), 10), -1) + body = strings.Replace(body, "$__interval", r.interval.Text, -1) + + payload.WriteString(body + "\n") } elapsed := time.Now().Sub(start) clientLog.Debug("Encoded batch requests to json", "took", elapsed) - return c.executeRequest(http.MethodPost, uriPath, payload.Bytes()) + return payload.Bytes(), nil } func (c *baseClientImpl) executeRequest(method, uriPath string, body []byte) (*http.Response, error) { @@ -173,7 +188,7 @@ func (c *baseClientImpl) executeRequest(method, uriPath string, body []byte) (*h req.SetBasicAuth(c.ds.User, c.ds.Password) } - httpClient, err := c.ds.GetHttpClient() + httpClient, err := newDatasourceHttpClient(c.ds) if err != nil { return nil, err } @@ -220,77 +235,26 @@ func (c *baseClientImpl) createMultiSearchRequests(searchRequests []*SearchReque multiRequests := []*multiRequest{} for _, searchReq := range searchRequests { - multiRequests = append(multiRequests, &multiRequest{ + mr := multiRequest{ header: map[string]interface{}{ "search_type": "query_then_fetch", "ignore_unavailable": true, "index": strings.Join(c.indices, ","), }, - body: searchReq, - }) - } + body: searchReq, + interval: searchReq.Interval, + } - return multiRequests -} + if c.version == 2 { + mr.header["search_type"] = "count" + } -type v2Client struct { - baseClient -} + if c.version >= 56 { + maxConcurrentShardRequests := c.getSettings().Get("maxConcurrentShardRequests").MustInt(256) + mr.header["max_concurrent_shard_requests"] = maxConcurrentShardRequests + } -func newV2Client(bc baseClient) (*v2Client, error) { - c := v2Client{ - baseClient: bc, - } - - return &c, nil -} - -func (c *v2Client) createMultiSearchRequests(searchRequests []*SearchRequest) []*multiRequest { - multiRequests := c.baseClient.createMultiSearchRequests(searchRequests) - - for _, mr := range multiRequests { - mr.header["search_type"] = "count" - } - - return multiRequests -} - -type v5Client struct { - baseClient -} - -func newV5Client(bc baseClient) (*v5Client, error) { - c := v5Client{ - baseClient: bc, - } - - return &c, nil -} - -type v56Client struct { - *v5Client - maxConcurrentShardRequests int -} - -func newV56Client(bc baseClient) (*v56Client, error) { - v5Client := v5Client{ - baseClient: bc, - } - maxConcurrentShardRequests := bc.getSettings().Get("maxConcurrentShardRequests").MustInt(256) - - c := v56Client{ - v5Client: &v5Client, - maxConcurrentShardRequests: maxConcurrentShardRequests, - } - - return &c, nil -} - -func (c *v56Client) createMultiSearchRequests(searchRequests []*SearchRequest) []*multiRequest { - multiRequests := c.v5Client.createMultiSearchRequests(searchRequests) - - for _, mr := range multiRequests { - mr.header["max_concurrent_shard_requests"] = c.maxConcurrentShardRequests + multiRequests = append(multiRequests, &mr) } return multiRequests diff --git a/pkg/tsdb/elasticsearch/client/client_test.go b/pkg/tsdb/elasticsearch/client/client_test.go index d557ceb28b1..11d1cdb1d71 100644 --- a/pkg/tsdb/elasticsearch/client/client_test.go +++ b/pkg/tsdb/elasticsearch/client/client_test.go @@ -1,10 +1,17 @@ package es import ( + "bytes" + "context" + "fmt" + "io/ioutil" "net/http" + "net/http/httptest" "testing" + "time" "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/tsdb" "github.com/grafana/grafana/pkg/models" . "github.com/smartystreets/goconvey/convey" @@ -85,131 +92,213 @@ func TestClient(t *testing.T) { }) }) - Convey("v2", func() { - ds := &models.DataSource{ - JsonData: simplejson.NewFromAny(map[string]interface{}{ - "esVersion": 2, - }), + Convey("Given a fake http client", func() { + var responseBuffer *bytes.Buffer + var req *http.Request + ts := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + req = r + buf, err := ioutil.ReadAll(r.Body) + if err != nil { + t.Fatalf("Failed to read response body, err=%v", err) + } + responseBuffer = bytes.NewBuffer(buf) + })) + + currentNewDatasourceHttpClient := newDatasourceHttpClient + + newDatasourceHttpClient = func(ds *models.DataSource) (*http.Client, error) { + return ts.Client(), nil } - c, err := newV2Client(newFakeBaseClient(ds, []string{"test-*"})) - So(err, ShouldBeNil) - So(c, ShouldNotBeNil) + from := time.Date(2018, 5, 15, 17, 50, 0, 0, time.UTC) + to := time.Date(2018, 5, 15, 17, 55, 0, 0, time.UTC) + fromStr := fmt.Sprintf("%d", from.UnixNano()/int64(time.Millisecond)) + toStr := fmt.Sprintf("%d", to.UnixNano()/int64(time.Millisecond)) + timeRange := tsdb.NewTimeRange(fromStr, toStr) - Convey("When creating multisearch requests should have correct headers", func() { - multiRequests := c.createMultiSearchRequests([]*SearchRequest{ - {Index: "test-*"}, - }) - So(multiRequests, ShouldHaveLength, 1) - header := multiRequests[0].header - So(header, ShouldHaveLength, 3) - So(header["index"], ShouldEqual, "test-*") - So(header["ignore_unavailable"], ShouldEqual, true) - So(header["search_type"], ShouldEqual, "count") - }) - }) - - Convey("v5", func() { - ds := &models.DataSource{ - JsonData: simplejson.NewFromAny(map[string]interface{}{ - "esVersion": 5, - }), - } - - c, err := newV5Client(newFakeBaseClient(ds, []string{"test-*"})) - So(err, ShouldBeNil) - So(c, ShouldNotBeNil) - - Convey("When creating multisearch requests should have correct headers", func() { - multiRequests := c.createMultiSearchRequests([]*SearchRequest{ - {Index: "test-*"}, - }) - So(multiRequests, ShouldHaveLength, 1) - header := multiRequests[0].header - So(header, ShouldHaveLength, 3) - So(header["index"], ShouldEqual, "test-*") - So(header["ignore_unavailable"], ShouldEqual, true) - So(header["search_type"], ShouldEqual, "query_then_fetch") - }) - }) - - Convey("v5.6", func() { - Convey("With default settings", func() { + Convey("and a v2.x client", func() { ds := models.DataSource{ + Database: "[metrics-]YYYY.MM.DD", + Url: ts.URL, JsonData: simplejson.NewFromAny(map[string]interface{}{ - "esVersion": 56, + "esVersion": 2, + "timeField": "@timestamp", + "interval": "Daily", }), } - c, err := newV56Client(newFakeBaseClient(&ds, []string{"test-*"})) + c, err := NewClient(context.Background(), &ds, timeRange) So(err, ShouldBeNil) So(c, ShouldNotBeNil) - Convey("When creating multisearch requests should have correct headers", func() { - multiRequests := c.createMultiSearchRequests([]*SearchRequest{ - {Index: "test-*"}, + Convey("When executing multi search", func() { + ms, err := createMultisearchForTest(c) + So(err, ShouldBeNil) + c.ExecuteMultisearch(ms) + + Convey("Should send correct request and payload", func() { + So(req, ShouldNotBeNil) + So(req.Method, ShouldEqual, http.MethodPost) + So(req.URL.Path, ShouldEqual, "/_msearch") + + So(responseBuffer, ShouldNotBeNil) + + headerBytes, err := responseBuffer.ReadBytes('\n') + So(err, ShouldBeNil) + bodyBytes := responseBuffer.Bytes() + + jHeader, err := simplejson.NewJson(headerBytes) + So(err, ShouldBeNil) + + jBody, err := simplejson.NewJson(bodyBytes) + So(err, ShouldBeNil) + + fmt.Println("body", string(headerBytes)) + + So(jHeader.Get("index").MustString(), ShouldEqual, "metrics-2018.05.15") + So(jHeader.Get("ignore_unavailable").MustBool(false), ShouldEqual, true) + So(jHeader.Get("search_type").MustString(), ShouldEqual, "count") + So(jHeader.Get("max_concurrent_shard_requests").MustInt(10), ShouldEqual, 10) + + Convey("and replace $__interval variable", func() { + So(jBody.GetPath("aggs", "2", "aggs", "1", "avg", "script").MustString(), ShouldEqual, "15000*@hostname") + }) + + Convey("and replace $__interval_ms variable", func() { + So(jBody.GetPath("aggs", "2", "date_histogram", "interval").MustString(), ShouldEqual, "15s") + }) }) - So(multiRequests, ShouldHaveLength, 1) - header := multiRequests[0].header - So(header, ShouldHaveLength, 4) - So(header["index"], ShouldEqual, "test-*") - So(header["ignore_unavailable"], ShouldEqual, true) - So(header["search_type"], ShouldEqual, "query_then_fetch") - So(header["max_concurrent_shard_requests"], ShouldEqual, 256) }) }) - Convey("With custom settings", func() { + Convey("and a v5.x client", func() { ds := models.DataSource{ + Database: "[metrics-]YYYY.MM.DD", + Url: ts.URL, + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 5, + "maxConcurrentShardRequests": 100, + "timeField": "@timestamp", + "interval": "Daily", + }), + } + + c, err := NewClient(context.Background(), &ds, timeRange) + So(err, ShouldBeNil) + So(c, ShouldNotBeNil) + + Convey("When executing multi search", func() { + ms, err := createMultisearchForTest(c) + So(err, ShouldBeNil) + c.ExecuteMultisearch(ms) + + Convey("Should send correct request and payload", func() { + So(req, ShouldNotBeNil) + So(req.Method, ShouldEqual, http.MethodPost) + So(req.URL.Path, ShouldEqual, "/_msearch") + + So(responseBuffer, ShouldNotBeNil) + + headerBytes, err := responseBuffer.ReadBytes('\n') + So(err, ShouldBeNil) + bodyBytes := responseBuffer.Bytes() + + jHeader, err := simplejson.NewJson(headerBytes) + So(err, ShouldBeNil) + + jBody, err := simplejson.NewJson(bodyBytes) + So(err, ShouldBeNil) + + fmt.Println("body", string(headerBytes)) + + So(jHeader.Get("index").MustString(), ShouldEqual, "metrics-2018.05.15") + So(jHeader.Get("ignore_unavailable").MustBool(false), ShouldEqual, true) + So(jHeader.Get("search_type").MustString(), ShouldEqual, "query_then_fetch") + So(jHeader.Get("max_concurrent_shard_requests").MustInt(10), ShouldEqual, 10) + + Convey("and replace $__interval variable", func() { + So(jBody.GetPath("aggs", "2", "aggs", "1", "avg", "script").MustString(), ShouldEqual, "15000*@hostname") + }) + + Convey("and replace $__interval_ms variable", func() { + So(jBody.GetPath("aggs", "2", "date_histogram", "interval").MustString(), ShouldEqual, "15s") + }) + }) + }) + }) + + Convey("and a v5.6 client", func() { + ds := models.DataSource{ + Database: "[metrics-]YYYY.MM.DD", + Url: ts.URL, JsonData: simplejson.NewFromAny(map[string]interface{}{ "esVersion": 56, "maxConcurrentShardRequests": 100, + "timeField": "@timestamp", + "interval": "Daily", }), } - c, err := newV56Client(newFakeBaseClient(&ds, []string{"test-*"})) + c, err := NewClient(context.Background(), &ds, timeRange) So(err, ShouldBeNil) So(c, ShouldNotBeNil) - Convey("When creating multisearch requests should have correct headers", func() { - multiRequests := c.createMultiSearchRequests([]*SearchRequest{ - {Index: "test-*"}, + + Convey("When executing multi search", func() { + ms, err := createMultisearchForTest(c) + So(err, ShouldBeNil) + c.ExecuteMultisearch(ms) + + Convey("Should send correct request and payload", func() { + So(req, ShouldNotBeNil) + So(req.Method, ShouldEqual, http.MethodPost) + So(req.URL.Path, ShouldEqual, "/_msearch") + + So(responseBuffer, ShouldNotBeNil) + + headerBytes, err := responseBuffer.ReadBytes('\n') + So(err, ShouldBeNil) + bodyBytes := responseBuffer.Bytes() + + jHeader, err := simplejson.NewJson(headerBytes) + So(err, ShouldBeNil) + + jBody, err := simplejson.NewJson(bodyBytes) + So(err, ShouldBeNil) + + fmt.Println("body", string(headerBytes)) + + So(jHeader.Get("index").MustString(), ShouldEqual, "metrics-2018.05.15") + So(jHeader.Get("ignore_unavailable").MustBool(false), ShouldEqual, true) + So(jHeader.Get("search_type").MustString(), ShouldEqual, "query_then_fetch") + So(jHeader.Get("max_concurrent_shard_requests").MustInt(), ShouldEqual, 100) + + Convey("and replace $__interval variable", func() { + So(jBody.GetPath("aggs", "2", "aggs", "1", "avg", "script").MustString(), ShouldEqual, "15000*@hostname") + }) + + Convey("and replace $__interval_ms variable", func() { + So(jBody.GetPath("aggs", "2", "date_histogram", "interval").MustString(), ShouldEqual, "15s") + }) }) - So(multiRequests, ShouldHaveLength, 1) - header := multiRequests[0].header - So(header, ShouldHaveLength, 4) - So(header["index"], ShouldEqual, "test-*") - So(header["ignore_unavailable"], ShouldEqual, true) - So(header["search_type"], ShouldEqual, "query_then_fetch") - So(header["max_concurrent_shard_requests"], ShouldEqual, 100) }) }) + + Reset(func() { + newDatasourceHttpClient = currentNewDatasourceHttpClient + }) }) }) } -type fakeBaseClient struct { - *baseClientImpl - ds *models.DataSource -} +func createMultisearchForTest(c Client) (*MultiSearchRequest, error) { + msb := c.MultiSearch() + s := msb.Search(tsdb.Interval{Value: 15 * time.Second, Text: "15s"}) + s.Agg().DateHistogram("2", "@timestamp", func(a *DateHistogramAgg, ab AggBuilder) { + a.Interval = "$__interval" -func newFakeBaseClient(ds *models.DataSource, indices []string) baseClient { - return &fakeBaseClient{ - baseClientImpl: &baseClientImpl{ - ds: ds, - indices: indices, - }, - ds: ds, - } -} - -func (c *fakeBaseClient) executeBatchRequest(uriPath string, requests []*multiRequest) (*http.Response, error) { - return nil, nil -} - -func (c *fakeBaseClient) executeRequest(method, uriPath string, body []byte) (*http.Response, error) { - return nil, nil -} - -func (c *fakeBaseClient) executeMultisearch(searchRequests []*SearchRequest) ([]*SearchResponse, error) { - return nil, nil + ab.Metric("1", "avg", "@hostname", func(a *MetricAggregation) { + a.Settings["script"] = "$__interval_ms*@hostname" + }) + }) + return msb.Build() } diff --git a/pkg/tsdb/elasticsearch/client/models.go b/pkg/tsdb/elasticsearch/client/models.go index 2f4f5dcd162..a5810a9b109 100644 --- a/pkg/tsdb/elasticsearch/client/models.go +++ b/pkg/tsdb/elasticsearch/client/models.go @@ -2,11 +2,14 @@ package es import ( "encoding/json" + + "github.com/grafana/grafana/pkg/tsdb" ) // SearchRequest represents a search request type SearchRequest struct { Index string + Interval tsdb.Interval Size int Sort map[string]interface{} Query *Query diff --git a/pkg/tsdb/elasticsearch/client/search_request.go b/pkg/tsdb/elasticsearch/client/search_request.go index a582d8ec247..2b833ce78d3 100644 --- a/pkg/tsdb/elasticsearch/client/search_request.go +++ b/pkg/tsdb/elasticsearch/client/search_request.go @@ -2,11 +2,14 @@ package es import ( "strings" + + "github.com/grafana/grafana/pkg/tsdb" ) // SearchRequestBuilder represents a builder which can build a search request type SearchRequestBuilder struct { version int + interval tsdb.Interval index string size int sort map[string]interface{} @@ -16,9 +19,10 @@ type SearchRequestBuilder struct { } // NewSearchRequestBuilder create a new search request builder -func NewSearchRequestBuilder(version int) *SearchRequestBuilder { +func NewSearchRequestBuilder(version int, interval tsdb.Interval) *SearchRequestBuilder { builder := &SearchRequestBuilder{ version: version, + interval: interval, sort: make(map[string]interface{}), customProps: make(map[string]interface{}), aggBuilders: make([]AggBuilder, 0), @@ -30,6 +34,7 @@ func NewSearchRequestBuilder(version int) *SearchRequestBuilder { func (b *SearchRequestBuilder) Build() (*SearchRequest, error) { sr := SearchRequest{ Index: b.index, + Interval: b.interval, Size: b.size, Sort: b.sort, CustomProps: b.customProps, @@ -128,8 +133,8 @@ func NewMultiSearchRequestBuilder(version int) *MultiSearchRequestBuilder { } // Search initiates and returns a new search request builder -func (m *MultiSearchRequestBuilder) Search() *SearchRequestBuilder { - b := NewSearchRequestBuilder(m.version) +func (m *MultiSearchRequestBuilder) Search(interval tsdb.Interval) *SearchRequestBuilder { + b := NewSearchRequestBuilder(m.version, interval) m.requestBuilders = append(m.requestBuilders, b) return b } diff --git a/pkg/tsdb/elasticsearch/client/search_request_test.go b/pkg/tsdb/elasticsearch/client/search_request_test.go index d93f8826442..b026578d64f 100644 --- a/pkg/tsdb/elasticsearch/client/search_request_test.go +++ b/pkg/tsdb/elasticsearch/client/search_request_test.go @@ -3,8 +3,10 @@ package es import ( "encoding/json" "testing" + "time" "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/tsdb" . "github.com/smartystreets/goconvey/convey" ) @@ -13,7 +15,7 @@ func TestSearchRequest(t *testing.T) { Convey("Test elasticsearch search request", t, func() { timeField := "@timestamp" Convey("Given new search request builder for es version 5", func() { - b := NewSearchRequestBuilder(5) + b := NewSearchRequestBuilder(5, tsdb.Interval{Value: 15 * time.Second, Text: "15s"}) Convey("When building search request", func() { sr, err := b.Build() @@ -388,7 +390,7 @@ func TestSearchRequest(t *testing.T) { }) Convey("Given new search request builder for es version 2", func() { - b := NewSearchRequestBuilder(2) + b := NewSearchRequestBuilder(2, tsdb.Interval{Value: 15 * time.Second, Text: "15s"}) Convey("When adding doc value field", func() { b.AddDocValueField(timeField) @@ -447,7 +449,7 @@ func TestMultiSearchRequest(t *testing.T) { b := NewMultiSearchRequestBuilder(0) Convey("When adding one search request", func() { - b.Search() + b.Search(tsdb.Interval{Value: 15 * time.Second, Text: "15s"}) Convey("When building search request should contain one search request", func() { mr, err := b.Build() @@ -457,8 +459,8 @@ func TestMultiSearchRequest(t *testing.T) { }) Convey("When adding two search requests", func() { - b.Search() - b.Search() + b.Search(tsdb.Interval{Value: 15 * time.Second, Text: "15s"}) + b.Search(tsdb.Interval{Value: 15 * time.Second, Text: "15s"}) Convey("When building search request should contain two search requests", func() { mr, err := b.Build() diff --git a/pkg/tsdb/elasticsearch/time_series_query.go b/pkg/tsdb/elasticsearch/time_series_query.go index ef59c62c1dc..c9bb05dd09a 100644 --- a/pkg/tsdb/elasticsearch/time_series_query.go +++ b/pkg/tsdb/elasticsearch/time_series_query.go @@ -3,8 +3,6 @@ package elasticsearch import ( "fmt" "strconv" - "strings" - "time" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/tsdb" @@ -47,7 +45,7 @@ func (e *timeSeriesQuery) execute() (*tsdb.Response, error) { } interval := e.intervalCalculator.Calculate(e.tsdbQuery.TimeRange, minInterval) - b := ms.Search() + b := ms.Search(interval) b.Size(0) filters := b.Query().Bool().Filter() filters.AddDateRangeFilter(e.client.GetTimeField(), to, from, es.DateFormatEpochMS) @@ -78,7 +76,7 @@ func (e *timeSeriesQuery) execute() (*tsdb.Response, error) { for _, bucketAgg := range q.BucketAggs { switch bucketAgg.Type { case "date_histogram": - aggBuilder = addDateHistogramAgg(aggBuilder, bucketAgg, from, to, interval) + aggBuilder = addDateHistogramAgg(aggBuilder, bucketAgg, from, to) case "histogram": aggBuilder = addHistogramAgg(aggBuilder, bucketAgg) case "filters": @@ -125,7 +123,7 @@ func (e *timeSeriesQuery) execute() (*tsdb.Response, error) { return rp.getTimeSeries() } -func addDateHistogramAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg, timeFrom, timeTo string, interval tsdb.Interval) es.AggBuilder { +func addDateHistogramAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg, timeFrom, timeTo string) es.AggBuilder { aggBuilder.DateHistogram(bucketAgg.ID, bucketAgg.Field, func(a *es.DateHistogramAgg, b es.AggBuilder) { a.Interval = bucketAgg.Settings.Get("interval").MustString("auto") a.MinDocCount = bucketAgg.Settings.Get("min_doc_count").MustInt(0) @@ -136,10 +134,6 @@ func addDateHistogramAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg, timeFro a.Interval = "$__interval" } - a.Interval = strings.Replace(a.Interval, "$interval", interval.Text, -1) - a.Interval = strings.Replace(a.Interval, "$__interval_ms", strconv.FormatInt(interval.Value.Nanoseconds()/int64(time.Millisecond), 10), -1) - a.Interval = strings.Replace(a.Interval, "$__interval", interval.Text, -1) - if missing, err := bucketAgg.Settings.Get("missing").String(); err == nil { a.Missing = &missing } diff --git a/pkg/tsdb/elasticsearch/time_series_query_test.go b/pkg/tsdb/elasticsearch/time_series_query_test.go index e2af4de749a..49bf5f5bc75 100644 --- a/pkg/tsdb/elasticsearch/time_series_query_test.go +++ b/pkg/tsdb/elasticsearch/time_series_query_test.go @@ -268,7 +268,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { So(firstLevel.Aggregation.Type, ShouldEqual, "date_histogram") hAgg := firstLevel.Aggregation.Aggregation.(*es.DateHistogramAgg) So(hAgg.Field, ShouldEqual, "@timestamp") - So(hAgg.Interval, ShouldEqual, "15s") + So(hAgg.Interval, ShouldEqual, "$__interval") So(hAgg.MinDocCount, ShouldEqual, 2) }) From b8ff3b1e3fa2fa061aaad41f2e44edb15272150b Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Thu, 31 May 2018 19:05:32 +0200 Subject: [PATCH 21/22] remove dead code --- pkg/tsdb/elasticsearch/client/client.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pkg/tsdb/elasticsearch/client/client.go b/pkg/tsdb/elasticsearch/client/client.go index 8ae45599484..0cd3eb0d21b 100644 --- a/pkg/tsdb/elasticsearch/client/client.go +++ b/pkg/tsdb/elasticsearch/client/client.go @@ -79,14 +79,6 @@ var NewClient = func(ctx context.Context, ds *models.DataSource, timeRange *tsdb return nil, fmt.Errorf("elasticsearch version=%d is not supported", version) } -type baseClient interface { - Client - getSettings() *simplejson.Json - executeBatchRequest(uriPath string, requests []*multiRequest) (*http.Response, error) - executeRequest(method, uriPath string, body []byte) (*http.Response, error) - createMultiSearchRequests(searchRequests []*SearchRequest) []*multiRequest -} - type baseClientImpl struct { ctx context.Context ds *models.DataSource From ce75afa413defc682fab6a22f02dad45c644f7ce Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Fri, 1 Jun 2018 12:26:20 +0200 Subject: [PATCH 22/22] docs: update alerting docs with alerting support for elasticsearch --- docs/sources/alerting/rules.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/sources/alerting/rules.md b/docs/sources/alerting/rules.md index bcca3c6b2fb..fa7332e7145 100644 --- a/docs/sources/alerting/rules.md +++ b/docs/sources/alerting/rules.md @@ -27,7 +27,9 @@ and the conditions that need to be met for the alert to change state and trigger ## Execution The alert rules are evaluated in the Grafana backend in a scheduler and query execution engine that is part -of core Grafana. Only some data sources are supported right now. They include `Graphite`, `Prometheus`, `InfluxDB`, `OpenTSDB`, `MySQL`, `Postgres` and `Cloudwatch`. +of core Grafana. Only some data sources are supported right now. They include `Graphite`, `Prometheus`, `Elasticsearch`, `InfluxDB`, `OpenTSDB`, `MySQL`, `Postgres` and `Cloudwatch`. + +> Alerting support for Elasticsearch is only available in Grafana v5.2 and above. ### Clustering @@ -152,6 +154,8 @@ filters = alerting.scheduler:debug \ tsdb.prometheus:debug \ tsdb.opentsdb:debug \ tsdb.influxdb:debug \ + tsdb.elasticsearch:debug \ + tsdb.elasticsearch.client:debug \ ``` If you want to log raw query sent to your TSDB and raw response in log you also have to set grafana.ini option `app_mode` to