From 8e7d23cdebc3df236d519777e3e4485d5ad32d12 Mon Sep 17 00:00:00 2001 From: wph95 Date: Fri, 23 Mar 2018 23:50:16 +0800 Subject: [PATCH 01/87] wip Signed-off-by: wph95 --- pkg/cmd/grafana-server/main.go | 1 + pkg/tsdb/elasticsearch/elasticsearch.go | 131 +++++++++++ pkg/tsdb/elasticsearch/model_parser.go | 97 +++++++++ pkg/tsdb/elasticsearch/models.go | 131 +++++++++++ pkg/tsdb/elasticsearch/query.go | 204 ++++++++++++++++++ pkg/tsdb/elasticsearch/response_parser.go | 111 ++++++++++ .../datasource/elasticsearch/plugin.json | 1 + 7 files changed, 676 insertions(+) create mode 100644 pkg/tsdb/elasticsearch/elasticsearch.go create mode 100644 pkg/tsdb/elasticsearch/model_parser.go create mode 100644 pkg/tsdb/elasticsearch/models.go create mode 100644 pkg/tsdb/elasticsearch/query.go create mode 100644 pkg/tsdb/elasticsearch/response_parser.go diff --git a/pkg/cmd/grafana-server/main.go b/pkg/cmd/grafana-server/main.go index ab0e12f2d9f..21090153bc0 100644 --- a/pkg/cmd/grafana-server/main.go +++ b/pkg/cmd/grafana-server/main.go @@ -21,6 +21,7 @@ import ( _ "github.com/grafana/grafana/pkg/services/alerting/conditions" _ "github.com/grafana/grafana/pkg/services/alerting/notifiers" _ "github.com/grafana/grafana/pkg/tsdb/cloudwatch" + _ "github.com/grafana/grafana/pkg/tsdb/elasticsearch" _ "github.com/grafana/grafana/pkg/tsdb/graphite" _ "github.com/grafana/grafana/pkg/tsdb/influxdb" _ "github.com/grafana/grafana/pkg/tsdb/mysql" diff --git a/pkg/tsdb/elasticsearch/elasticsearch.go b/pkg/tsdb/elasticsearch/elasticsearch.go new file mode 100644 index 00000000000..d67b4ad902d --- /dev/null +++ b/pkg/tsdb/elasticsearch/elasticsearch.go @@ -0,0 +1,131 @@ +package elasticsearch + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "github.com/davecgh/go-spew/spew" + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/tsdb" + "golang.org/x/net/context/ctxhttp" + "net/http" + "net/url" + "path" + "strings" + "time" +) + +type ElasticsearchExecutor struct { + Transport *http.Transport +} + +var ( + glog log.Logger + intervalCalculator tsdb.IntervalCalculator +) + +func NewElasticsearchExecutor(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) { + transport, err := dsInfo.GetHttpTransport() + if err != nil { + return nil, err + } + + return &ElasticsearchExecutor{ + Transport: transport, + }, nil +} + +func init() { + glog = log.New("tsdb.elasticsearch") + tsdb.RegisterTsdbQueryEndpoint("elasticsearch", NewElasticsearchExecutor) + intervalCalculator = tsdb.NewIntervalCalculator(&tsdb.IntervalOptions{MinInterval: time.Millisecond * 1}) +} + +func (e *ElasticsearchExecutor) Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) { + result := &tsdb.Response{} + result.Results = make(map[string]*tsdb.QueryResult) + + queryParser := ElasticSearchQueryParser{ + dsInfo, + tsdbQuery.TimeRange, + tsdbQuery.Queries, + glog, + } + + glog.Warn(spew.Sdump(dsInfo)) + glog.Warn(spew.Sdump(tsdbQuery)) + + payload, err := queryParser.Parse() + if err != nil { + return nil, err + } + + if setting.Env == setting.DEV { + glog.Debug("Elasticsearch playload", "raw playload", payload) + } + glog.Info("Elasticsearch playload", "raw playload", payload) + + req, err := e.createRequest(dsInfo, payload) + if err != nil { + return nil, err + } + + httpClient, err := dsInfo.GetHttpClient() + if err != nil { + return nil, err + } + + resp, err := ctxhttp.Do(ctx, httpClient, req) + if err != nil { + return nil, err + } + + if resp.StatusCode/100 != 2 { + return nil, fmt.Errorf("elasticsearch returned statuscode invalid status code: %v", resp.Status) + } + + var responses Responses + dec := json.NewDecoder(resp.Body) + defer resp.Body.Close() + dec.UseNumber() + err = dec.Decode(&responses) + if err != nil { + return nil, err + } + + glog.Warn(spew.Sdump(responses)) + for _, res := range responses.Responses { + if res.Err != nil { + return nil, errors.New(res.getErrMsg()) + } + + } + + return result, nil +} + +func (e *ElasticsearchExecutor) createRequest(dsInfo *models.DataSource, query string) (*http.Request, error) { + u, _ := url.Parse(dsInfo.Url) + u.Path = path.Join(u.Path, "_msearch") + req, err := http.NewRequest(http.MethodPost, u.String(), strings.NewReader(query)) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", "Grafana") + req.Header.Set("Content-Type", "application/json") + + if dsInfo.BasicAuth { + req.SetBasicAuth(dsInfo.BasicAuthUser, dsInfo.BasicAuthPassword) + } + + if !dsInfo.BasicAuth && dsInfo.User != "" { + req.SetBasicAuth(dsInfo.User, dsInfo.Password) + } + + glog.Debug("Elasticsearch request", "url", req.URL.String()) + glog.Debug("Elasticsearch request", "body", query) + return req, nil +} diff --git a/pkg/tsdb/elasticsearch/model_parser.go b/pkg/tsdb/elasticsearch/model_parser.go new file mode 100644 index 00000000000..136db6baed7 --- /dev/null +++ b/pkg/tsdb/elasticsearch/model_parser.go @@ -0,0 +1,97 @@ +package elasticsearch + +import ( + "bytes" + "encoding/json" + "fmt" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/tsdb" + "src/github.com/davecgh/go-spew/spew" + "strconv" + "strings" + "time" +) + +type ElasticSearchQueryParser struct { + DsInfo *models.DataSource + TimeRange *tsdb.TimeRange + Queries []*tsdb.Query + glog log.Logger +} + +func (qp *ElasticSearchQueryParser) Parse() (string, error) { + payload := bytes.Buffer{} + queryHeader := qp.getQueryHeader() + + for _, q := range qp.Queries { + timeField, err := q.Model.Get("timeField").String() + if err != nil { + return "", err + } + rawQuery := q.Model.Get("query").MustString("") + bucketAggs := q.Model.Get("bucketAggs").MustArray() + metrics := q.Model.Get("metrics").MustArray() + alias := q.Model.Get("alias").MustString("") + builder := QueryBuilder{timeField, rawQuery, bucketAggs, metrics, alias} + + query, err := builder.Build() + if err != nil { + return "", err + } + queryBytes, err := json.Marshal(query) + if err != nil { + return "", err + } + + payload.WriteString(queryHeader.String() + "\n") + payload.WriteString(string(queryBytes) + "\n") + } + + return qp.payloadReplace(payload.String(), qp.DsInfo.JsonData) + +} + +func (qp *ElasticSearchQueryParser) getQueryHeader() *QueryHeader { + var header QueryHeader + esVersion := qp.DsInfo.JsonData.Get("esVersion").MustInt() + + searchType := "query_then_fetch" + if esVersion < 5 { + searchType = "count" + } + header.SearchType = searchType + header.IgnoreUnavailable = true + header.Index = qp.getIndexList() + + if esVersion >= 56 { + header.MaxConcurrentShardRequests = qp.DsInfo.JsonData.Get("maxConcurrentShardRequests").MustInt() + } + return &header +} +func (qp *ElasticSearchQueryParser) payloadReplace(payload string, model *simplejson.Json) (string, error) { + parsedInterval, err := tsdb.GetIntervalFrom(qp.DsInfo, model, time.Millisecond) + if err != nil { + return "", nil + } + + interval := intervalCalculator.Calculate(qp.TimeRange, parsedInterval) + glog.Warn(spew.Sdump(interval)) + payload = strings.Replace(payload, "$timeFrom", fmt.Sprintf("%d", qp.TimeRange.GetFromAsMsEpoch()), -1) + payload = strings.Replace(payload, "$timeTo", fmt.Sprintf("%d", qp.TimeRange.GetToAsMsEpoch()), -1) + payload = strings.Replace(payload, "$interval", interval.Text, -1) + payload = strings.Replace(payload, "$__interval_ms", strconv.FormatInt(interval.Value.Nanoseconds()/int64(time.Millisecond), 10), -1) + payload = strings.Replace(payload, "$__interval", interval.Text, -1) + + return payload, nil +} + +func (qp *ElasticSearchQueryParser) getIndexList() string { + _, err := qp.DsInfo.JsonData.Get("interval").String() + if err != nil { + return qp.DsInfo.Database + } + // todo: support interval + return qp.DsInfo.Database +} diff --git a/pkg/tsdb/elasticsearch/models.go b/pkg/tsdb/elasticsearch/models.go new file mode 100644 index 00000000000..8662f6efbd3 --- /dev/null +++ b/pkg/tsdb/elasticsearch/models.go @@ -0,0 +1,131 @@ +package elasticsearch + +import ( + "github.com/grafana/grafana/pkg/components/simplejson" + "bytes" + "fmt" + "encoding/json" +) + +type QueryHeader struct { + SearchType string `json:"search_type"` + IgnoreUnavailable bool `json:"ignore_unavailable"` + Index interface{} `json:"index"` + MaxConcurrentShardRequests int `json:"max_concurrent_shard_requests"` +} + +func (q *QueryHeader) String() (string) { + r, _ := json.Marshal(q) + return string(r) +} + +type Query struct { + Query map[string]interface{} `json:"query"` + Aggs Aggs `json:"aggs"` + Size int `json:"size"` +} + +type Aggs map[string]interface{} + +type HistogramAgg struct { + Interval string `json:"interval,omitempty"` + Field string `json:"field"` + MinDocCount int `json:"min_doc_count"` + Missing string `json:"missing,omitempty"` +} + +type DateHistogramAgg struct { + HistogramAgg + ExtendedBounds ExtendedBounds `json:"extended_bounds"` + Format string `json:"format"` +} + +type FiltersAgg struct { + Filter map[string]interface{} `json:"filter"` +} + +type TermsAggSetting struct { + Field string `json:"field"` + Size int `json:"size"` + Order map[string]interface{} `json:"order"` + MinDocCount int `json:"min_doc_count"` + Missing string `json:"missing"` +} + +type TermsAgg struct { + Terms TermsAggSetting `json:"terms"` + Aggs Aggs `json:"aggs"` +} + +type ExtendedBounds struct { + Min string `json:"min"` + Max string `json:"max"` +} + +type RangeFilter struct { + Range map[string]RangeFilterSetting `json:"range"` +} +type RangeFilterSetting struct { + Gte string `json:"gte"` + Lte string `json:"lte"` + Format string `json:"format"` +} + +func newRangeFilter(field string, rangeFilterSetting RangeFilterSetting) *RangeFilter { + return &RangeFilter{ + map[string]RangeFilterSetting{field: rangeFilterSetting}} +} + +type QueryStringFilter struct { + QueryString QueryStringFilterSetting `json:"query_string"` +} +type QueryStringFilterSetting struct { + AnalyzeWildcard bool `json:"analyze_wildcard"` + Query string `json:"query"` +} + +func newQueryStringFilter(analyzeWildcard bool, query string) *QueryStringFilter { + return &QueryStringFilter{QueryStringFilterSetting{AnalyzeWildcard: analyzeWildcard, Query: query}} +} + +type BoolQuery struct { + Filter []interface{} `json:"filter"` +} + +type Metric map[string]interface{} + +type Responses struct { + Responses []Response `json:"responses"` +} + +type Response struct { + Status int `json:"status"` + Err map[string]interface{} `json:"error"` + Aggregations map[string]interface{} `json:"aggregations"` +} + +func (r *Response) getErrMsg() (string) { + var msg bytes.Buffer + errJson := simplejson.NewFromAny(r.Err) + errType, err := errJson.Get("type").String() + if err == nil { + msg.WriteString(fmt.Sprintf("type:%s", errType)) + } + + reason, err := errJson.Get("type").String() + if err == nil { + msg.WriteString(fmt.Sprintf("reason:%s", reason)) + } + return msg.String() +} + +type PercentilesResult struct { + Buckets struct { + map[string]struct { + Values map[string]string `json:"values"` + } + KeyAsString string `json:"key_as_string"` + Key int64 `json:"key"` + DocCount int `json:"doc_count"` + } `json:"buckets"` +} diff --git a/pkg/tsdb/elasticsearch/query.go b/pkg/tsdb/elasticsearch/query.go new file mode 100644 index 00000000000..69dd5caa3b4 --- /dev/null +++ b/pkg/tsdb/elasticsearch/query.go @@ -0,0 +1,204 @@ +package elasticsearch + +import ( + "errors" + "github.com/grafana/grafana/pkg/components/simplejson" +) + +var rangeFilterSetting = RangeFilterSetting{Gte: "$timeFrom", + Lte: "$timeTo", + Format: "epoch_millis"} + +type QueryBuilder struct { + TimeField string + RawQuery string + BucketAggs []interface{} + Metrics []interface{} + Alias string +} + +func (b *QueryBuilder) Build() (Query, error) { + var err error + var res Query + res.Query = make(map[string]interface{}) + res.Size = 0 + + if err != nil { + return res, err + } + + boolQuery := BoolQuery{} + boolQuery.Filter = append(boolQuery.Filter, newRangeFilter(b.TimeField, rangeFilterSetting)) + boolQuery.Filter = append(boolQuery.Filter, newQueryStringFilter(true, b.RawQuery)) + res.Query["bool"] = boolQuery + + // handle document query + if len(b.BucketAggs) == 0 { + if len(b.Metrics) > 0 { + metric := simplejson.NewFromAny(b.Metrics[0]) + if metric.Get("type").MustString("") == "raw_document" { + return res, errors.New("alert not support Raw_Document") + } + } + } + aggs, err := b.parseAggs(b.BucketAggs, b.Metrics) + res.Aggs = aggs["aggs"].(Aggs) + + return res, err +} + +func (b *QueryBuilder) parseAggs(bucketAggs []interface{}, metrics []interface{}) (Aggs, error) { + query := make(Aggs) + nestedAggs := query + for _, aggRaw := range bucketAggs { + esAggs := make(Aggs) + aggJson := simplejson.NewFromAny(aggRaw) + aggType, err := aggJson.Get("type").String() + if err != nil { + return nil, err + } + id, err := aggJson.Get("id").String() + if err != nil { + return nil, err + } + + switch aggType { + case "date_histogram": + esAggs["date_histogram"] = b.getDateHistogramAgg(aggJson) + case "histogram": + esAggs["histogram"] = b.getHistogramAgg(aggJson) + case "filters": + esAggs["filters"] = b.getFilters(aggJson) + case "terms": + terms := b.getTerms(aggJson) + esAggs["terms"] = terms.Terms + esAggs["aggs"] = terms.Aggs + case "geohash_grid": + return nil, errors.New("alert not support Geo_Hash_Grid") + } + + if _, ok := nestedAggs["aggs"]; !ok { + nestedAggs["aggs"] = make(Aggs) + } + + if aggs, ok := (nestedAggs["aggs"]).(Aggs); ok { + aggs[id] = esAggs + } + nestedAggs = esAggs + + } + nestedAggs["aggs"] = make(Aggs) + + for _, metricRaw := range metrics { + metric := make(Metric) + metricJson := simplejson.NewFromAny(metricRaw) + + id, err := metricJson.Get("id").String() + if err != nil { + return nil, err + } + metricType, err := metricJson.Get("type").String() + if err != nil { + return nil, err + } + if metricType == "count" { + continue + } + + // todo support pipeline Agg + + settings := metricJson.Get("settings").MustMap() + settings["field"] = metricJson.Get("field").MustString() + metric[metricType] = settings + nestedAggs["aggs"].(Aggs)[id] = metric + } + return query, nil +} + +func (b *QueryBuilder) getDateHistogramAgg(model *simplejson.Json) DateHistogramAgg { + agg := &DateHistogramAgg{} + settings := simplejson.NewFromAny(model.Get("settings").Interface()) + interval, err := settings.Get("interval").String() + if err == nil { + agg.Interval = interval + } + agg.Field = b.TimeField + agg.MinDocCount = settings.Get("min_doc_count").MustInt(0) + agg.ExtendedBounds = ExtendedBounds{"$timeFrom", "$timeTo"} + agg.Format = "epoch_millis" + + if agg.Interval == "auto" { + agg.Interval = "$__interval" + } + + missing, err := settings.Get("missing").String() + if err == nil { + agg.Missing = missing + } + return *agg +} + +func (b *QueryBuilder) getHistogramAgg(model *simplejson.Json) HistogramAgg { + agg := &HistogramAgg{} + settings := simplejson.NewFromAny(model.Get("settings").Interface()) + interval, err := settings.Get("interval").String() + if err == nil { + agg.Interval = interval + } + field, err := model.Get("field").String() + if err == nil { + agg.Field = field + } + agg.MinDocCount = settings.Get("min_doc_count").MustInt(0) + missing, err := settings.Get("missing").String() + if err == nil { + agg.Missing = missing + } + return *agg +} + +func (b *QueryBuilder) getFilters(model *simplejson.Json) FiltersAgg { + agg := &FiltersAgg{} + settings := simplejson.NewFromAny(model.Get("settings").Interface()) + for filter := range settings.Get("filters").MustArray() { + filterJson := simplejson.NewFromAny(filter) + query := filterJson.Get("query").MustString("") + label := filterJson.Get("label").MustString("") + if label == "" { + label = query + } + agg.Filter[label] = newQueryStringFilter(true, query) + } + return *agg +} + +func (b *QueryBuilder) getTerms(model *simplejson.Json) TermsAgg { + agg := &TermsAgg{} + settings := simplejson.NewFromAny(model.Get("settings").Interface()) + agg.Terms.Field = model.Get("field").MustString() + if settings == nil { + return *agg + } + agg.Terms.Size = settings.Get("size").MustInt(0) + if agg.Terms.Size == 0 { + agg.Terms.Size = 500 + } + orderBy := settings.Get("orderBy").MustString("") + if orderBy != "" { + agg.Terms.Order[orderBy] = settings.Get("order").MustString("") + // if orderBy is a int, means this fields is metric result value + // TODO set subAggs + } + + minDocCount, err := settings.Get("min_doc_count").Int() + if err == nil { + agg.Terms.MinDocCount = minDocCount + } + + missing, err := settings.Get("missing").String() + if err == nil { + agg.Terms.Missing = missing + } + + return *agg +} diff --git a/pkg/tsdb/elasticsearch/response_parser.go b/pkg/tsdb/elasticsearch/response_parser.go new file mode 100644 index 00000000000..bc47a3f935e --- /dev/null +++ b/pkg/tsdb/elasticsearch/response_parser.go @@ -0,0 +1,111 @@ +package elasticsearch + +import ( + "errors" + "fmt" + "github.com/grafana/grafana/pkg/components/null" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/tsdb" + "strconv" +) + +type ElasticsearchResponseParser struct { + Responses []Response + Targets []QueryBuilder +} + +func (rp *ElasticsearchResponseParser) getTimeSeries() []interface{} { + for i, res := range rp.Responses { + var series []interface{} + target := rp.Targets[i] + props := make(map[string]interface{}) + rp.processBuckets(res.Aggregations, target, &series, props, 0) + } +} + +func findAgg(target QueryBuilder, aggId string) (*simplejson.Json, error) { + for _, v := range target.BucketAggs { + aggDef := simplejson.NewFromAny(v) + if aggId == aggDef.Get("id").MustString() { + return aggDef, nil + } + } + return nil, errors.New("can't found aggDef, aggID:" + aggId) +} + +func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{}, target QueryBuilder, series *[]interface{}, props map[string]interface{}, depth int) error { + maxDepth := len(target.BucketAggs) - 1 + for aggId, v := range aggs { + aggDef, _ := findAgg(target, aggId) + esAgg := simplejson.NewFromAny(v) + if aggDef == nil { + continue + } + + if depth == maxDepth { + if aggDef.Get("type").MustString() == "date_histogram" { + rp.processMetrics(esAgg, target, series, props) + } + } + + } + +} + +func mapCopy(originalMap, newMap *map[string]string) { + for k, v := range originalMap { + newMap[k] = v + } + +} + +func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, target QueryBuilder, props map[string]string) ([]*tsdb.TimeSeries, error) { + var series []*tsdb.TimeSeries + for _, v := range target.Metrics { + metric := simplejson.NewFromAny(v) + if metric.Get("hide").MustBool(false) { + continue + } + metricId := fmt.Sprintf("%d", metric.Get("id").MustInt()) + metricField := metric.Get("field").MustString() + + switch metric.Get("type").MustString() { + case "count": + newSeries := tsdb.TimeSeries{} + for _, v := range esAgg.Get("buckets").MustMap() { + bucket := simplejson.NewFromAny(v) + value := bucket.Get("doc_count").MustFloat64() + key := bucket.Get("key").MustFloat64() + newSeries.Points = append(newSeries.Points, tsdb.TimePoint{null.FloatFromPtr(&value), null.FloatFromPtr(&key)}) + } + newSeries.Tags = props + newSeries.Tags["metric"] = "count" + series = append(series, &newSeries) + + case "percentiles": + buckets := esAgg.Get("buckets").MustArray() + if len(buckets) == 0 { + break + } + + firstBucket := simplejson.NewFromAny(buckets[0]) + percentiles := firstBucket.GetPath(metricId, "values").MustMap() + + for percentileName := range percentiles { + newSeries := tsdb.TimeSeries{} + newSeries.Tags = props + newSeries.Tags["metric"] = "p" + percentileName + newSeries.Tags["field"] = metricField + for _, v := range buckets { + bucket := simplejson.NewFromAny(v) + valueStr := bucket.GetPath(metricId, "values", percentileName).MustString() + value, _ := strconv.ParseFloat(valueStr, 64) + key := bucket.Get("key").MustFloat64() + newSeries.Points = append(newSeries.Points, tsdb.TimePoint{null.FloatFromPtr(&value), null.FloatFromPtr(&key)}) + } + series = append(series, &newSeries) + } + } + } + return series +} diff --git a/public/app/plugins/datasource/elasticsearch/plugin.json b/public/app/plugins/datasource/elasticsearch/plugin.json index 59d26b785ac..89cca1251d5 100644 --- a/public/app/plugins/datasource/elasticsearch/plugin.json +++ b/public/app/plugins/datasource/elasticsearch/plugin.json @@ -20,6 +20,7 @@ "version": "5.0.0" }, + "alerting": true, "annotations": true, "metrics": true, From bc5b59737c2f6f99b64b395de2e20b888d043c97 Mon Sep 17 00:00:00 2001 From: wph95 Date: Sat, 24 Mar 2018 13:06:21 +0800 Subject: [PATCH 02/87] finished CODING PHASE 1 Signed-off-by: wph95 --- pkg/tsdb/elasticsearch/elasticsearch.go | 13 ++++--------- pkg/tsdb/elasticsearch/model_parser.go | 16 ++++++++-------- pkg/tsdb/elasticsearch/models.go | 11 ----------- 3 files changed, 12 insertions(+), 28 deletions(-) diff --git a/pkg/tsdb/elasticsearch/elasticsearch.go b/pkg/tsdb/elasticsearch/elasticsearch.go index d67b4ad902d..8fd82a179e8 100644 --- a/pkg/tsdb/elasticsearch/elasticsearch.go +++ b/pkg/tsdb/elasticsearch/elasticsearch.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/davecgh/go-spew/spew" "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/setting" @@ -52,13 +51,9 @@ func (e *ElasticsearchExecutor) Query(ctx context.Context, dsInfo *models.DataSo dsInfo, tsdbQuery.TimeRange, tsdbQuery.Queries, - glog, } - glog.Warn(spew.Sdump(dsInfo)) - glog.Warn(spew.Sdump(tsdbQuery)) - - payload, err := queryParser.Parse() + payload, targets, err := queryParser.Parse() if err != nil { return nil, err } @@ -96,14 +91,14 @@ func (e *ElasticsearchExecutor) Query(ctx context.Context, dsInfo *models.DataSo return nil, err } - glog.Warn(spew.Sdump(responses)) for _, res := range responses.Responses { if res.Err != nil { return nil, errors.New(res.getErrMsg()) } - } - + responseParser := ElasticsearchResponseParser{responses.Responses, targets} + queryRes := responseParser.getTimeSeries() + result.Results["A"] = queryRes return result, nil } diff --git a/pkg/tsdb/elasticsearch/model_parser.go b/pkg/tsdb/elasticsearch/model_parser.go index 136db6baed7..233a35efdc6 100644 --- a/pkg/tsdb/elasticsearch/model_parser.go +++ b/pkg/tsdb/elasticsearch/model_parser.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/tsdb" "src/github.com/davecgh/go-spew/spew" @@ -18,38 +17,39 @@ type ElasticSearchQueryParser struct { DsInfo *models.DataSource TimeRange *tsdb.TimeRange Queries []*tsdb.Query - glog log.Logger } -func (qp *ElasticSearchQueryParser) Parse() (string, error) { +func (qp *ElasticSearchQueryParser) Parse() (string, []*QueryBuilder, error) { payload := bytes.Buffer{} queryHeader := qp.getQueryHeader() - + targets := make([]*QueryBuilder, 0) for _, q := range qp.Queries { timeField, err := q.Model.Get("timeField").String() if err != nil { - return "", err + return "", nil, err } rawQuery := q.Model.Get("query").MustString("") bucketAggs := q.Model.Get("bucketAggs").MustArray() metrics := q.Model.Get("metrics").MustArray() alias := q.Model.Get("alias").MustString("") builder := QueryBuilder{timeField, rawQuery, bucketAggs, metrics, alias} + targets = append(targets, &builder) query, err := builder.Build() if err != nil { - return "", err + return "", nil, err } queryBytes, err := json.Marshal(query) if err != nil { - return "", err + return "", nil, err } payload.WriteString(queryHeader.String() + "\n") payload.WriteString(string(queryBytes) + "\n") } + p, err := qp.payloadReplace(payload.String(), qp.DsInfo.JsonData) - return qp.payloadReplace(payload.String(), qp.DsInfo.JsonData) + return p, targets, err } diff --git a/pkg/tsdb/elasticsearch/models.go b/pkg/tsdb/elasticsearch/models.go index 8662f6efbd3..d758e2159de 100644 --- a/pkg/tsdb/elasticsearch/models.go +++ b/pkg/tsdb/elasticsearch/models.go @@ -118,14 +118,3 @@ func (r *Response) getErrMsg() (string) { } return msg.String() } - -type PercentilesResult struct { - Buckets struct { - map[string]struct { - Values map[string]string `json:"values"` - } - KeyAsString string `json:"key_as_string"` - Key int64 `json:"key"` - DocCount int `json:"doc_count"` - } `json:"buckets"` -} From 1e275d0cd1ff976f44dfce6affe8661160cdd873 Mon Sep 17 00:00:00 2001 From: wph95 Date: Sun, 25 Mar 2018 02:18:28 +0800 Subject: [PATCH 03/87] set right series name Signed-off-by: wph95 --- pkg/tsdb/elasticsearch/query.go | 14 +- pkg/tsdb/elasticsearch/query_def.go | 26 +++ pkg/tsdb/elasticsearch/response_parser.go | 219 ++++++++++++++++++---- 3 files changed, 215 insertions(+), 44 deletions(-) create mode 100644 pkg/tsdb/elasticsearch/query_def.go diff --git a/pkg/tsdb/elasticsearch/query.go b/pkg/tsdb/elasticsearch/query.go index 69dd5caa3b4..d6d70e79a2a 100644 --- a/pkg/tsdb/elasticsearch/query.go +++ b/pkg/tsdb/elasticsearch/query.go @@ -3,10 +3,11 @@ package elasticsearch import ( "errors" "github.com/grafana/grafana/pkg/components/simplejson" + "strconv" ) var rangeFilterSetting = RangeFilterSetting{Gte: "$timeFrom", - Lte: "$timeTo", + Lte: "$timeTo", Format: "epoch_millis"} type QueryBuilder struct { @@ -173,18 +174,21 @@ func (b *QueryBuilder) getFilters(model *simplejson.Json) FiltersAgg { } func (b *QueryBuilder) getTerms(model *simplejson.Json) TermsAgg { - agg := &TermsAgg{} + agg := &TermsAgg{Aggs: make(Aggs)} settings := simplejson.NewFromAny(model.Get("settings").Interface()) agg.Terms.Field = model.Get("field").MustString() if settings == nil { return *agg } - agg.Terms.Size = settings.Get("size").MustInt(0) - if agg.Terms.Size == 0 { - agg.Terms.Size = 500 + sizeStr := settings.Get("size").MustString("") + size, err := strconv.Atoi(sizeStr) + if err != nil { + size = 500 } + agg.Terms.Size = size orderBy := settings.Get("orderBy").MustString("") if orderBy != "" { + agg.Terms.Order = make(map[string]interface{}) agg.Terms.Order[orderBy] = settings.Get("order").MustString("") // if orderBy is a int, means this fields is metric result value // TODO set subAggs diff --git a/pkg/tsdb/elasticsearch/query_def.go b/pkg/tsdb/elasticsearch/query_def.go new file mode 100644 index 00000000000..5dc02aa359e --- /dev/null +++ b/pkg/tsdb/elasticsearch/query_def.go @@ -0,0 +1,26 @@ +package elasticsearch + +var metricAggType = map[string]string{ + "count": "Count", + "avg": "Average", + "sum": "Sum", + "max": "Max", + "min": "Min", + "extended_stats": "Extended Stats", + "percentiles": "Percentiles", + "cardinality": "Unique Count", + "moving_avg": "Moving Average", + "derivative": "Derivative", + "raw_document": "Raw Document", +} + +var extendedStats = map[string]string{ + "avg": "Avg", + "min": "Min", + "max": "Max", + "sum": "Sum", + "count": "Count", + "std_deviation": "Std Dev", + "std_deviation_bounds_upper": "Std Dev Upper", + "std_deviation_bounds_lower": "Std Dev Lower", +} diff --git a/pkg/tsdb/elasticsearch/response_parser.go b/pkg/tsdb/elasticsearch/response_parser.go index bc47a3f935e..a2a8565641f 100644 --- a/pkg/tsdb/elasticsearch/response_parser.go +++ b/pkg/tsdb/elasticsearch/response_parser.go @@ -7,33 +7,30 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/tsdb" "strconv" + "regexp" + "strings" ) type ElasticsearchResponseParser struct { Responses []Response - Targets []QueryBuilder + Targets []*QueryBuilder } -func (rp *ElasticsearchResponseParser) getTimeSeries() []interface{} { +func (rp *ElasticsearchResponseParser) getTimeSeries() *tsdb.QueryResult { + queryRes := tsdb.NewQueryResult() for i, res := range rp.Responses { - var series []interface{} target := rp.Targets[i] - props := make(map[string]interface{}) + props := make(map[string]string) + series := make([]*tsdb.TimeSeries, 0) rp.processBuckets(res.Aggregations, target, &series, props, 0) + rp.nameSeries(&series, target) + queryRes.Series = append(queryRes.Series, series...) } + return queryRes } -func findAgg(target QueryBuilder, aggId string) (*simplejson.Json, error) { - for _, v := range target.BucketAggs { - aggDef := simplejson.NewFromAny(v) - if aggId == aggDef.Get("id").MustString() { - return aggDef, nil - } - } - return nil, errors.New("can't found aggDef, aggID:" + aggId) -} - -func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{}, target QueryBuilder, series *[]interface{}, props map[string]interface{}, depth int) error { +func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{}, target *QueryBuilder, series *[]*tsdb.TimeSeries, props map[string]string, depth int) (error) { + var err error maxDepth := len(target.BucketAggs) - 1 for aggId, v := range aggs { aggDef, _ := findAgg(target, aggId) @@ -44,43 +41,59 @@ func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{ if depth == maxDepth { if aggDef.Get("type").MustString() == "date_histogram" { - rp.processMetrics(esAgg, target, series, props) + err = rp.processMetrics(esAgg, target, series, props) + if err != nil { + return err + } + } else { + return fmt.Errorf("not support type:%s", aggDef.Get("type").MustString()) + } + } else { + for i, b := range esAgg.Get("buckets").MustArray() { + field := aggDef.Get("field").MustString() + bucket := simplejson.NewFromAny(b) + newProps := props + if key, err := bucket.Get("key").String(); err == nil { + newProps[field] = key + } else { + props["filter"] = strconv.Itoa(i) + } + + if key, err := bucket.Get("key_as_string").String(); err == nil { + props[field] = key + } + rp.processBuckets(bucket.MustMap(), target, series, newProps, depth+1) } } } + return nil } -func mapCopy(originalMap, newMap *map[string]string) { - for k, v := range originalMap { - newMap[k] = v - } - -} - -func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, target QueryBuilder, props map[string]string) ([]*tsdb.TimeSeries, error) { - var series []*tsdb.TimeSeries +func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, target *QueryBuilder, series *[]*tsdb.TimeSeries, props map[string]string) (error) { for _, v := range target.Metrics { metric := simplejson.NewFromAny(v) if metric.Get("hide").MustBool(false) { continue } - metricId := fmt.Sprintf("%d", metric.Get("id").MustInt()) - metricField := metric.Get("field").MustString() - switch metric.Get("type").MustString() { + metricId := metric.Get("id").MustString() + metricField := metric.Get("field").MustString() + metricType := metric.Get("type").MustString() + + switch metricType { case "count": newSeries := tsdb.TimeSeries{} - for _, v := range esAgg.Get("buckets").MustMap() { + for _, v := range esAgg.Get("buckets").MustArray() { bucket := simplejson.NewFromAny(v) - value := bucket.Get("doc_count").MustFloat64() - key := bucket.Get("key").MustFloat64() - newSeries.Points = append(newSeries.Points, tsdb.TimePoint{null.FloatFromPtr(&value), null.FloatFromPtr(&key)}) + value := castToNullFloat(bucket.Get("doc_count")) + key := castToNullFloat(bucket.Get("key")) + newSeries.Points = append(newSeries.Points, tsdb.TimePoint{value, key}) } newSeries.Tags = props newSeries.Tags["metric"] = "count" - series = append(series, &newSeries) + *series = append(*series, &newSeries) case "percentiles": buckets := esAgg.Get("buckets").MustArray() @@ -98,14 +111,142 @@ func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, ta newSeries.Tags["field"] = metricField for _, v := range buckets { bucket := simplejson.NewFromAny(v) - valueStr := bucket.GetPath(metricId, "values", percentileName).MustString() - value, _ := strconv.ParseFloat(valueStr, 64) - key := bucket.Get("key").MustFloat64() - newSeries.Points = append(newSeries.Points, tsdb.TimePoint{null.FloatFromPtr(&value), null.FloatFromPtr(&key)}) + value := castToNullFloat(bucket.GetPath(metricId, "values", percentileName)) + key := castToNullFloat(bucket.Get("key")) + newSeries.Points = append(newSeries.Points, tsdb.TimePoint{value, key}) } - series = append(series, &newSeries) + *series = append(*series, &newSeries) + } + default: + newSeries := tsdb.TimeSeries{} + newSeries.Tags = props + newSeries.Tags["metric"] = metricType + newSeries.Tags["field"] = metricField + for _, v := range esAgg.Get("buckets").MustArray() { + bucket := simplejson.NewFromAny(v) + key := castToNullFloat(bucket.Get("key")) + valueObj, err := bucket.Get(metricId).Map() + if err != nil { + break + } + var value null.Float + if _, ok := valueObj["normalized_value"]; ok { + value = castToNullFloat(bucket.GetPath(metricId, "normalized_value")) + } else { + value = castToNullFloat(bucket.GetPath(metricId, "value")) + } + newSeries.Points = append(newSeries.Points, tsdb.TimePoint{value, key}) + } + *series = append(*series, &newSeries) + } + } + return nil +} + +func (rp *ElasticsearchResponseParser) nameSeries(seriesList *[]*tsdb.TimeSeries, target *QueryBuilder) { + set := make(map[string]string) + for _, v := range *seriesList { + if metricType, exists := v.Tags["metric"]; exists { + if _, ok := set[metricType]; !ok { + set[metricType] = "" } } } - return series + metricTypeCount := len(set) + for _, series := range *seriesList { + series.Name = rp.getSeriesName(series, target, metricTypeCount) + } + +} + +func (rp *ElasticsearchResponseParser) getSeriesName(series *tsdb.TimeSeries, target *QueryBuilder, metricTypeCount int) (string) { + metricName := rp.getMetricName(series.Tags["metric"]) + delete(series.Tags, "metric") + + field := "" + if v, ok := series.Tags["field"]; ok { + field = v + delete(series.Tags, "field") + } + + if target.Alias != "" { + var re = regexp.MustCompile(`{{([\s\S]+?)}}`) + for _, match := range re.FindAllString(target.Alias, -1) { + group := match[2:len(match)-2] + + if strings.HasPrefix(group, "term ") { + if term, ok := series.Tags["term "]; ok { + strings.Replace(target.Alias, match, term, 1) + } + } + if v, ok := series.Tags[group]; ok { + strings.Replace(target.Alias, match, v, 1) + } + + switch group { + case "metric": + strings.Replace(target.Alias, match, metricName, 1) + case "field": + strings.Replace(target.Alias, match, field, 1) + } + + } + } + // todo, if field and pipelineAgg + if field != "" { + metricName += " " + field + } + + if len(series.Tags) == 0 { + return metricName + } + + name := "" + for _, v := range series.Tags { + name += v + " " + } + + if metricTypeCount == 1 { + return strings.TrimSpace(name) + } + + return strings.TrimSpace(name) + " " + metricName + +} + +func (rp *ElasticsearchResponseParser) getMetricName(metric string) string { + if text, ok := metricAggType[metric]; ok { + return text + } + + if text, ok := extendedStats[metric]; ok { + return text + } + + return metric +} + +func castToNullFloat(j *simplejson.Json) null.Float { + f, err := j.Float64() + if err == nil { + return null.FloatFrom(f) + } + + s, err := j.String() + if err == nil { + v, _ := strconv.ParseFloat(s, 64) + return null.FloatFromPtr(&v) + } + + return null.NewFloat(0, false) +} + +func findAgg(target *QueryBuilder, aggId string) (*simplejson.Json, error) { + for _, v := range target.BucketAggs { + aggDef := simplejson.NewFromAny(v) + if aggId == aggDef.Get("id").MustString() { + return aggDef, nil + } + } + return nil, errors.New("can't found aggDef, aggID:" + aggId) } From d6cdc2497c929039f93830dd8b7a61661046ae57 Mon Sep 17 00:00:00 2001 From: wph95 Date: Mon, 26 Mar 2018 16:13:14 +0800 Subject: [PATCH 04/87] Handle Interval Date Format similar to the JS variant https://github.com/grafana/grafana/pull/10343/commits/7e14e272fa37df5b4d412c16845d1e525711f726 --- Gopkg.lock | 8 +- Gopkg.toml | 4 + pkg/tsdb/elasticsearch/model_parser.go | 46 +- pkg/tsdb/elasticsearch/model_parser_test.go | 49 + vendor/github.com/leibowitz/moment/diff.go | 75 ++ vendor/github.com/leibowitz/moment/moment.go | 1185 +++++++++++++++++ .../leibowitz/moment/moment_parser.go | 100 ++ .../github.com/leibowitz/moment/parse_day.go | 32 + .../leibowitz/moment/strftime_parser.go | 68 + 9 files changed, 1559 insertions(+), 8 deletions(-) create mode 100644 pkg/tsdb/elasticsearch/model_parser_test.go create mode 100644 vendor/github.com/leibowitz/moment/diff.go create mode 100644 vendor/github.com/leibowitz/moment/moment.go create mode 100644 vendor/github.com/leibowitz/moment/moment_parser.go create mode 100644 vendor/github.com/leibowitz/moment/parse_day.go create mode 100644 vendor/github.com/leibowitz/moment/strftime_parser.go diff --git a/Gopkg.lock b/Gopkg.lock index ebadad8331b..78316b77664 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -295,6 +295,12 @@ packages = ["."] revision = "7cafcd837844e784b526369c9bce262804aebc60" +[[projects]] + branch = "master" + name = "github.com/leibowitz/moment" + packages = ["."] + revision = "8548108dcca204a1110b99e5fec966817499fe84" + [[projects]] branch = "master" name = "github.com/lib/pq" @@ -642,6 +648,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "5e65aeace832f1b4be17e7ff5d5714513c40f31b94b885f64f98f2332968d7c6" + inputs-digest = "9895ff7b1516b9639d0fc280ca155c8958486656a2086fc45e91f727fccea0d2" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index df163e01ed3..1f8cbba6e11 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -201,3 +201,7 @@ ignored = [ [[constraint]] name = "github.com/denisenkom/go-mssqldb" revision = "270bc3860bb94dd3a3ffd047377d746c5e276726" + +[[constraint]] + branch = "master" + name = "github.com/leibowitz/moment" diff --git a/pkg/tsdb/elasticsearch/model_parser.go b/pkg/tsdb/elasticsearch/model_parser.go index 233a35efdc6..7da6765e06c 100644 --- a/pkg/tsdb/elasticsearch/model_parser.go +++ b/pkg/tsdb/elasticsearch/model_parser.go @@ -7,6 +7,7 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/tsdb" + "github.com/leibowitz/moment" "src/github.com/davecgh/go-spew/spew" "strconv" "strings" @@ -63,7 +64,7 @@ func (qp *ElasticSearchQueryParser) getQueryHeader() *QueryHeader { } header.SearchType = searchType header.IgnoreUnavailable = true - header.Index = qp.getIndexList() + header.Index = getIndexList(qp.DsInfo.Database, qp.DsInfo.JsonData.Get("interval").MustString(""), qp.TimeRange) if esVersion >= 56 { header.MaxConcurrentShardRequests = qp.DsInfo.JsonData.Get("maxConcurrentShardRequests").MustInt() @@ -87,11 +88,42 @@ func (qp *ElasticSearchQueryParser) payloadReplace(payload string, model *simple return payload, nil } -func (qp *ElasticSearchQueryParser) getIndexList() string { - _, err := qp.DsInfo.JsonData.Get("interval").String() - if err != nil { - return qp.DsInfo.Database +func getIndexList(pattern string, interval string, timeRange *tsdb.TimeRange) string { + if interval == "" { + return pattern } - // todo: support interval - return qp.DsInfo.Database + + var indexes []string + indexParts := strings.Split(strings.TrimLeft(pattern, "["), "]") + indexBase := indexParts[0] + if len(indexParts) <= 1 { + return pattern + } + + indexDateFormat := indexParts[1] + + start := moment.NewMoment(timeRange.MustGetFrom()) + end := moment.NewMoment(timeRange.MustGetTo()) + + indexes = append(indexes, fmt.Sprintf("%s%s", indexBase, start.Format(indexDateFormat))) + for start.IsBefore(*end) { + switch interval { + case "Hourly": + start = start.AddHours(1) + + case "Daily": + start = start.AddDay() + + case "Weekly": + start = start.AddWeeks(1) + + case "Monthly": + start = start.AddMonths(1) + + case "Yearly": + start = start.AddYears(1) + } + indexes = append(indexes, fmt.Sprintf("%s%s", indexBase, start.Format(indexDateFormat))) + } + return strings.Join(indexes, ",") } diff --git a/pkg/tsdb/elasticsearch/model_parser_test.go b/pkg/tsdb/elasticsearch/model_parser_test.go new file mode 100644 index 00000000000..aa7336fb69b --- /dev/null +++ b/pkg/tsdb/elasticsearch/model_parser_test.go @@ -0,0 +1,49 @@ +package elasticsearch + +import ( + "github.com/grafana/grafana/pkg/tsdb" + . "github.com/smartystreets/goconvey/convey" + "strconv" + "strings" + "testing" +) + +func makeTime(hour int) string { + //unixtime 1500000000 == 2017-07-14T02:40:00+00:00 + return strconv.Itoa((1500000000 + hour*60*60) * 1000) +} + +func getIndexListByTime(pattern string, interval string, hour int) string { + timeRange := &tsdb.TimeRange{ + From: makeTime(0), + To: makeTime(hour), + } + return getIndexList(pattern, interval, timeRange) +} + +func TestElasticsearchGetIndexList(t *testing.T) { + Convey("Test Elasticsearch getIndex ", t, func() { + + Convey("Parse Interval Formats", func() { + So(getIndexListByTime("[logstash-]YYYY.MM.DD", "Daily", 48), + ShouldEqual, "logstash-2017.07.14,logstash-2017.07.15,logstash-2017.07.16") + + So(len(strings.Split(getIndexListByTime("[logstash-]YYYY.MM.DD.HH", "Hourly", 3), ",")), + ShouldEqual, 4) + + So(getIndexListByTime("[logstash-]YYYY.W", "Weekly", 100), + ShouldEqual, "logstash-2017.28,logstash-2017.29") + + So(getIndexListByTime("[logstash-]YYYY.MM", "Monthly", 700), + ShouldEqual, "logstash-2017.07,logstash-2017.08") + + So(getIndexListByTime("[logstash-]YYYY", "Yearly", 10000), + ShouldEqual, "logstash-2017,logstash-2018,logstash-2019") + }) + + Convey("No Interval", func() { + index := getIndexListByTime("logstash-test", "", 1) + So(index, ShouldEqual, "logstash-test") + }) + }) +} diff --git a/vendor/github.com/leibowitz/moment/diff.go b/vendor/github.com/leibowitz/moment/diff.go new file mode 100644 index 00000000000..0d6b3935adf --- /dev/null +++ b/vendor/github.com/leibowitz/moment/diff.go @@ -0,0 +1,75 @@ +package moment + +import ( + "fmt" + "math" + "time" +) + +// @todo In months/years requires the old and new to calculate correctly, right? +// @todo decide how to handle rounding (i.e. always floor?) +type Diff struct { + duration time.Duration +} + +func (d *Diff) InSeconds() int { + return int(d.duration.Seconds()) +} + +func (d *Diff) InMinutes() int { + return int(d.duration.Minutes()) +} + +func (d *Diff) InHours() int { + return int(d.duration.Hours()) +} + +func (d *Diff) InDays() int { + return int(math.Floor(float64(d.InSeconds()) / 86400)) +} + +// This depends on where the weeks fall? +func (d *Diff) InWeeks() int { + return int(math.Floor(float64(d.InDays() / 7))) +} + +func (d *Diff) InMonths() int { + return 0 +} + +func (d *Diff) InYears() int { + return 0 +} + +// http://momentjs.com/docs/#/durations/humanize/ +func (d *Diff) Humanize() string { + diffInSeconds := d.InSeconds() + + if diffInSeconds <= 45 { + return fmt.Sprintf("%d seconds ago", diffInSeconds) + } else if diffInSeconds <= 90 { + return "a minute ago" + } + + diffInMinutes := d.InMinutes() + + if diffInMinutes <= 45 { + return fmt.Sprintf("%d minutes ago", diffInMinutes) + } else if diffInMinutes <= 90 { + return "an hour ago" + } + + diffInHours := d.InHours() + + if diffInHours <= 22 { + return fmt.Sprintf("%d hours ago", diffInHours) + } else if diffInHours <= 36 { + return "a day ago" + } + + return "diff is in days" +} + +// In Months + +// In years diff --git a/vendor/github.com/leibowitz/moment/moment.go b/vendor/github.com/leibowitz/moment/moment.go new file mode 100644 index 00000000000..13c8ef7dbef --- /dev/null +++ b/vendor/github.com/leibowitz/moment/moment.go @@ -0,0 +1,1185 @@ +package moment + +import ( + "fmt" + "regexp" + "strconv" + "strings" + "time" +) + +// links +// http://en.wikipedia.org/wiki/ISO_week_date +// http://golang.org/src/pkg/time/format.go +// http://www.php.net/manual/en/class.datetime.php#datetime.constants.rfc822 +// http://php.net/manual/en/function.date.php +// http://www.php.net/manual/en/datetime.formats.relative.php + +// @todo are these constants needed if they are in the time package? +// There are a lot of extras here, and RFC822 doesn't match up. Why? +// Also, is timezone usage wrong? Double-check +const ( + ATOM = "2006-01-02T15:04:05Z07:00" + COOKIE = "Monday, 02-Jan-06 15:04:05 MST" + ISO8601 = "2006-01-02T15:04:05Z0700" + RFC822 = "Mon, 02 Jan 06 15:04:05 Z0700" + RFC850 = "Monday, 02-Jan-06 15:04:05 MST" + RFC1036 = "Mon, 02 Jan 06 15:04:05 Z0700" + RFC1123 = "Mon, 02 Jan 2006 15:04:05 Z0700" + RFC2822 = "Mon, 02 Jan 2006 15:04:05 Z0700" + RFC3339 = "2006-01-02T15:04:05Z07:00" + RSS = "Mon, 02 Jan 2006 15:04:05 Z0700" + W3C = "2006-01-02T15:04:05Z07:00" +) + +var ( + regex_days = "monday|mon|tuesday|tues|wednesday|wed|thursday|thurs|friday|fri|saturday|sat|sunday|sun" + regex_period = "second|minute|hour|day|week|month|year" + regex_numbers = "one|two|three|four|five|six|seven|eight|nine|ten" +) + +// regexp +var ( + compiled = regexp.MustCompile(`\s{2,}`) + relativeday = regexp.MustCompile(`(yesterday|today|tomorrow)`) + //relative1 = regexp.MustCompile(`(first|last) day of (this|next|last|previous) (week|month|year)`) + //relative2 = regexp.MustCompile(`(first|last) day of (` + "jan|january|feb|february|mar|march|apr|april|may|jun|june|jul|july|aug|august|sep|september|oct|october|nov|november|dec|december" + `)(?:\s(\d{4,4}))?`) + relative3 = regexp.MustCompile(`((?Pthis|next|last|previous) )?(` + regex_days + `)`) + //relativeval = regexp.MustCompile(`([0-9]+) (day|week|month|year)s? ago`) + ago = regexp.MustCompile(`([0-9]+) (` + regex_period + `)s? ago`) + ordinal = regexp.MustCompile("([0-9]+)(st|nd|rd|th)") + written = regexp.MustCompile(regex_numbers) + relativediff = regexp.MustCompile(`([\+\-])?([0-9]+),? ?(` + regex_period + `)s?`) + relativetime = regexp.MustCompile(`(?P\d\d?):(?P\d\d?)(:(?P\d\d?))?\s?(?Pam|pm)?\s?(?P[a-z]{3,3})?|(?Pnoon|midnight)`) + yearmonthday = regexp.MustCompile(`(?P\d{4})-(?P\d{1,2})-(?P\d{1,2})`) + relativeperiod = regexp.MustCompile(`(?Pthis|next|last) (week|month|year)`) + numberRegex = regexp.MustCompile("([0-9]+)(?:)") +) + +// http://golang.org/src/pkg/time/format.go?s=12686:12728#L404 + +// Timezone implementation +// https://groups.google.com/forum/#!topic/golang-nuts/XEVN4QwTvHw +// http://en.wikipedia.org/wiki/Zone.tab + +// Support ISO8601 Duration Parsing? +// http://en.wikipedia.org/wiki/ISO_8601 + +// Differences +// Months are NOT zero-index, MOmentJS they are +// Weeks are 0 indexed +// -- Sunday being the last day of the week ISO-8601 - is that diff from Moment? +// From/FromNow Return a Diff object rather than strings + +// Support for locale and languages with English as default + +// Support for strftime +// https://github.com/benjaminoakes/moment-strftime +// Format: https://php.net/strftime + +type Moment struct { + time time.Time + + Parser +} + +type Parser interface { + Convert(string) string +} + +func New() *Moment { + m := &Moment{time.Now(), new(MomentParser)} + + return m +} + +func NewMoment(t time.Time) *Moment { + m := &Moment{t, new(MomentParser)} + + return m +} + +func (m *Moment) GetTime() time.Time { + return m.time +} + +func (m *Moment) Now() *Moment { + m.time = time.Now().In(m.GetTime().Location()) + + return m +} + +func (m *Moment) Moment(layout string, datetime string) *Moment { + return m.MomentGo(m.Convert(layout), datetime) +} + +func (m *Moment) MomentGo(layout string, datetime string) *Moment { + time, _ := time.Parse(layout, datetime) + + m.time = time + + return m +} + +// This method is nowhere near done - requires lots of work. +func (m *Moment) Strtotime(str string) *Moment { + str = strings.ToLower(strings.TrimSpace(str)) + str = compiled.ReplaceAllString(str, " ") + + // Replace written numbers (i.e. nine, ten) with actual numbers (9, 10) + str = written.ReplaceAllStringFunc(str, func(n string) string { + switch n { + case "one": + return "1" + case "two": + return "2" + case "three": + return "3" + case "four": + return "4" + case "five": + return "5" + case "six": + return "6" + case "seven": + return "7" + case "eight": + return "8" + case "nine": + return "9" + case "ten": + return "10" + } + + return "" + }) + + // Remove ordinal suffixes st, nd, rd, th + str = ordinal.ReplaceAllString(str, "$1") + + // Replace n second|minute|hour... ago to -n second|minute|hour... to consolidate parsing + str = ago.ReplaceAllString(str, "-$1 $2") + + // Look for relative +1day, +3 days 5 hours 15 minutes + if match := relativediff.FindAllStringSubmatch(str, -1); match != nil { + for i := range match { + switch match[i][1] { + case "-": + number, _ := strconv.Atoi(match[i][2]) + m.Subtract(match[i][3], number) + default: + number, _ := strconv.Atoi(match[i][2]) + m.Add(match[i][3], number) + } + + str = strings.Replace(str, match[i][0], "", 1) + } + } + + // Remove any words that aren't needed for consistency + str = strings.Replace(str, " at ", " ", -1) + str = strings.Replace(str, " on ", " ", -1) + + // Support for interchangeable previous/last + str = strings.Replace(str, "previous", "last", -1) + + var dateDefaults = map[string]int{ + "year": 0, + "month": 0, + "day": 0, + } + + dateMatches := dateDefaults + if match := yearmonthday.FindStringSubmatch(str); match != nil { + for i, name := range yearmonthday.SubexpNames() { + if i == 0 { + str = strings.Replace(str, match[i], "", 1) + continue + } + + if match[i] == "" { + continue + } + + if name == "year" || name == "month" || name == "day" { + dateMatches[name], _ = strconv.Atoi(match[i]) + } + + } + + defer m.strtotimeSetDate(dateMatches) + if str == "" { + // Nothing left to parse + return m + } + + str = strings.TrimSpace(str) + } + + // Try to parse out time from the string + var timeDefaults = map[string]int{ + "hour": 0, + "minutes": 0, + "seconds": 0, + } + + timeMatches := timeDefaults + var zone string + if match := relativetime.FindStringSubmatch(str); match != nil { + for i, name := range relativetime.SubexpNames() { + if i == 0 { + str = strings.Replace(str, match[i], "", 1) + continue + } + + if match[i] == "" { + continue + } + + // Midnight is all zero's so nothing to do + if name == "relativetime" && match[i] == "noon" { + timeDefaults["hour"] = 12 + } + + if name == "zone" { + zone = match[i] + } + + if name == "meridiem" && match[i] == "pm" && timeMatches["hour"] < 12 { + timeMatches["hour"] += 12 + } + + if name == "hour" || name == "minutes" || name == "seconds" { + timeMatches[name], _ = strconv.Atoi(match[i]) + } + } + + // Processing time is always last + defer m.strtotimeSetTime(timeMatches, zone) + + if str == "" { + // Nothing left to parse + return m + } + + str = strings.TrimSpace(str) + } + + // m.StartOf("month", "January").GoTo(time.Sunday) + + if match := relativeperiod.FindStringSubmatch(str); match != nil { + period := match[1] + unit := match[2] + + str = strings.Replace(str, match[0], "", 1) + + switch period { + case "next": + if unit == "year" { + m.AddYears(1) + } + if unit == "month" { + m.AddMonths(1) + } + if unit == "week" { + m.AddWeeks(1) + } + case "last": + if unit == "year" { + m.SubYears(1) + } + if unit == "month" { + m.SubMonths(1) + } + if unit == "week" { + m.SubWeeks(1) + } + } + + str = strings.TrimSpace(str) + + // first := regexp.MustCompile("(?Pfirst|last)?") + } + + /* + + relativeday: first day of + relativeperiod: this, last, next + relativeperiodunit week, month, year + day: monday, tues, wednesday + month: january, feb + + + YYYY-MM-DD (HH:MM:SS MST)? + MM-DD-YYYY (HH:MM:SS MST) + 10 September 2015 (HH:MM:SS MST)? + September, 10 2015 (HH:MM:SS MST)? + September 10 2015 (HH:MM:SS M + + this year 2014 + next year 2015 + last year 2013 + + this month April + next month May + last month Mar + + first day of April + last day of April + + + DONE 3PM + DONE 3:00 PM + DONE 3:00:05 MST + 3PM on January 5th + January 5th at 3:00PM + first saturday _of_ next month + first saturday _of_ next month _at_ 3:00PM + saturday of next week + saturday of last week + saturday next week + monday next week + saturday of this week + saturday at 3:00pm + saturday at 4:00PM + saturday at midn + first of january + last of january + january of next year + first day of january + last day of january + first day of February + + DONE midnight + DONE noon + DONE 3 days ago + DONE ten days + DONE 9 weeks ago // Convert to -9 weeks + DONE -9 weeks + + */ + + if match := relativeday.FindStringSubmatch(str); match != nil && len(match) > 1 { + day := match[1] + + str = strings.Replace(str, match[0], "", 1) + + switch day { + case "today": + m.Today() + case "yesterday": + m.Yesterday() + case "tomorrow": + m.Tomorrow() + } + } + + if match := relative3.FindStringSubmatch(str); match != nil { + var when string + for i, name := range relative3.SubexpNames() { + if name == "relperiod" { + when = match[i] + } + } + weekDay := match[len(match)-1] + + str = strings.Replace(str, match[0], "", 1) + + wDay, err := ParseWeekDay(weekDay) + if err == nil { + switch when { + case "last", "previous": + m.GoBackTo(wDay, true) + + case "next": + m.GoTo(wDay, true) + + case "", "this": + m.GoTo(wDay, false) + default: + m.GoTo(wDay, false) + } + } + } + + /* + + + yesterday 11:00 + today 11:00 + tomorrow 11:00 + midnight + noon + DONE +n (second|day|week|month|year)s? + DONE -n (second|day|week|month|year)s? + next (monday|tuesday|wednesday|thursday|friday|saturday|sunday) 11:00 + last (monday|tuesday|wednesday|thursday|friday|saturday|sunday) 11:00 + next (month|year) + last (month|year) + first day of (january|february|march...|december) 2014 + last day of (january|february|march...|december) 2014 + first day of (this|next|last) (week|month|year) + last day of (this|next|last) (week|month|year) + first (monday|tuesday|wednesday) of July 2014 + last (monday|tuesday|wednesday) of July 2014 + n (day|week|month|year)s? ago + Monday|Tuesday|Wednesday|Thursday|Friday + Monday (last|this|next) week + + DONE +1 week 2 days 3 hours 4 minutes 5 seconds + */ + + return m +} + +// @todo deal with timezone +func (m *Moment) strtotimeSetTime(time map[string]int, zone string) { + m.SetHour(time["hour"]).SetMinute(time["minutes"]).SetSecond(time["seconds"]) +} + +func (m *Moment) strtotimeSetDate(date map[string]int) { + m.SetYear(date["year"]).SetMonth(time.Month(date["month"])).SetDay(date["day"]) +} + +func (m Moment) Clone() *Moment { + copy := New() + copy.time = m.GetTime() + + return copy +} + +/** + * Getters + * + */ +// https://groups.google.com/forum/#!topic/golang-nuts/pret7hjDc70 +func (m *Moment) Millisecond() { + +} + +func (m *Moment) Second() int { + return m.GetTime().Second() +} + +func (m *Moment) Minute() int { + return m.GetTime().Minute() +} + +func (m *Moment) Hour() int { + return m.GetTime().Hour() +} + +// Day of month +func (m *Moment) Date() int { + return m.DayOfMonth() +} + +// Carbon convenience method +func (m *Moment) DayOfMonth() int { + return m.GetTime().Day() +} + +// Day of week (int or string) +func (m *Moment) Day() time.Weekday { + return m.DayOfWeek() +} + +// Carbon convenience method +func (m *Moment) DayOfWeek() time.Weekday { + return m.GetTime().Weekday() +} + +func (m *Moment) DayOfWeekISO() int { + day := m.GetTime().Weekday() + + if day == time.Sunday { + return 7 + } + + return int(day) +} + +func (m *Moment) DayOfYear() int { + return m.GetTime().YearDay() +} + +// Day of Year with zero padding +func (m *Moment) dayOfYearZero() string { + day := m.GetTime().YearDay() + + if day < 10 { + return fmt.Sprintf("00%d", day) + } + + if day < 100 { + return fmt.Sprintf("0%d", day) + } + + return fmt.Sprintf("%d", day) +} + +// todo panic? +func (m *Moment) Weekday(index int) string { + if index > 6 { + panic("Weekday index must be between 0 and 6") + } + + return time.Weekday(index).String() +} + +func (m *Moment) Week() int { + return 0 +} + +// Is this the week number where as ISOWeekYear is the number of weeks in the year? +// @see http://stackoverflow.com/questions/18478741/get-weeks-in-year +func (m *Moment) ISOWeek() int { + _, week := m.GetTime().ISOWeek() + + return week +} + +// @todo Consider language support +func (m *Moment) Month() time.Month { + return m.GetTime().Month() +} + +func (m *Moment) Quarter() (quarter int) { + quarter = 4 + + switch m.Month() { + case time.January, time.February, time.March: + quarter = 1 + case time.April, time.May, time.June: + quarter = 2 + case time.July, time.August, time.September: + quarter = 3 + } + + return +} + +func (m *Moment) Year() int { + return m.GetTime().Year() +} + +// @see comments for ISOWeek +func (m *Moment) WeekYear() { + +} + +func (m *Moment) ISOWeekYear() { + +} + +/** + * Manipulate + * + */ +func (m *Moment) Add(key string, value int) *Moment { + switch key { + case "years", "year", "y": + m.AddYears(value) + case "months", "month", "M": + m.AddMonths(value) + case "weeks", "week", "w": + m.AddWeeks(value) + case "days", "day", "d": + m.AddDays(value) + case "hours", "hour", "h": + m.AddHours(value) + case "minutes", "minute", "m": + m.AddMinutes(value) + case "seconds", "second", "s": + m.AddSeconds(value) + case "milliseconds", "millisecond", "ms": + + } + + return m +} + +// Carbon +func (m *Moment) AddSeconds(seconds int) *Moment { + return m.addTime(time.Second * time.Duration(seconds)) +} + +// Carbon +func (m *Moment) AddMinutes(minutes int) *Moment { + return m.addTime(time.Minute * time.Duration(minutes)) +} + +// Carbon +func (m *Moment) AddHours(hours int) *Moment { + return m.addTime(time.Hour * time.Duration(hours)) +} + +// Carbon +func (m *Moment) AddDay() *Moment { + return m.AddDays(1) +} + +// Carbon +func (m *Moment) AddDays(days int) *Moment { + m.time = m.GetTime().AddDate(0, 0, days) + + return m +} + +// Carbon +func (m *Moment) AddWeeks(weeks int) *Moment { + return m.AddDays(weeks * 7) +} + +// Carbon +func (m *Moment) AddMonths(months int) *Moment { + m.time = m.GetTime().AddDate(0, months, 0) + + return m +} + +// Carbon +func (m *Moment) AddYears(years int) *Moment { + m.time = m.GetTime().AddDate(years, 0, 0) + + return m +} + +func (m *Moment) addTime(d time.Duration) *Moment { + m.time = m.GetTime().Add(d) + + return m +} + +func (m *Moment) Subtract(key string, value int) *Moment { + switch key { + case "years", "year", "y": + m.SubYears(value) + case "months", "month", "M": + m.SubMonths(value) + case "weeks", "week", "w": + m.SubWeeks(value) + case "days", "day", "d": + m.SubDays(value) + case "hours", "hour", "h": + m.SubHours(value) + case "minutes", "minute", "m": + m.SubMinutes(value) + case "seconds", "second", "s": + m.SubSeconds(value) + case "milliseconds", "millisecond", "ms": + + } + + return m +} + +// Carbon +func (m *Moment) SubSeconds(seconds int) *Moment { + return m.addTime(time.Second * time.Duration(seconds*-1)) +} + +// Carbon +func (m *Moment) SubMinutes(minutes int) *Moment { + return m.addTime(time.Minute * time.Duration(minutes*-1)) +} + +// Carbon +func (m *Moment) SubHours(hours int) *Moment { + return m.addTime(time.Hour * time.Duration(hours*-1)) +} + +// Carbon +func (m *Moment) SubDay() *Moment { + return m.SubDays(1) +} + +// Carbon +func (m *Moment) SubDays(days int) *Moment { + return m.AddDays(days * -1) +} + +func (m *Moment) SubWeeks(weeks int) *Moment { + return m.SubDays(weeks * 7) +} + +// Carbon +func (m *Moment) SubMonths(months int) *Moment { + return m.AddMonths(months * -1) +} + +// Carbon +func (m *Moment) SubYears(years int) *Moment { + return m.AddYears(years * -1) +} + +// Carbon +func (m *Moment) Today() *Moment { + return m.Now() +} + +// Carbon +func (m *Moment) Tomorrow() *Moment { + return m.Today().AddDay() +} + +// Carbon +func (m *Moment) Yesterday() *Moment { + return m.Today().SubDay() +} + +func (m *Moment) StartOf(key string) *Moment { + switch key { + case "year", "y": + m.StartOfYear() + case "month", "M": + m.StartOfMonth() + case "week", "w": + m.StartOfWeek() + case "day", "d": + m.StartOfDay() + case "hour", "h": + if m.Minute() > 0 { + m.SubMinutes(m.Minute()) + } + + if m.Second() > 0 { + m.SubSeconds(m.Second()) + } + case "minute", "m": + if m.Second() > 0 { + m.SubSeconds(m.Second()) + } + case "second", "s": + + } + + return m +} + +// Carbon +func (m *Moment) StartOfDay() *Moment { + if m.Hour() > 0 { + _, timeOffset := m.GetTime().Zone() + m.SubHours(m.Hour()) + + _, newTimeOffset := m.GetTime().Zone() + diffOffset := timeOffset - newTimeOffset + if diffOffset != 0 { + // we need to adjust for time zone difference + m.AddSeconds(diffOffset) + } + } + + return m.StartOf("hour") +} + +// @todo ISO8601 Starts on Monday +func (m *Moment) StartOfWeek() *Moment { + return m.GoBackTo(time.Monday, false).StartOfDay() +} + +// Carbon +func (m *Moment) StartOfMonth() *Moment { + return m.SetDay(1).StartOfDay() +} + +// Carbon +func (m *Moment) StartOfYear() *Moment { + return m.SetMonth(time.January).SetDay(1).StartOfDay() +} + +// Carbon +func (m *Moment) EndOf(key string) *Moment { + switch key { + case "year", "y": + m.EndOfYear() + case "month", "M": + m.EndOfMonth() + case "week", "w": + m.EndOfWeek() + case "day", "d": + m.EndOfDay() + case "hour", "h": + if m.Minute() < 59 { + m.AddMinutes(59 - m.Minute()) + } + case "minute", "m": + if m.Second() < 59 { + m.AddSeconds(59 - m.Second()) + } + case "second", "s": + + } + + return m +} + +// Carbon +func (m *Moment) EndOfDay() *Moment { + if m.Hour() < 23 { + _, timeOffset := m.GetTime().Zone() + m.AddHours(23 - m.Hour()) + + _, newTimeOffset := m.GetTime().Zone() + diffOffset := newTimeOffset - timeOffset + if diffOffset != 0 { + // we need to adjust for time zone difference + m.SubSeconds(diffOffset) + } + } + + return m.EndOf("hour") +} + +// @todo ISO8601 Ends on Sunday +func (m *Moment) EndOfWeek() *Moment { + return m.GoTo(time.Sunday, false).EndOfDay() +} + +// Carbon +func (m *Moment) EndOfMonth() *Moment { + return m.SetDay(m.DaysInMonth()).EndOfDay() +} + +// Carbon +func (m *Moment) EndOfYear() *Moment { + return m.GoToMonth(time.December, false).EndOfMonth() +} + +// Custom +func (m *Moment) GoTo(day time.Weekday, next bool) *Moment { + if m.Day() == day { + if !next { + return m + } else { + m.AddDay() + } + } + + var diff int + if diff = int(day) - int(m.Day()); diff > 0 { + return m.AddDays(diff) + } + + return m.AddDays(7 + diff) +} + +// Custom +func (m *Moment) GoBackTo(day time.Weekday, previous bool) *Moment { + if m.Day() == day { + if !previous { + return m + } else { + m.SubDay() + } + } + + var diff int + if diff = int(day) - int(m.Day()); diff > 0 { + return m.SubDays(7 - diff) + } + + return m.SubDays(diff * -1) +} + +// Custom +func (m *Moment) GoToMonth(month time.Month, next bool) *Moment { + if m.Month() == month { + if !next { + return m + } else { + m.AddMonths(1) + } + } + + var diff int + if diff = int(month - m.Month()); diff > 0 { + return m.AddMonths(diff) + } + + return m.AddMonths(12 + diff) +} + +// Custom +func (m *Moment) GoBackToMonth(month time.Month, previous bool) *Moment { + if m.Month() == month { + if !previous { + return m + } else { + m.SubMonths(1) + } + } + + var diff int + if diff = int(month) - int(m.Month()); diff > 0 { + return m.SubMonths(12 - diff) + } + + return m.SubMonths(diff * -1) +} + +func (m *Moment) SetSecond(seconds int) *Moment { + if seconds >= 0 && seconds <= 60 { + return m.AddSeconds(seconds - m.Second()) + } + + return m +} + +func (m *Moment) SetMinute(minute int) *Moment { + if minute >= 0 && minute <= 60 { + return m.AddMinutes(minute - m.Minute()) + } + + return m +} + +func (m *Moment) SetHour(hour int) *Moment { + if hour >= 0 && hour <= 23 { + return m.AddHours(hour - m.Hour()) + } + + return m +} + +// Custom +func (m *Moment) SetDay(day int) *Moment { + if m.DayOfMonth() == day { + return m + } + + return m.AddDays(day - m.DayOfMonth()) +} + +// Custom +func (m *Moment) SetMonth(month time.Month) *Moment { + if m.Month() > month { + return m.GoBackToMonth(month, false) + } + + return m.GoToMonth(month, false) +} + +// Custom +func (m *Moment) SetYear(year int) *Moment { + if m.Year() == year { + return m + } + + return m.AddYears(year - m.Year()) +} + +// UTC Mode. @see http://momentjs.com/docs/#/parsing/utc/ +func (m *Moment) UTC() *Moment { + return m +} + +// http://momentjs.com/docs/#/manipulating/timezone-offset/ +func (m *Moment) Zone() int { + _, offset := m.GetTime().Zone() + + return (offset / 60) * -1 +} + +/** + * Display + * + */ +func (m *Moment) Format(layout string) string { + format := m.Convert(layout) + hasCustom := false + + formatted := m.GetTime().Format(format) + + if strings.Contains(formatted, "", fmt.Sprintf("%d", m.Unix()), -1) + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", m.ISOWeek()), -1) + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", m.DayOfWeek()), -1) + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", m.DayOfWeekISO()), -1) + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", m.DayOfYear()), -1) + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", m.Quarter()), -1) + formatted = strings.Replace(formatted, "", m.dayOfYearZero(), -1) + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", m.Hour()), -1) + } + + // This has to happen after time.Format + if hasCustom && strings.Contains(formatted, "") { + formatted = numberRegex.ReplaceAllStringFunc(formatted, func(n string) string { + ordinal, _ := strconv.Atoi(strings.Replace(n, "", "", 1)) + return m.ordinal(ordinal) + }) + } + + return formatted +} + +func (m *Moment) FormatGo(layout string) string { + return m.GetTime().Format(layout) +} + +// From Dmytro Shteflyuk @https://groups.google.com/forum/#!topic/golang-nuts/l8NhI74jl-4 +func (m *Moment) ordinal(x int) string { + suffix := "th" + switch x % 10 { + case 1: + if x%100 != 11 { + suffix = "st" + } + case 2: + if x%100 != 12 { + suffix = "nd" + } + case 3: + if x%100 != 13 { + suffix = "rd" + } + } + + return strconv.Itoa(x) + suffix +} + +func (m *Moment) FromNow() Diff { + now := new(Moment) + now.Now() + + return m.From(now) +} + +// Carbon +func (m *Moment) From(f *Moment) Diff { + return m.GetDiff(f) +} + +/** + * Difference + * + */ +func (m *Moment) Diff(t *Moment, unit string) int { + diff := m.GetDiff(t) + + switch unit { + case "years": + return diff.InYears() + case "months": + return diff.InMonths() + case "weeks": + return diff.InWeeks() + case "days": + return diff.InDays() + case "hours": + return diff.InHours() + case "minutes": + return diff.InMinutes() + case "seconds": + return diff.InSeconds() + } + + return 0 +} + +// Custom +func (m *Moment) GetDiff(t *Moment) Diff { + duration := m.GetTime().Sub(t.GetTime()) + + return Diff{duration} +} + +/** + * Display + * + */ +func (m *Moment) ValueOf() int64 { + return m.Unix() * 1000 +} + +func (m *Moment) Unix() int64 { + return m.GetTime().Unix() +} + +func (m *Moment) DaysInMonth() int { + days := 31 + switch m.Month() { + case time.April, time.June, time.September, time.November: + days = 30 + break + case time.February: + days = 28 + if m.IsLeapYear() { + days = 29 + } + break + } + + return days +} + +// or ToSlice? +func (m *Moment) ToArray() []int { + return []int{ + m.Year(), + int(m.Month()), + m.DayOfMonth(), + m.Hour(), + m.Minute(), + m.Second(), + } +} + +/** + * Query + * + */ +func (m *Moment) IsBefore(t Moment) bool { + return m.GetTime().Before(t.GetTime()) +} + +func (m *Moment) IsSame(t *Moment, layout string) bool { + return m.Format(layout) == t.Format(layout) +} + +func (m *Moment) IsAfter(t Moment) bool { + return m.GetTime().After(t.GetTime()) +} + +// Carbon +func (m *Moment) IsToday() bool { + today := m.Clone().Today() + + return m.Year() == today.Year() && m.Month() == today.Month() && m.Day() == today.Day() +} + +// Carbon +func (m *Moment) IsTomorrow() bool { + tomorrow := m.Clone().Tomorrow() + + return m.Year() == tomorrow.Year() && m.Month() == tomorrow.Month() && m.Day() == tomorrow.Day() +} + +// Carbon +func (m *Moment) IsYesterday() bool { + yesterday := m.Clone().Yesterday() + + return m.Year() == yesterday.Year() && m.Month() == yesterday.Month() && m.Day() == yesterday.Day() +} + +// Carbon +func (m *Moment) IsWeekday() bool { + return !m.IsWeekend() +} + +// Carbon +func (m *Moment) IsWeekend() bool { + return m.DayOfWeek() == time.Sunday || m.DayOfWeek() == time.Saturday +} + +func (m *Moment) IsLeapYear() bool { + year := m.Year() + return year%4 == 0 && (year%100 != 0 || year%400 == 0) +} + +// Custom +func (m *Moment) Range(start Moment, end Moment) bool { + return m.IsAfter(start) && m.IsBefore(end) +} diff --git a/vendor/github.com/leibowitz/moment/moment_parser.go b/vendor/github.com/leibowitz/moment/moment_parser.go new file mode 100644 index 00000000000..3361cfba113 --- /dev/null +++ b/vendor/github.com/leibowitz/moment/moment_parser.go @@ -0,0 +1,100 @@ +package moment + +import ( + "regexp" + "strings" +) + +type MomentParser struct{} + +var ( + date_pattern = regexp.MustCompile("(LT|LL?L?L?|l{1,4}|Mo|MM?M?M?|Do|DDDo|DD?D?D?|ddd?d?|do?|w[o|w]?|W[o|W]?|YYYYY|YYYY|YY|gg(ggg?)?|GG(GGG?)?|e|E|a|A|hh?|HH?|mm?|ss?|SS?S?|X|zz?|ZZ?|Q)") +) + +/* + + S (makes any number before it ordinal) + + stdDayOfYear 1,2,365 + + stdDayOfYearZero 001, 002, 365 + + stdDayOfWeek w 0, 1, 2 numeric day of the week (0 = sunday) + + stdDayOfWeekISO N 1 = Monday + + stdWeekOfYear W Iso week number of year + + stdUnix U + + stdQuarter +*/ + +// Thanks to https://github.com/fightbulc/moment.php for replacement keys and regex +var moment_replacements = map[string]string{ + "M": "1", // stdNumMonth 1 2 ... 11 12 + "Mo": "1", // stdNumMonth 1st 2nd ... 11th 12th + "MM": "01", // stdZeroMonth 01 02 ... 11 12 + "MMM": "Jan", // stdMonth Jan Feb ... Nov Dec + "MMMM": "January", // stdLongMonth January February ... November December + "D": "2", // stdDay 1 2 ... 30 30 + "Do": "2", // stdDay 1st 2nd ... 30th 31st @todo support st nd th etch + "DD": "02", // stdZeroDay 01 02 ... 30 31 + "DDD": "", // Day of the year 1 2 ... 364 365 + "DDDo": "", // Day of the year 1st 2nd ... 364th 365th + "DDDD": "", // Day of the year 001 002 ... 364 365 @todo**** + "d": "", // Numeric representation of day of the week 0 1 ... 5 6 + "do": "", // 0th 1st ... 5th 6th + "dd": "Mon", // ***Su Mo ... Fr Sa @todo + "ddd": "Mon", // Sun Mon ... Fri Sat + "dddd": "Monday", // stdLongWeekDay Sunday Monday ... Friday Saturday + "e": "", // Numeric representation of day of the week 0 1 ... 5 6 @todo + "E": "", // ISO-8601 numeric representation of the day of the week (added in PHP 5.1.0) 1 2 ... 6 7 @todo + "w": "", // 1 2 ... 52 53 + "wo": "", // 1st 2nd ... 52nd 53rd + "ww": "", // ***01 02 ... 52 53 @todo + "W": "", // 1 2 ... 52 53 + "Wo": "", // 1st 2nd ... 52nd 53rd + "WW": "", // ***01 02 ... 52 53 @todo + "YY": "06", // stdYear 70 71 ... 29 30 + "YYYY": "2006", // stdLongYear 1970 1971 ... 2029 2030 + // "gg" : "o", // ISO-8601 year number 70 71 ... 29 30 @todo + // "gggg" : "o", // ***1970 1971 ... 2029 2030 @todo + // "GG" : "o", //70 71 ... 29 30 @todo + // "GGGG" : "o", // ***1970 1971 ... 2029 2030 @todo + "Q": "", + "A": "PM", // stdPM AM PM + "a": "pm", // stdpm am pm + "H": "", // stdHour 0 1 ... 22 23 + "HH": "15", // 00 01 ... 22 23 + "h": "3", // stdHour12 1 2 ... 11 12 + "hh": "03", // stdZeroHour12 01 02 ... 11 12 + "m": "4", // stdZeroMinute 0 1 ... 58 59 + "mm": "04", // stdZeroMinute 00 01 ... 58 59 + "s": "5", // stdSecond 0 1 ... 58 59 + "ss": "05", // stdZeroSecond ***00 01 ... 58 59 + // "S" : "", //0 1 ... 8 9 + // "SS" : "", //0 1 ... 98 99 + // "SSS" : "", //0 1 ... 998 999 + "z": "MST", //EST CST ... MST PST + "zz": "MST", //EST CST ... MST PST + "Z": "Z07:00", // stdNumColonTZ -07:00 -06:00 ... +06:00 +07:00 + "ZZ": "-0700", // stdNumTZ -0700 -0600 ... +0600 +0700 + "X": "", // Seconds since unix epoch 1360013296 + "LT": "3:04 PM", // 8:30 PM + "L": "01/02/2006", //09/04/1986 + "l": "1/2/2006", //9/4/1986 + "LL": "January 2 2006", //September 4th 1986 the php s flag isn't supported + "ll": "Jan 2 2006", //Sep 4 1986 + "LLL": "January 2 2006 3:04 PM", //September 4th 1986 8:30 PM @todo the php s flag isn't supported + "lll": "Jan 2 2006 3:04 PM", //Sep 4 1986 8:30 PM + "LLLL": "Monday, January 2 2006 3:04 PM", //Thursday, September 4th 1986 8:30 PM the php s flag isn't supported + "llll": "Mon, Jan 2 2006 3:04 PM", //Thu, Sep 4 1986 8:30 PM +} + +func (p *MomentParser) Convert(layout string) string { + var match [][]string + if match = date_pattern.FindAllStringSubmatch(layout, -1); match == nil { + return layout + } + + for i := range match { + if replace, ok := moment_replacements[match[i][0]]; ok { + layout = strings.Replace(layout, match[i][0], replace, 1) + } + } + + return layout +} diff --git a/vendor/github.com/leibowitz/moment/parse_day.go b/vendor/github.com/leibowitz/moment/parse_day.go new file mode 100644 index 00000000000..e8e890a462e --- /dev/null +++ b/vendor/github.com/leibowitz/moment/parse_day.go @@ -0,0 +1,32 @@ +package moment + +import ( + "fmt" + "strings" + "time" +) + +var ( + days = []time.Weekday{ + time.Sunday, + time.Monday, + time.Tuesday, + time.Wednesday, + time.Thursday, + time.Friday, + time.Saturday, + } +) + +func ParseWeekDay(day string) (time.Weekday, error) { + + day = strings.ToLower(day) + + for _, d := range days { + if day == strings.ToLower(d.String()) { + return d, nil + } + } + + return -1, fmt.Errorf("Unable to parse %s as week day", day) +} diff --git a/vendor/github.com/leibowitz/moment/strftime_parser.go b/vendor/github.com/leibowitz/moment/strftime_parser.go new file mode 100644 index 00000000000..3c024376535 --- /dev/null +++ b/vendor/github.com/leibowitz/moment/strftime_parser.go @@ -0,0 +1,68 @@ +package moment + +import ( + "regexp" + "strings" +) + +type StrftimeParser struct{} + +var ( + replacements_pattern = regexp.MustCompile("%[mbhBedjwuaAVgyGYpPkHlIMSZzsTrRTDFXx]") +) + +// Not implemented +// U +// C + +var strftime_replacements = map[string]string{ + "%m": "01", // stdZeroMonth 01 02 ... 11 12 + "%b": "Jan", // stdMonth Jan Feb ... Nov Dec + "%h": "Jan", + "%B": "January", // stdLongMonth January February ... November December + "%e": "2", // stdDay 1 2 ... 30 30 + "%d": "02", // stdZeroDay 01 02 ... 30 31 + "%j": "", // Day of the year ***001 002 ... 364 365 @todo**** + "%w": "", // Numeric representation of day of the week 0 1 ... 5 6 + "%u": "", // ISO-8601 numeric representation of the day of the week (added in PHP 5.1.0) 1 2 ... 6 7 @todo + "%a": "Mon", // Sun Mon ... Fri Sat + "%A": "Monday", // stdLongWeekDay Sunday Monday ... Friday Saturday + "%V": "", // ***01 02 ... 52 53 @todo begin with zeros + "%g": "06", // stdYear 70 71 ... 29 30 + "%y": "06", + "%G": "2006", // stdLongYear 1970 1971 ... 2029 2030 + "%Y": "2006", + "%p": "PM", // stdPM AM PM + "%P": "pm", // stdpm am pm + "%k": "15", // stdHour 0 1 ... 22 23 + "%H": "15", // 00 01 ... 22 23 + "%l": "3", // stdHour12 1 2 ... 11 12 + "%I": "03", // stdZeroHour12 01 02 ... 11 12 + "%M": "04", // stdZeroMinute 00 01 ... 58 59 + "%S": "05", // stdZeroSecond ***00 01 ... 58 59 + "%Z": "MST", //EST CST ... MST PST + "%z": "-0700", // stdNumTZ -0700 -0600 ... +0600 +0700 + "%s": "", // Seconds since unix epoch 1360013296 + "%r": "03:04:05 PM", + "%R": "15:04", + "%T": "15:04:05", + "%D": "01/02/06", + "%F": "2006-01-02", + "%X": "15:04:05", + "%x": "01/02/06", +} + +func (p *StrftimeParser) Convert(layout string) string { + var match [][]string + if match = replacements_pattern.FindAllStringSubmatch(layout, -1); match == nil { + return layout + } + + for i := range match { + if replace, ok := strftime_replacements[match[i][0]]; ok { + layout = strings.Replace(layout, match[i][0], replace, 1) + } + } + + return layout +} From 63a200686e065a79fdd7ade563fd942236c4feda Mon Sep 17 00:00:00 2001 From: wph95 Date: Mon, 26 Mar 2018 19:48:57 +0800 Subject: [PATCH 05/87] - pipeline aggs support - add some test --- pkg/tsdb/elasticsearch/elasticsearch.go | 42 ++- pkg/tsdb/elasticsearch/model_parser.go | 81 ++---- pkg/tsdb/elasticsearch/models.go | 21 +- pkg/tsdb/elasticsearch/query.go | 182 +++++++----- pkg/tsdb/elasticsearch/query_def.go | 18 ++ pkg/tsdb/elasticsearch/query_test.go | 331 ++++++++++++++++++++++ pkg/tsdb/elasticsearch/response_parser.go | 34 ++- 7 files changed, 557 insertions(+), 152 deletions(-) create mode 100644 pkg/tsdb/elasticsearch/query_test.go diff --git a/pkg/tsdb/elasticsearch/elasticsearch.go b/pkg/tsdb/elasticsearch/elasticsearch.go index 8fd82a179e8..0ce9eca0972 100644 --- a/pkg/tsdb/elasticsearch/elasticsearch.go +++ b/pkg/tsdb/elasticsearch/elasticsearch.go @@ -1,6 +1,7 @@ package elasticsearch import ( + "bytes" "context" "encoding/json" "errors" @@ -18,7 +19,8 @@ import ( ) type ElasticsearchExecutor struct { - Transport *http.Transport + QueryParser *ElasticSearchQueryParser + Transport *http.Transport } var ( @@ -47,17 +49,21 @@ func (e *ElasticsearchExecutor) Query(ctx context.Context, dsInfo *models.DataSo result := &tsdb.Response{} result.Results = make(map[string]*tsdb.QueryResult) - queryParser := ElasticSearchQueryParser{ - dsInfo, - tsdbQuery.TimeRange, - tsdbQuery.Queries, - } - - payload, targets, err := queryParser.Parse() + queries, err := e.getQuery(dsInfo, tsdbQuery) if err != nil { return nil, err } + buff := bytes.Buffer{} + for _, q := range queries { + s, err := q.Build(tsdbQuery, dsInfo) + if err != nil { + return nil, err + } + buff.WriteString(s) + } + payload := buff.String() + if setting.Env == setting.DEV { glog.Debug("Elasticsearch playload", "raw playload", payload) } @@ -96,12 +102,30 @@ func (e *ElasticsearchExecutor) Query(ctx context.Context, dsInfo *models.DataSo return nil, errors.New(res.getErrMsg()) } } - responseParser := ElasticsearchResponseParser{responses.Responses, targets} + responseParser := ElasticsearchResponseParser{responses.Responses, queries} queryRes := responseParser.getTimeSeries() result.Results["A"] = queryRes return result, nil } +func (e *ElasticsearchExecutor) getQuery(dsInfo *models.DataSource, context *tsdb.TsdbQuery) ([]*Query, error) { + queries := make([]*Query, 0) + if len(context.Queries) == 0 { + return nil, fmt.Errorf("query request contains no queries") + } + for _, v := range context.Queries { + + query, err := e.QueryParser.Parse(v.Model, dsInfo) + if err != nil { + return nil, err + } + queries = append(queries, query) + + } + return queries, nil + +} + func (e *ElasticsearchExecutor) createRequest(dsInfo *models.DataSource, query string) (*http.Request, error) { u, _ := url.Parse(dsInfo.Url) u.Path = path.Join(u.Path, "_msearch") diff --git a/pkg/tsdb/elasticsearch/model_parser.go b/pkg/tsdb/elasticsearch/model_parser.go index 7da6765e06c..0d016dc58a5 100644 --- a/pkg/tsdb/elasticsearch/model_parser.go +++ b/pkg/tsdb/elasticsearch/model_parser.go @@ -1,62 +1,45 @@ package elasticsearch import ( - "bytes" - "encoding/json" "fmt" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/tsdb" "github.com/leibowitz/moment" - "src/github.com/davecgh/go-spew/spew" - "strconv" "strings" "time" ) type ElasticSearchQueryParser struct { - DsInfo *models.DataSource - TimeRange *tsdb.TimeRange - Queries []*tsdb.Query } -func (qp *ElasticSearchQueryParser) Parse() (string, []*QueryBuilder, error) { - payload := bytes.Buffer{} - queryHeader := qp.getQueryHeader() - targets := make([]*QueryBuilder, 0) - for _, q := range qp.Queries { - timeField, err := q.Model.Get("timeField").String() - if err != nil { - return "", nil, err - } - rawQuery := q.Model.Get("query").MustString("") - bucketAggs := q.Model.Get("bucketAggs").MustArray() - metrics := q.Model.Get("metrics").MustArray() - alias := q.Model.Get("alias").MustString("") - builder := QueryBuilder{timeField, rawQuery, bucketAggs, metrics, alias} - targets = append(targets, &builder) - - query, err := builder.Build() - if err != nil { - return "", nil, err - } - queryBytes, err := json.Marshal(query) - if err != nil { - return "", nil, err - } - - payload.WriteString(queryHeader.String() + "\n") - payload.WriteString(string(queryBytes) + "\n") +func (qp *ElasticSearchQueryParser) Parse(model *simplejson.Json, dsInfo *models.DataSource) (*Query, error) { + //payload := bytes.Buffer{} + //queryHeader := qp.getQueryHeader() + timeField, err := model.Get("timeField").String() + if err != nil { + return nil, err + } + rawQuery := model.Get("query").MustString("") + bucketAggs := model.Get("bucketAggs").MustArray() + metrics := model.Get("metrics").MustArray() + alias := model.Get("alias").MustString("") + parsedInterval, err := tsdb.GetIntervalFrom(dsInfo, model, time.Millisecond) + if err != nil { + return nil, err } - p, err := qp.payloadReplace(payload.String(), qp.DsInfo.JsonData) - - return p, targets, err + return &Query{timeField, + rawQuery, + bucketAggs, + metrics, + alias, + parsedInterval}, nil } -func (qp *ElasticSearchQueryParser) getQueryHeader() *QueryHeader { +func getRequestHeader(timeRange *tsdb.TimeRange, dsInfo *models.DataSource) *QueryHeader { var header QueryHeader - esVersion := qp.DsInfo.JsonData.Get("esVersion").MustInt() + esVersion := dsInfo.JsonData.Get("esVersion").MustInt() searchType := "query_then_fetch" if esVersion < 5 { @@ -64,29 +47,13 @@ func (qp *ElasticSearchQueryParser) getQueryHeader() *QueryHeader { } header.SearchType = searchType header.IgnoreUnavailable = true - header.Index = getIndexList(qp.DsInfo.Database, qp.DsInfo.JsonData.Get("interval").MustString(""), qp.TimeRange) + header.Index = getIndexList(dsInfo.Database, dsInfo.JsonData.Get("interval").MustString(""), timeRange) if esVersion >= 56 { - header.MaxConcurrentShardRequests = qp.DsInfo.JsonData.Get("maxConcurrentShardRequests").MustInt() + header.MaxConcurrentShardRequests = dsInfo.JsonData.Get("maxConcurrentShardRequests").MustInt() } return &header } -func (qp *ElasticSearchQueryParser) payloadReplace(payload string, model *simplejson.Json) (string, error) { - parsedInterval, err := tsdb.GetIntervalFrom(qp.DsInfo, model, time.Millisecond) - if err != nil { - return "", nil - } - - interval := intervalCalculator.Calculate(qp.TimeRange, parsedInterval) - glog.Warn(spew.Sdump(interval)) - payload = strings.Replace(payload, "$timeFrom", fmt.Sprintf("%d", qp.TimeRange.GetFromAsMsEpoch()), -1) - payload = strings.Replace(payload, "$timeTo", fmt.Sprintf("%d", qp.TimeRange.GetToAsMsEpoch()), -1) - payload = strings.Replace(payload, "$interval", interval.Text, -1) - payload = strings.Replace(payload, "$__interval_ms", strconv.FormatInt(interval.Value.Nanoseconds()/int64(time.Millisecond), 10), -1) - payload = strings.Replace(payload, "$__interval", interval.Text, -1) - - return payload, nil -} func getIndexList(pattern string, interval string, timeRange *tsdb.TimeRange) string { if interval == "" { diff --git a/pkg/tsdb/elasticsearch/models.go b/pkg/tsdb/elasticsearch/models.go index d758e2159de..822df2dd4d1 100644 --- a/pkg/tsdb/elasticsearch/models.go +++ b/pkg/tsdb/elasticsearch/models.go @@ -1,25 +1,25 @@ package elasticsearch import ( - "github.com/grafana/grafana/pkg/components/simplejson" "bytes" - "fmt" "encoding/json" + "fmt" + "github.com/grafana/grafana/pkg/components/simplejson" ) type QueryHeader struct { SearchType string `json:"search_type"` IgnoreUnavailable bool `json:"ignore_unavailable"` Index interface{} `json:"index"` - MaxConcurrentShardRequests int `json:"max_concurrent_shard_requests"` + MaxConcurrentShardRequests int `json:"max_concurrent_shard_requests,omitempty"` } -func (q *QueryHeader) String() (string) { +func (q *QueryHeader) String() string { r, _ := json.Marshal(q) return string(r) } -type Query struct { +type Request struct { Query map[string]interface{} `json:"query"` Aggs Aggs `json:"aggs"` Size int `json:"size"` @@ -45,11 +45,10 @@ type FiltersAgg struct { } type TermsAggSetting struct { - Field string `json:"field"` - Size int `json:"size"` - Order map[string]interface{} `json:"order"` - MinDocCount int `json:"min_doc_count"` - Missing string `json:"missing"` + Field string `json:"field"` + Size int `json:"size"` + Order map[string]interface{} `json:"order"` + Missing string `json:"missing,omitempty"` } type TermsAgg struct { @@ -104,7 +103,7 @@ type Response struct { Aggregations map[string]interface{} `json:"aggregations"` } -func (r *Response) getErrMsg() (string) { +func (r *Response) getErrMsg() string { var msg bytes.Buffer errJson := simplejson.NewFromAny(r.Err) errType, err := errJson.Get("type").String() diff --git a/pkg/tsdb/elasticsearch/query.go b/pkg/tsdb/elasticsearch/query.go index d6d70e79a2a..51f1ebb5d7a 100644 --- a/pkg/tsdb/elasticsearch/query.go +++ b/pkg/tsdb/elasticsearch/query.go @@ -1,81 +1,103 @@ package elasticsearch import ( + "bytes" + "encoding/json" "errors" + "fmt" "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/tsdb" "strconv" + "strings" + "time" ) var rangeFilterSetting = RangeFilterSetting{Gte: "$timeFrom", - Lte: "$timeTo", + Lte: "$timeTo", Format: "epoch_millis"} -type QueryBuilder struct { - TimeField string - RawQuery string - BucketAggs []interface{} - Metrics []interface{} - Alias string +type Query struct { + TimeField string `json:"timeField"` + RawQuery string `json:"query"` + BucketAggs []interface{} `json:"bucketAggs"` + Metrics []interface{} `json:"metrics"` + Alias string `json:"Alias"` + Interval time.Duration } -func (b *QueryBuilder) Build() (Query, error) { - var err error - var res Query - res.Query = make(map[string]interface{}) - res.Size = 0 +func (q *Query) Build(queryContext *tsdb.TsdbQuery, dsInfo *models.DataSource) (string, error) { + var req Request + payload := bytes.Buffer{} - if err != nil { - return res, err - } - - boolQuery := BoolQuery{} - boolQuery.Filter = append(boolQuery.Filter, newRangeFilter(b.TimeField, rangeFilterSetting)) - boolQuery.Filter = append(boolQuery.Filter, newQueryStringFilter(true, b.RawQuery)) - res.Query["bool"] = boolQuery + req.Size = 0 + q.renderReqQuery(&req) // handle document query - if len(b.BucketAggs) == 0 { - if len(b.Metrics) > 0 { - metric := simplejson.NewFromAny(b.Metrics[0]) + if q.isRawDocumentQuery() { + return "", errors.New("alert not support Raw_Document") + } + + err := q.parseAggs(&req) + if err != nil { + return "", err + } + + reqBytes, err := json.Marshal(req) + reqHeader := getRequestHeader(queryContext.TimeRange, dsInfo) + payload.WriteString(reqHeader.String() + "\n") + payload.WriteString(string(reqBytes) + "\n") + return q.renderTemplate(payload.String(), queryContext) +} + +func (q *Query) isRawDocumentQuery() bool { + if len(q.BucketAggs) == 0 { + if len(q.Metrics) > 0 { + metric := simplejson.NewFromAny(q.Metrics[0]) if metric.Get("type").MustString("") == "raw_document" { - return res, errors.New("alert not support Raw_Document") + return true } } } - aggs, err := b.parseAggs(b.BucketAggs, b.Metrics) - res.Aggs = aggs["aggs"].(Aggs) - - return res, err + return false } -func (b *QueryBuilder) parseAggs(bucketAggs []interface{}, metrics []interface{}) (Aggs, error) { - query := make(Aggs) - nestedAggs := query - for _, aggRaw := range bucketAggs { +func (q *Query) renderReqQuery(req *Request) { + req.Query = make(map[string]interface{}) + boolQuery := BoolQuery{} + boolQuery.Filter = append(boolQuery.Filter, newRangeFilter(q.TimeField, rangeFilterSetting)) + boolQuery.Filter = append(boolQuery.Filter, newQueryStringFilter(true, q.RawQuery)) + req.Query["bool"] = boolQuery +} + +func (q *Query) parseAggs(req *Request) error { + aggs := make(Aggs) + nestedAggs := aggs + for _, aggRaw := range q.BucketAggs { esAggs := make(Aggs) aggJson := simplejson.NewFromAny(aggRaw) aggType, err := aggJson.Get("type").String() if err != nil { - return nil, err + return err } id, err := aggJson.Get("id").String() if err != nil { - return nil, err + return err } switch aggType { case "date_histogram": - esAggs["date_histogram"] = b.getDateHistogramAgg(aggJson) + esAggs["date_histogram"] = q.getDateHistogramAgg(aggJson) case "histogram": - esAggs["histogram"] = b.getHistogramAgg(aggJson) + esAggs["histogram"] = q.getHistogramAgg(aggJson) case "filters": - esAggs["filters"] = b.getFilters(aggJson) + esAggs["filters"] = q.getFilters(aggJson) case "terms": - terms := b.getTerms(aggJson) + terms := q.getTerms(aggJson) esAggs["terms"] = terms.Terms esAggs["aggs"] = terms.Aggs case "geohash_grid": - return nil, errors.New("alert not support Geo_Hash_Grid") + return errors.New("alert not support Geo_Hash_Grid") } if _, ok := nestedAggs["aggs"]; !ok { @@ -90,40 +112,51 @@ func (b *QueryBuilder) parseAggs(bucketAggs []interface{}, metrics []interface{} } nestedAggs["aggs"] = make(Aggs) - for _, metricRaw := range metrics { + for _, metricRaw := range q.Metrics { metric := make(Metric) metricJson := simplejson.NewFromAny(metricRaw) id, err := metricJson.Get("id").String() if err != nil { - return nil, err + return err } metricType, err := metricJson.Get("type").String() if err != nil { - return nil, err + return err } if metricType == "count" { continue } - // todo support pipeline Agg + settings := metricJson.Get("settings").MustMap(map[string]interface{}{}) + + if isPipelineAgg(metricType) { + pipelineAgg := metricJson.Get("pipelineAgg").MustString("") + if _, err := strconv.Atoi(pipelineAgg); err == nil { + settings["buckets_path"] = pipelineAgg + } else { + continue + } + + } else { + settings["field"] = metricJson.Get("field").MustString() + } - settings := metricJson.Get("settings").MustMap() - settings["field"] = metricJson.Get("field").MustString() metric[metricType] = settings nestedAggs["aggs"].(Aggs)[id] = metric } - return query, nil + req.Aggs = aggs["aggs"].(Aggs) + return nil } -func (b *QueryBuilder) getDateHistogramAgg(model *simplejson.Json) DateHistogramAgg { +func (q *Query) getDateHistogramAgg(model *simplejson.Json) *DateHistogramAgg { agg := &DateHistogramAgg{} settings := simplejson.NewFromAny(model.Get("settings").Interface()) interval, err := settings.Get("interval").String() if err == nil { agg.Interval = interval } - agg.Field = b.TimeField + agg.Field = q.TimeField agg.MinDocCount = settings.Get("min_doc_count").MustInt(0) agg.ExtendedBounds = ExtendedBounds{"$timeFrom", "$timeTo"} agg.Format = "epoch_millis" @@ -136,10 +169,10 @@ func (b *QueryBuilder) getDateHistogramAgg(model *simplejson.Json) DateHistogram if err == nil { agg.Missing = missing } - return *agg + return agg } -func (b *QueryBuilder) getHistogramAgg(model *simplejson.Json) HistogramAgg { +func (q *Query) getHistogramAgg(model *simplejson.Json) *HistogramAgg { agg := &HistogramAgg{} settings := simplejson.NewFromAny(model.Get("settings").Interface()) interval, err := settings.Get("interval").String() @@ -155,10 +188,10 @@ func (b *QueryBuilder) getHistogramAgg(model *simplejson.Json) HistogramAgg { if err == nil { agg.Missing = missing } - return *agg + return agg } -func (b *QueryBuilder) getFilters(model *simplejson.Json) FiltersAgg { +func (q *Query) getFilters(model *simplejson.Json) *FiltersAgg { agg := &FiltersAgg{} settings := simplejson.NewFromAny(model.Get("settings").Interface()) for filter := range settings.Get("filters").MustArray() { @@ -170,15 +203,15 @@ func (b *QueryBuilder) getFilters(model *simplejson.Json) FiltersAgg { } agg.Filter[label] = newQueryStringFilter(true, query) } - return *agg + return agg } -func (b *QueryBuilder) getTerms(model *simplejson.Json) TermsAgg { +func (q *Query) getTerms(model *simplejson.Json) *TermsAgg { agg := &TermsAgg{Aggs: make(Aggs)} settings := simplejson.NewFromAny(model.Get("settings").Interface()) agg.Terms.Field = model.Get("field").MustString() if settings == nil { - return *agg + return agg } sizeStr := settings.Get("size").MustString("") size, err := strconv.Atoi(sizeStr) @@ -186,17 +219,25 @@ func (b *QueryBuilder) getTerms(model *simplejson.Json) TermsAgg { size = 500 } agg.Terms.Size = size - orderBy := settings.Get("orderBy").MustString("") - if orderBy != "" { + orderBy, err := settings.Get("orderBy").String() + if err == nil { agg.Terms.Order = make(map[string]interface{}) agg.Terms.Order[orderBy] = settings.Get("order").MustString("") - // if orderBy is a int, means this fields is metric result value - // TODO set subAggs - } - - minDocCount, err := settings.Get("min_doc_count").Int() - if err == nil { - agg.Terms.MinDocCount = minDocCount + if _, err := strconv.Atoi(orderBy); err != nil { + for _, metricI := range q.Metrics { + metric := simplejson.NewFromAny(metricI) + metricId := metric.Get("id").MustString() + if metricId == orderBy { + subAggs := make(Aggs) + metricField := metric.Get("field").MustString() + metricType := metric.Get("type").MustString() + subAggs[metricType] = map[string]string{"field": metricField} + agg.Aggs = make(Aggs) + agg.Aggs[metricId] = subAggs + break + } + } + } } missing, err := settings.Get("missing").String() @@ -204,5 +245,16 @@ func (b *QueryBuilder) getTerms(model *simplejson.Json) TermsAgg { agg.Terms.Missing = missing } - return *agg + return agg +} + +func (q *Query) renderTemplate(payload string, queryContext *tsdb.TsdbQuery) (string, error) { + timeRange := queryContext.TimeRange + interval := intervalCalculator.Calculate(timeRange, q.Interval) + payload = strings.Replace(payload, "$timeFrom", fmt.Sprintf("%d", timeRange.GetFromAsMsEpoch()), -1) + payload = strings.Replace(payload, "$timeTo", fmt.Sprintf("%d", timeRange.GetToAsMsEpoch()), -1) + payload = strings.Replace(payload, "$interval", interval.Text, -1) + payload = strings.Replace(payload, "$__interval_ms", strconv.FormatInt(interval.Value.Nanoseconds()/int64(time.Millisecond), 10), -1) + payload = strings.Replace(payload, "$__interval", interval.Text, -1) + return payload, nil } diff --git a/pkg/tsdb/elasticsearch/query_def.go b/pkg/tsdb/elasticsearch/query_def.go index 5dc02aa359e..6f78f02f346 100644 --- a/pkg/tsdb/elasticsearch/query_def.go +++ b/pkg/tsdb/elasticsearch/query_def.go @@ -24,3 +24,21 @@ var extendedStats = map[string]string{ "std_deviation_bounds_upper": "Std Dev Upper", "std_deviation_bounds_lower": "Std Dev Lower", } + +var pipelineOptions = map[string]string{ + "moving_avg": "moving_avg", + "derivative": "derivative", +} + +func isPipelineAgg(metricType string) bool { + if _, ok := pipelineOptions[metricType]; ok { + return true + } + return false +} + +func describeMetric(metricType, field string) string { + text := metricAggType[metricType] + return text + " " + field + +} diff --git a/pkg/tsdb/elasticsearch/query_test.go b/pkg/tsdb/elasticsearch/query_test.go new file mode 100644 index 00000000000..992469175b6 --- /dev/null +++ b/pkg/tsdb/elasticsearch/query_test.go @@ -0,0 +1,331 @@ +package elasticsearch + +import ( + "encoding/json" + "fmt" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/tsdb" + . "github.com/smartystreets/goconvey/convey" + "reflect" + "strconv" + "strings" + "testing" +) + +func testElasticSearchResponse(requestJSON string, expectedElasticSearchRequestJSON string) { + var queryExpectedJSONInterface, queryJSONInterface interface{} + parser := ElasticSearchQueryParser{} + model := &Query{} + + err := json.Unmarshal([]byte(requestJSON), model) + So(err, ShouldBeNil) + jsonDate, _ := simplejson.NewJson([]byte(`{"esVersion":2}`)) + dsInfo := &models.DataSource{ + Database: "grafana-test", + JsonData: jsonDate, + } + + testTimeRange := tsdb.NewTimeRange("5m", "now") + + req, _ := simplejson.NewJson([]byte(requestJSON)) + query, err := parser.Parse(req, dsInfo) + s, err := query.Build(&tsdb.TsdbQuery{TimeRange: testTimeRange}, dsInfo) + + queryJSON := strings.Split(s, "\n")[1] + err = json.Unmarshal([]byte(queryJSON), &queryJSONInterface) + So(err, ShouldBeNil) + + expectedElasticSearchRequestJSON = strings.Replace( + expectedElasticSearchRequestJSON, + "", + strconv.FormatInt(testTimeRange.GetFromAsMsEpoch(), 10), + -1, + ) + + expectedElasticSearchRequestJSON = strings.Replace( + expectedElasticSearchRequestJSON, + "", + strconv.FormatInt(testTimeRange.GetToAsMsEpoch(), 10), + -1, + ) + + err = json.Unmarshal([]byte(expectedElasticSearchRequestJSON), &queryExpectedJSONInterface) + So(err, ShouldBeNil) + + result := reflect.DeepEqual(queryExpectedJSONInterface, queryJSONInterface) + if !result { + fmt.Printf("ERROR: %s \n != \n %s", expectedElasticSearchRequestJSON, queryJSON) + } + So(result, ShouldBeTrue) +} +func TestElasticSearchQueryBuilder(t *testing.T) { + Convey("Elasticsearch QueryBuilder query testing", t, func() { + Convey("Build test average metric with moving average", func() { + var testElasticsearchModelRequestJSON = ` + { + "bucketAggs": [ + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": 0, + "trimEdges": 0 + }, + "type": "date_histogram" + } + ], + "dsType": "elasticsearch", + "metrics": [ + { + "field": "value", + "id": "1", + "inlineScript": "_value * 2", + "meta": {}, + "settings": { + "script": { + "inline": "_value * 2" + } + }, + "type": "avg" + }, + { + "field": "1", + "id": "3", + "meta": {}, + "pipelineAgg": "1", + "settings": { + "minimize": false, + "model": "simple", + "window": 5 + }, + "type": "moving_avg" + } + ], + "query": "(test:query) AND (name:sample)", + "refId": "A", + "timeField": "timestamp" + } + ` + + var expectedElasticsearchQueryJSON = ` + { + "size": 0, + "query": { + "bool": { + "filter": [ + { + "range": { + "timestamp": { + "gte": "", + "lte": "", + "format": "epoch_millis" + } + } + }, + { + "query_string": { + "analyze_wildcard": true, + "query": "(test:query) AND (name:sample)" + } + } + ] + } + }, + "aggs": { + "2": { + "date_histogram": { + "interval": "200ms", + "field": "timestamp", + "min_doc_count": 0, + "extended_bounds": { + "min": "", + "max": "" + }, + "format": "epoch_millis" + }, + "aggs": { + "1": { + "avg": { + "field": "value", + "script": { + "inline": "_value * 2" + } + } + }, + "3": { + "moving_avg": { + "buckets_path": "1", + "window": 5, + "model": "simple", + "minimize": false + } + } + } + } + } + }` + + testElasticSearchResponse(testElasticsearchModelRequestJSON, expectedElasticsearchQueryJSON) + }) + Convey("Test Wildcards and Quotes", func() { + testElasticsearchModelRequestJSON := ` + { + "alias": "New", + "bucketAggs": [ + { + "field": "timestamp", + "id": "2", + "type": "date_histogram" + } + ], + "dsType": "elasticsearch", + "metrics": [ + { + "type": "sum", + "field": "value", + "id": "1" + } + ], + "query": "scope:$location.leagueconnect.api AND name:*CreateRegistration AND name:\"*.201-responses.rate\"", + "refId": "A", + "timeField": "timestamp" + }` + + expectedElasticsearchQueryJSON := ` + { + "size": 0, + "query": { + "bool": { + "filter": [ + { + "range": { + "timestamp": { + "gte": "", + "lte": "", + "format": "epoch_millis" + } + } + }, + { + "query_string": { + "analyze_wildcard": true, + "query": "scope:$location.leagueconnect.api AND name:*CreateRegistration AND name:\"*.201-responses.rate\"" + } + } + ] + } + }, + "aggs": { + "2": { + "aggs": { + "1": { + "sum": { + "field": "value" + } + } + }, + "date_histogram": { + "extended_bounds": { + "max": "", + "min": "" + }, + "field": "timestamp", + "format": "epoch_millis", + "min_doc_count": 0 + } + } + } + }` + + testElasticSearchResponse(testElasticsearchModelRequestJSON, expectedElasticsearchQueryJSON) + }) + Convey("Test Term Aggregates", func() { + testElasticsearchModelRequestJSON := ` + { + "bucketAggs": [{ + "field": "name_raw", + "id": "4", + "settings": { + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "1m", + "min_doc_count": 0, + "trimEdges": 0 + }, + "type": "date_histogram" + }], + "dsType": "elasticsearch", + "filters": [{ + "boolOp": "AND", + "not": false, + "type": "rfc190Scope", + "value": "*.hmp.metricsd" + }, { + "boolOp": "AND", + "not": false, + "type": "name_raw", + "value": "builtin.general.*_instance_count" + }], + "metricObject": {}, + "metrics": [{ + "field": "value", + "id": "1", + "meta": {}, + "options": {}, + "settings": {}, + "type": "sum" + }], + "mode": 0, + "numToGraph": 10, + "prependHostName": false, + "query": "(scope:*.hmp.metricsd) AND (name_raw:builtin.general.*_instance_count)", + "refId": "A", + "regexAlias": false, + "selectedApplication": "", + "selectedHost": "", + "selectedLocation": "", + "timeField": "timestamp", + "useFullHostName": "", + "useQuery": false + }` + + expectedElasticsearchQueryJSON := ` + { + "size": 0, + "query": { + "bool": { + "filter": [ + { + "range": { + "timestamp": { + "gte": "", + "lte": "", + "format": "epoch_millis" + } + } + }, + { + "query_string": { + "analyze_wildcard": true, + "query": "(scope:*.hmp.metricsd) AND (name_raw:builtin.general.*_instance_count)" + } + } + ] + } + }, + "aggs": {"4":{"aggs":{"2":{"aggs":{"1":{"sum":{"field":"value"}}},"date_histogram":{"extended_bounds":{"max":"","min":""},"field":"timestamp","format":"epoch_millis","interval":"1m","min_doc_count":0}}},"terms":{"field":"name_raw","order":{"_term":"desc"},"size":10}}} + }` + + testElasticSearchResponse(testElasticsearchModelRequestJSON, expectedElasticsearchQueryJSON) + }) + }) +} diff --git a/pkg/tsdb/elasticsearch/response_parser.go b/pkg/tsdb/elasticsearch/response_parser.go index a2a8565641f..01b8cb1d235 100644 --- a/pkg/tsdb/elasticsearch/response_parser.go +++ b/pkg/tsdb/elasticsearch/response_parser.go @@ -6,14 +6,14 @@ import ( "github.com/grafana/grafana/pkg/components/null" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/tsdb" - "strconv" "regexp" + "strconv" "strings" ) type ElasticsearchResponseParser struct { Responses []Response - Targets []*QueryBuilder + Targets []*Query } func (rp *ElasticsearchResponseParser) getTimeSeries() *tsdb.QueryResult { @@ -29,7 +29,7 @@ func (rp *ElasticsearchResponseParser) getTimeSeries() *tsdb.QueryResult { return queryRes } -func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{}, target *QueryBuilder, series *[]*tsdb.TimeSeries, props map[string]string, depth int) (error) { +func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{}, target *Query, series *[]*tsdb.TimeSeries, props map[string]string, depth int) error { var err error maxDepth := len(target.BucketAggs) - 1 for aggId, v := range aggs { @@ -71,7 +71,7 @@ func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{ } -func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, target *QueryBuilder, series *[]*tsdb.TimeSeries, props map[string]string) (error) { +func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, target *Query, series *[]*tsdb.TimeSeries, props map[string]string) error { for _, v := range target.Metrics { metric := simplejson.NewFromAny(v) if metric.Get("hide").MustBool(false) { @@ -143,7 +143,7 @@ func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, ta return nil } -func (rp *ElasticsearchResponseParser) nameSeries(seriesList *[]*tsdb.TimeSeries, target *QueryBuilder) { +func (rp *ElasticsearchResponseParser) nameSeries(seriesList *[]*tsdb.TimeSeries, target *Query) { set := make(map[string]string) for _, v := range *seriesList { if metricType, exists := v.Tags["metric"]; exists { @@ -159,8 +159,9 @@ func (rp *ElasticsearchResponseParser) nameSeries(seriesList *[]*tsdb.TimeSeries } -func (rp *ElasticsearchResponseParser) getSeriesName(series *tsdb.TimeSeries, target *QueryBuilder, metricTypeCount int) (string) { - metricName := rp.getMetricName(series.Tags["metric"]) +func (rp *ElasticsearchResponseParser) getSeriesName(series *tsdb.TimeSeries, target *Query, metricTypeCount int) string { + metricType := series.Tags["metric"] + metricName := rp.getMetricName(metricType) delete(series.Tags, "metric") field := "" @@ -172,7 +173,7 @@ func (rp *ElasticsearchResponseParser) getSeriesName(series *tsdb.TimeSeries, ta if target.Alias != "" { var re = regexp.MustCompile(`{{([\s\S]+?)}}`) for _, match := range re.FindAllString(target.Alias, -1) { - group := match[2:len(match)-2] + group := match[2 : len(match)-2] if strings.HasPrefix(group, "term ") { if term, ok := series.Tags["term "]; ok { @@ -193,7 +194,20 @@ func (rp *ElasticsearchResponseParser) getSeriesName(series *tsdb.TimeSeries, ta } } // todo, if field and pipelineAgg - if field != "" { + if field != "" && isPipelineAgg(metricType) { + found := false + for _, targetMetricI := range target.Metrics { + targetMetric := simplejson.NewFromAny(targetMetricI) + if targetMetric.Get("id").MustString() == field { + metricName += " " + describeMetric(targetMetric.Get("type").MustString(), field) + found = true + } + } + if !found { + metricName = "Unset" + } + + } else if field != "" { metricName += " " + field } @@ -241,7 +255,7 @@ func castToNullFloat(j *simplejson.Json) null.Float { return null.NewFloat(0, false) } -func findAgg(target *QueryBuilder, aggId string) (*simplejson.Json, error) { +func findAgg(target *Query, aggId string) (*simplejson.Json, error) { for _, v := range target.BucketAggs { aggDef := simplejson.NewFromAny(v) if aggId == aggDef.Get("id").MustString() { From 4042e4b225ad4b989f3f1ecabb52271547ff2af2 Mon Sep 17 00:00:00 2001 From: wph95 Date: Tue, 27 Mar 2018 02:12:43 +0800 Subject: [PATCH 06/87] fix a terms bug and add test --- pkg/tsdb/elasticsearch/models.go | 2 +- pkg/tsdb/elasticsearch/query.go | 6 +- pkg/tsdb/elasticsearch/query_test.go | 97 ++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 3 deletions(-) diff --git a/pkg/tsdb/elasticsearch/models.go b/pkg/tsdb/elasticsearch/models.go index 822df2dd4d1..6ab6fa9f43e 100644 --- a/pkg/tsdb/elasticsearch/models.go +++ b/pkg/tsdb/elasticsearch/models.go @@ -41,7 +41,7 @@ type DateHistogramAgg struct { } type FiltersAgg struct { - Filter map[string]interface{} `json:"filter"` + Filters map[string]interface{} `json:"filters"` } type TermsAggSetting struct { diff --git a/pkg/tsdb/elasticsearch/query.go b/pkg/tsdb/elasticsearch/query.go index 51f1ebb5d7a..c4e30cfcbf4 100644 --- a/pkg/tsdb/elasticsearch/query.go +++ b/pkg/tsdb/elasticsearch/query.go @@ -193,15 +193,17 @@ func (q *Query) getHistogramAgg(model *simplejson.Json) *HistogramAgg { func (q *Query) getFilters(model *simplejson.Json) *FiltersAgg { agg := &FiltersAgg{} + agg.Filters = map[string]interface{}{} settings := simplejson.NewFromAny(model.Get("settings").Interface()) - for filter := range settings.Get("filters").MustArray() { + + for _, filter := range settings.Get("filters").MustArray() { filterJson := simplejson.NewFromAny(filter) query := filterJson.Get("query").MustString("") label := filterJson.Get("label").MustString("") if label == "" { label = query } - agg.Filter[label] = newQueryStringFilter(true, query) + agg.Filters[label] = newQueryStringFilter(true, query) } return agg } diff --git a/pkg/tsdb/elasticsearch/query_test.go b/pkg/tsdb/elasticsearch/query_test.go index 992469175b6..4f7b4d9147e 100644 --- a/pkg/tsdb/elasticsearch/query_test.go +++ b/pkg/tsdb/elasticsearch/query_test.go @@ -325,6 +325,103 @@ func TestElasticSearchQueryBuilder(t *testing.T) { "aggs": {"4":{"aggs":{"2":{"aggs":{"1":{"sum":{"field":"value"}}},"date_histogram":{"extended_bounds":{"max":"","min":""},"field":"timestamp","format":"epoch_millis","interval":"1m","min_doc_count":0}}},"terms":{"field":"name_raw","order":{"_term":"desc"},"size":10}}} }` + testElasticSearchResponse(testElasticsearchModelRequestJSON, expectedElasticsearchQueryJSON) + }) + Convey("Test Filters Aggregates", func() { + testElasticsearchModelRequestJSON := ` + { + "bucketAggs": [ + { + "id": "3", + "settings": { + "filters": [{ + "label": "hello", + "query": "host:\"67.65.185.232\"" + }] + }, + "type": "filters" + }, + { + "field": "time", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": 0, + "trimEdges": 0 + }, + "type": "date_histogram" + } + ], + "metrics": [ + { + "pipelineAgg": "select metric", + "field": "bytesSent", + "id": "1", + "meta": {}, + "settings": {}, + "type": "count" + } + ], + "query": "*", + "refId": "A", + "timeField": "time" + }` + + expectedElasticsearchQueryJSON := `{ + "size": 0, + "query": { + "bool": { + "filter": [ + { + "range": { + "time": { + "gte": "", + "lte": "", + "format": "epoch_millis" + } + } + }, + { + "query_string": { + "analyze_wildcard": true, + "query": "*" + } + } + ] + } + }, + "aggs": { + "3": { + "filters": { + "filters": { + "hello": { + "query_string": { + "query": "host:\"67.65.185.232\"", + "analyze_wildcard": true + } + } + } + }, + "aggs": { + "2": { + "date_histogram": { + "interval": "200ms", + "field": "time", + "min_doc_count": 0, + "extended_bounds": { + "min": "", + "max": "" + }, + "format": "epoch_millis" + }, + "aggs": {} + } + } + } + } + } + ` + testElasticSearchResponse(testElasticsearchModelRequestJSON, expectedElasticsearchQueryJSON) }) }) From 06f73321560defb2ac074e4d90af1c94f459943d Mon Sep 17 00:00:00 2001 From: wph95 Date: Wed, 28 Mar 2018 01:42:25 +0800 Subject: [PATCH 07/87] cleanup and add more test --- pkg/tsdb/elasticsearch/elasticsearch_test.go | 121 ++++++++++++ pkg/tsdb/elasticsearch/model_parser.go | 65 ++++++- pkg/tsdb/elasticsearch/models.go | 26 ++- pkg/tsdb/elasticsearch/query.go | 113 +++++------ pkg/tsdb/elasticsearch/query_def.go | 1 - pkg/tsdb/elasticsearch/query_test.go | 186 +------------------ pkg/tsdb/elasticsearch/response_parser.go | 50 +++-- 7 files changed, 274 insertions(+), 288 deletions(-) create mode 100644 pkg/tsdb/elasticsearch/elasticsearch_test.go diff --git a/pkg/tsdb/elasticsearch/elasticsearch_test.go b/pkg/tsdb/elasticsearch/elasticsearch_test.go new file mode 100644 index 00000000000..ad905299166 --- /dev/null +++ b/pkg/tsdb/elasticsearch/elasticsearch_test.go @@ -0,0 +1,121 @@ +package elasticsearch + +import ( + "github.com/grafana/grafana/pkg/components/simplejson" + "time" +) + +var avgWithMovingAvg = Query{ + TimeField: "timestamp", + RawQuery: "(test:query) AND (name:sample)", + Interval: time.Millisecond, + BucketAggs: []*BucketAgg{{ + Field: "timestamp", + ID: "2", + Type: "date_histogram", + Settings: simplejson.NewFromAny(map[string]interface{}{ + "interval": "auto", + "min_doc_count": 0, + "trimEdges": 0, + }), + }}, + Metrics: []*Metric{{ + Field: "value", + ID: "1", + Type: "avg", + Settings: simplejson.NewFromAny(map[string]interface{}{ + "script": map[string]string{ + "inline": "_value * 2", + }, + }), + }, { + Field: "1", + ID: "3", + Type: "moving_avg", + PipelineAggregate: "1", + Settings: simplejson.NewFromAny(map[string]interface{}{ + "minimize": false, + "model": "simple", + "window": 5, + }), + }}, +} + +var wildcardsAndQuotes = Query{ + TimeField: "timestamp", + RawQuery: "scope:$location.leagueconnect.api AND name:*CreateRegistration AND name:\"*.201-responses.rate\"", + Interval: time.Millisecond, + BucketAggs: []*BucketAgg{{ + Field: "timestamp", + ID: "2", + Type: "date_histogram", + Settings: simplejson.NewFromAny(map[string]interface{}{}), + }}, + Metrics: []*Metric{{ + Field: "value", + ID: "1", + Type: "sum", + Settings: simplejson.NewFromAny(map[string]interface{}{}), + }}, +} +var termAggs = Query{ + TimeField: "timestamp", + RawQuery: "(scope:*.hmp.metricsd) AND (name_raw:builtin.general.*_instance_count)", + Interval: time.Millisecond, + BucketAggs: []*BucketAgg{{ + Field: "name_raw", + ID: "4", + Type: "terms", + Settings: simplejson.NewFromAny(map[string]interface{}{ + "order": "desc", + "orderBy": "_term", + "size": "10", + }), + }, { + Field: "timestamp", + ID: "2", + Type: "date_histogram", + Settings: simplejson.NewFromAny(map[string]interface{}{ + "interval": "auto", + "min_doc_count": 0, + "trimEdges": 0, + }), + }}, + Metrics: []*Metric{{ + Field: "value", + ID: "1", + Type: "sum", + Settings: simplejson.NewFromAny(map[string]interface{}{}), + }}, +} + +var filtersAggs = Query{ + TimeField: "time", + RawQuery: "*", + Interval: time.Millisecond, + BucketAggs: []*BucketAgg{{ + ID: "3", + Type: "filters", + Settings: simplejson.NewFromAny(map[string]interface{}{ + "filters": []interface{}{ + map[string]interface{}{"label": "hello", "query": "host:\"67.65.185.232\""}, + }, + }), + }, { + Field: "timestamp", + ID: "2", + Type: "date_histogram", + Settings: simplejson.NewFromAny(map[string]interface{}{ + "interval": "auto", + "min_doc_count": 0, + "trimEdges": 0, + }), + }}, + Metrics: []*Metric{{ + Field: "bytesSent", + ID: "1", + Type: "count", + PipelineAggregate: "select metric", + Settings: simplejson.NewFromAny(map[string]interface{}{}), + }}, +} diff --git a/pkg/tsdb/elasticsearch/model_parser.go b/pkg/tsdb/elasticsearch/model_parser.go index 0d016dc58a5..5d94aebef1a 100644 --- a/pkg/tsdb/elasticsearch/model_parser.go +++ b/pkg/tsdb/elasticsearch/model_parser.go @@ -20,9 +20,15 @@ func (qp *ElasticSearchQueryParser) Parse(model *simplejson.Json, dsInfo *models if err != nil { return nil, err } - rawQuery := model.Get("query").MustString("") - bucketAggs := model.Get("bucketAggs").MustArray() - metrics := model.Get("metrics").MustArray() + rawQuery := model.Get("query").MustString() + bucketAggs, err := qp.parseBucketAggs(model) + if err != nil { + return nil, err + } + metrics, err := qp.parseMetrics(model) + if err != nil { + return nil, err + } alias := model.Get("alias").MustString("") parsedInterval, err := tsdb.GetIntervalFrom(dsInfo, model, time.Millisecond) if err != nil { @@ -37,6 +43,57 @@ func (qp *ElasticSearchQueryParser) Parse(model *simplejson.Json, dsInfo *models parsedInterval}, nil } +func (qp *ElasticSearchQueryParser) parseBucketAggs(model *simplejson.Json) ([]*BucketAgg, error) { + var err error + var result []*BucketAgg + for _, t := range model.Get("bucketAggs").MustArray() { + aggJson := simplejson.NewFromAny(t) + agg := &BucketAgg{} + + agg.Type, err = aggJson.Get("type").String() + if err != nil { + return nil, err + } + + agg.ID, err = aggJson.Get("id").String() + if err != nil { + return nil, err + } + + agg.Field = aggJson.Get("field").MustString() + agg.Settings = simplejson.NewFromAny(aggJson.Get("settings").MustMap()) + + result = append(result, agg) + } + return result, nil +} + +func (qp *ElasticSearchQueryParser) parseMetrics(model *simplejson.Json) ([]*Metric, error) { + var err error + var result []*Metric + for _, t := range model.Get("metrics").MustArray() { + metricJson := simplejson.NewFromAny(t) + metric := &Metric{} + + metric.Field = metricJson.Get("field").MustString() + metric.Hide = metricJson.Get("hide").MustBool(false) + metric.ID, err = metricJson.Get("id").String() + if err != nil { + return nil, err + } + + metric.PipelineAggregate = metricJson.Get("pipelineAgg").MustString() + metric.Settings = simplejson.NewFromAny(metricJson.Get("settings").MustMap()) + + metric.Type, err = metricJson.Get("type").String() + if err != nil { + return nil, err + } + + result = append(result, metric) + } + return result, nil +} func getRequestHeader(timeRange *tsdb.TimeRange, dsInfo *models.DataSource) *QueryHeader { var header QueryHeader esVersion := dsInfo.JsonData.Get("esVersion").MustInt() @@ -47,7 +104,7 @@ func getRequestHeader(timeRange *tsdb.TimeRange, dsInfo *models.DataSource) *Que } header.SearchType = searchType header.IgnoreUnavailable = true - header.Index = getIndexList(dsInfo.Database, dsInfo.JsonData.Get("interval").MustString(""), timeRange) + header.Index = getIndexList(dsInfo.Database, dsInfo.JsonData.Get("interval").MustString(), timeRange) if esVersion >= 56 { header.MaxConcurrentShardRequests = dsInfo.JsonData.Get("maxConcurrentShardRequests").MustInt() diff --git a/pkg/tsdb/elasticsearch/models.go b/pkg/tsdb/elasticsearch/models.go index 6ab6fa9f43e..9cf295cbd0e 100644 --- a/pkg/tsdb/elasticsearch/models.go +++ b/pkg/tsdb/elasticsearch/models.go @@ -7,6 +7,22 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" ) +type BucketAgg struct { + Field string `json:"field"` + ID string `json:"id"` + Settings *simplejson.Json `json:"settings"` + Type string `jsons:"type"` +} + +type Metric struct { + Field string `json:"field"` + Hide bool `json:"hide"` + ID string `json:"id"` + PipelineAggregate string `json:"pipelineAgg"` + Settings *simplejson.Json `json:"settings"` + Type string `json:"type"` +} + type QueryHeader struct { SearchType string `json:"search_type"` IgnoreUnavailable bool `json:"ignore_unavailable"` @@ -44,16 +60,16 @@ type FiltersAgg struct { Filters map[string]interface{} `json:"filters"` } -type TermsAggSetting struct { +type TermsAgg struct { Field string `json:"field"` Size int `json:"size"` Order map[string]interface{} `json:"order"` Missing string `json:"missing,omitempty"` } -type TermsAgg struct { - Terms TermsAggSetting `json:"terms"` - Aggs Aggs `json:"aggs"` +type TermsAggWrap struct { + Terms TermsAgg `json:"terms"` + Aggs Aggs `json:"aggs"` } type ExtendedBounds struct { @@ -91,8 +107,6 @@ type BoolQuery struct { Filter []interface{} `json:"filter"` } -type Metric map[string]interface{} - type Responses struct { Responses []Response `json:"responses"` } diff --git a/pkg/tsdb/elasticsearch/query.go b/pkg/tsdb/elasticsearch/query.go index c4e30cfcbf4..a63529df2df 100644 --- a/pkg/tsdb/elasticsearch/query.go +++ b/pkg/tsdb/elasticsearch/query.go @@ -18,11 +18,11 @@ var rangeFilterSetting = RangeFilterSetting{Gte: "$timeFrom", Format: "epoch_millis"} type Query struct { - TimeField string `json:"timeField"` - RawQuery string `json:"query"` - BucketAggs []interface{} `json:"bucketAggs"` - Metrics []interface{} `json:"metrics"` - Alias string `json:"Alias"` + TimeField string `json:"timeField"` + RawQuery string `json:"query"` + BucketAggs []*BucketAgg `json:"bucketAggs"` + Metrics []*Metric `json:"metrics"` + Alias string `json:"Alias"` Interval time.Duration } @@ -73,27 +73,17 @@ func (q *Query) renderReqQuery(req *Request) { func (q *Query) parseAggs(req *Request) error { aggs := make(Aggs) nestedAggs := aggs - for _, aggRaw := range q.BucketAggs { + for _, agg := range q.BucketAggs { esAggs := make(Aggs) - aggJson := simplejson.NewFromAny(aggRaw) - aggType, err := aggJson.Get("type").String() - if err != nil { - return err - } - id, err := aggJson.Get("id").String() - if err != nil { - return err - } - - switch aggType { + switch agg.Type { case "date_histogram": - esAggs["date_histogram"] = q.getDateHistogramAgg(aggJson) + esAggs["date_histogram"] = q.getDateHistogramAgg(agg) case "histogram": - esAggs["histogram"] = q.getHistogramAgg(aggJson) + esAggs["histogram"] = q.getHistogramAgg(agg) case "filters": - esAggs["filters"] = q.getFilters(aggJson) + esAggs["filters"] = q.getFilters(agg) case "terms": - terms := q.getTerms(aggJson) + terms := q.getTerms(agg) esAggs["terms"] = terms.Terms esAggs["aggs"] = terms.Aggs case "geohash_grid": @@ -105,59 +95,47 @@ func (q *Query) parseAggs(req *Request) error { } if aggs, ok := (nestedAggs["aggs"]).(Aggs); ok { - aggs[id] = esAggs + aggs[agg.ID] = esAggs } nestedAggs = esAggs } nestedAggs["aggs"] = make(Aggs) - for _, metricRaw := range q.Metrics { - metric := make(Metric) - metricJson := simplejson.NewFromAny(metricRaw) + for _, metric := range q.Metrics { + subAgg := make(Aggs) - id, err := metricJson.Get("id").String() - if err != nil { - return err - } - metricType, err := metricJson.Get("type").String() - if err != nil { - return err - } - if metricType == "count" { + if metric.Type == "count" { continue } + settings := metric.Settings.MustMap(make(map[string]interface{})) - settings := metricJson.Get("settings").MustMap(map[string]interface{}{}) - - if isPipelineAgg(metricType) { - pipelineAgg := metricJson.Get("pipelineAgg").MustString("") - if _, err := strconv.Atoi(pipelineAgg); err == nil { - settings["buckets_path"] = pipelineAgg + if isPipelineAgg(metric.Type) { + if _, err := strconv.Atoi(metric.PipelineAggregate); err == nil { + settings["buckets_path"] = metric.PipelineAggregate } else { continue } } else { - settings["field"] = metricJson.Get("field").MustString() + settings["field"] = metric.Field } - metric[metricType] = settings - nestedAggs["aggs"].(Aggs)[id] = metric + subAgg[metric.Type] = settings + nestedAggs["aggs"].(Aggs)[metric.ID] = subAgg } req.Aggs = aggs["aggs"].(Aggs) return nil } -func (q *Query) getDateHistogramAgg(model *simplejson.Json) *DateHistogramAgg { +func (q *Query) getDateHistogramAgg(target *BucketAgg) *DateHistogramAgg { agg := &DateHistogramAgg{} - settings := simplejson.NewFromAny(model.Get("settings").Interface()) - interval, err := settings.Get("interval").String() + interval, err := target.Settings.Get("interval").String() if err == nil { agg.Interval = interval } agg.Field = q.TimeField - agg.MinDocCount = settings.Get("min_doc_count").MustInt(0) + agg.MinDocCount = target.Settings.Get("min_doc_count").MustInt(0) agg.ExtendedBounds = ExtendedBounds{"$timeFrom", "$timeTo"} agg.Format = "epoch_millis" @@ -165,66 +143,63 @@ func (q *Query) getDateHistogramAgg(model *simplejson.Json) *DateHistogramAgg { agg.Interval = "$__interval" } - missing, err := settings.Get("missing").String() + missing, err := target.Settings.Get("missing").String() if err == nil { agg.Missing = missing } return agg } -func (q *Query) getHistogramAgg(model *simplejson.Json) *HistogramAgg { +func (q *Query) getHistogramAgg(target *BucketAgg) *HistogramAgg { agg := &HistogramAgg{} - settings := simplejson.NewFromAny(model.Get("settings").Interface()) - interval, err := settings.Get("interval").String() + interval, err := target.Settings.Get("interval").String() if err == nil { agg.Interval = interval } - field, err := model.Get("field").String() - if err == nil { - agg.Field = field + + if target.Field != "" { + agg.Field = target.Field } - agg.MinDocCount = settings.Get("min_doc_count").MustInt(0) - missing, err := settings.Get("missing").String() + agg.MinDocCount = target.Settings.Get("min_doc_count").MustInt(0) + missing, err := target.Settings.Get("missing").String() if err == nil { agg.Missing = missing } return agg } -func (q *Query) getFilters(model *simplejson.Json) *FiltersAgg { +func (q *Query) getFilters(target *BucketAgg) *FiltersAgg { agg := &FiltersAgg{} agg.Filters = map[string]interface{}{} - settings := simplejson.NewFromAny(model.Get("settings").Interface()) - - for _, filter := range settings.Get("filters").MustArray() { + for _, filter := range target.Settings.Get("filters").MustArray() { filterJson := simplejson.NewFromAny(filter) query := filterJson.Get("query").MustString("") label := filterJson.Get("label").MustString("") if label == "" { label = query } + agg.Filters[label] = newQueryStringFilter(true, query) } return agg } -func (q *Query) getTerms(model *simplejson.Json) *TermsAgg { - agg := &TermsAgg{Aggs: make(Aggs)} - settings := simplejson.NewFromAny(model.Get("settings").Interface()) - agg.Terms.Field = model.Get("field").MustString() - if settings == nil { +func (q *Query) getTerms(target *BucketAgg) *TermsAggWrap { + agg := &TermsAggWrap{Aggs: make(Aggs)} + agg.Terms.Field = target.Field + if len(target.Settings.MustMap()) == 0 { return agg } - sizeStr := settings.Get("size").MustString("") + sizeStr := target.Settings.Get("size").MustString("") size, err := strconv.Atoi(sizeStr) if err != nil { size = 500 } agg.Terms.Size = size - orderBy, err := settings.Get("orderBy").String() + orderBy, err := target.Settings.Get("orderBy").String() if err == nil { agg.Terms.Order = make(map[string]interface{}) - agg.Terms.Order[orderBy] = settings.Get("order").MustString("") + agg.Terms.Order[orderBy] = target.Settings.Get("order").MustString("") if _, err := strconv.Atoi(orderBy); err != nil { for _, metricI := range q.Metrics { metric := simplejson.NewFromAny(metricI) @@ -242,7 +217,7 @@ func (q *Query) getTerms(model *simplejson.Json) *TermsAgg { } } - missing, err := settings.Get("missing").String() + missing, err := target.Settings.Get("missing").String() if err == nil { agg.Terms.Missing = missing } diff --git a/pkg/tsdb/elasticsearch/query_def.go b/pkg/tsdb/elasticsearch/query_def.go index 6f78f02f346..128e752d97a 100644 --- a/pkg/tsdb/elasticsearch/query_def.go +++ b/pkg/tsdb/elasticsearch/query_def.go @@ -40,5 +40,4 @@ func isPipelineAgg(metricType string) bool { func describeMetric(metricType, field string) string { text := metricAggType[metricType] return text + " " + field - } diff --git a/pkg/tsdb/elasticsearch/query_test.go b/pkg/tsdb/elasticsearch/query_test.go index 4f7b4d9147e..aecca9f4734 100644 --- a/pkg/tsdb/elasticsearch/query_test.go +++ b/pkg/tsdb/elasticsearch/query_test.go @@ -13,13 +13,8 @@ import ( "testing" ) -func testElasticSearchResponse(requestJSON string, expectedElasticSearchRequestJSON string) { +func testElasticSearchResponse(query Query, expectedElasticSearchRequestJSON string) { var queryExpectedJSONInterface, queryJSONInterface interface{} - parser := ElasticSearchQueryParser{} - model := &Query{} - - err := json.Unmarshal([]byte(requestJSON), model) - So(err, ShouldBeNil) jsonDate, _ := simplejson.NewJson([]byte(`{"esVersion":2}`)) dsInfo := &models.DataSource{ Database: "grafana-test", @@ -28,10 +23,8 @@ func testElasticSearchResponse(requestJSON string, expectedElasticSearchRequestJ testTimeRange := tsdb.NewTimeRange("5m", "now") - req, _ := simplejson.NewJson([]byte(requestJSON)) - query, err := parser.Parse(req, dsInfo) s, err := query.Build(&tsdb.TsdbQuery{TimeRange: testTimeRange}, dsInfo) - + So(err, ShouldBeNil) queryJSON := strings.Split(s, "\n")[1] err = json.Unmarshal([]byte(queryJSON), &queryJSONInterface) So(err, ShouldBeNil) @@ -62,53 +55,6 @@ func testElasticSearchResponse(requestJSON string, expectedElasticSearchRequestJ func TestElasticSearchQueryBuilder(t *testing.T) { Convey("Elasticsearch QueryBuilder query testing", t, func() { Convey("Build test average metric with moving average", func() { - var testElasticsearchModelRequestJSON = ` - { - "bucketAggs": [ - { - "field": "timestamp", - "id": "2", - "settings": { - "interval": "auto", - "min_doc_count": 0, - "trimEdges": 0 - }, - "type": "date_histogram" - } - ], - "dsType": "elasticsearch", - "metrics": [ - { - "field": "value", - "id": "1", - "inlineScript": "_value * 2", - "meta": {}, - "settings": { - "script": { - "inline": "_value * 2" - } - }, - "type": "avg" - }, - { - "field": "1", - "id": "3", - "meta": {}, - "pipelineAgg": "1", - "settings": { - "minimize": false, - "model": "simple", - "window": 5 - }, - "type": "moving_avg" - } - ], - "query": "(test:query) AND (name:sample)", - "refId": "A", - "timeField": "timestamp" - } - ` - var expectedElasticsearchQueryJSON = ` { "size": 0, @@ -167,32 +113,9 @@ func TestElasticSearchQueryBuilder(t *testing.T) { } }` - testElasticSearchResponse(testElasticsearchModelRequestJSON, expectedElasticsearchQueryJSON) + testElasticSearchResponse(avgWithMovingAvg, expectedElasticsearchQueryJSON) }) Convey("Test Wildcards and Quotes", func() { - testElasticsearchModelRequestJSON := ` - { - "alias": "New", - "bucketAggs": [ - { - "field": "timestamp", - "id": "2", - "type": "date_histogram" - } - ], - "dsType": "elasticsearch", - "metrics": [ - { - "type": "sum", - "field": "value", - "id": "1" - } - ], - "query": "scope:$location.leagueconnect.api AND name:*CreateRegistration AND name:\"*.201-responses.rate\"", - "refId": "A", - "timeField": "timestamp" - }` - expectedElasticsearchQueryJSON := ` { "size": 0, @@ -239,65 +162,9 @@ func TestElasticSearchQueryBuilder(t *testing.T) { } }` - testElasticSearchResponse(testElasticsearchModelRequestJSON, expectedElasticsearchQueryJSON) + testElasticSearchResponse(wildcardsAndQuotes, expectedElasticsearchQueryJSON) }) Convey("Test Term Aggregates", func() { - testElasticsearchModelRequestJSON := ` - { - "bucketAggs": [{ - "field": "name_raw", - "id": "4", - "settings": { - "order": "desc", - "orderBy": "_term", - "size": "10" - }, - "type": "terms" - }, { - "field": "timestamp", - "id": "2", - "settings": { - "interval": "1m", - "min_doc_count": 0, - "trimEdges": 0 - }, - "type": "date_histogram" - }], - "dsType": "elasticsearch", - "filters": [{ - "boolOp": "AND", - "not": false, - "type": "rfc190Scope", - "value": "*.hmp.metricsd" - }, { - "boolOp": "AND", - "not": false, - "type": "name_raw", - "value": "builtin.general.*_instance_count" - }], - "metricObject": {}, - "metrics": [{ - "field": "value", - "id": "1", - "meta": {}, - "options": {}, - "settings": {}, - "type": "sum" - }], - "mode": 0, - "numToGraph": 10, - "prependHostName": false, - "query": "(scope:*.hmp.metricsd) AND (name_raw:builtin.general.*_instance_count)", - "refId": "A", - "regexAlias": false, - "selectedApplication": "", - "selectedHost": "", - "selectedLocation": "", - "timeField": "timestamp", - "useFullHostName": "", - "useQuery": false - }` - expectedElasticsearchQueryJSON := ` { "size": 0, @@ -322,51 +189,12 @@ func TestElasticSearchQueryBuilder(t *testing.T) { ] } }, - "aggs": {"4":{"aggs":{"2":{"aggs":{"1":{"sum":{"field":"value"}}},"date_histogram":{"extended_bounds":{"max":"","min":""},"field":"timestamp","format":"epoch_millis","interval":"1m","min_doc_count":0}}},"terms":{"field":"name_raw","order":{"_term":"desc"},"size":10}}} + "aggs": {"4":{"aggs":{"2":{"aggs":{"1":{"sum":{"field":"value"}}},"date_histogram":{"extended_bounds":{"max":"","min":""},"field":"timestamp","format":"epoch_millis","interval":"200ms","min_doc_count":0}}},"terms":{"field":"name_raw","order":{"_term":"desc"},"size":10}}} }` - testElasticSearchResponse(testElasticsearchModelRequestJSON, expectedElasticsearchQueryJSON) + testElasticSearchResponse(termAggs, expectedElasticsearchQueryJSON) }) Convey("Test Filters Aggregates", func() { - testElasticsearchModelRequestJSON := ` - { - "bucketAggs": [ - { - "id": "3", - "settings": { - "filters": [{ - "label": "hello", - "query": "host:\"67.65.185.232\"" - }] - }, - "type": "filters" - }, - { - "field": "time", - "id": "2", - "settings": { - "interval": "auto", - "min_doc_count": 0, - "trimEdges": 0 - }, - "type": "date_histogram" - } - ], - "metrics": [ - { - "pipelineAgg": "select metric", - "field": "bytesSent", - "id": "1", - "meta": {}, - "settings": {}, - "type": "count" - } - ], - "query": "*", - "refId": "A", - "timeField": "time" - }` - expectedElasticsearchQueryJSON := `{ "size": 0, "query": { @@ -422,7 +250,7 @@ func TestElasticSearchQueryBuilder(t *testing.T) { } ` - testElasticSearchResponse(testElasticsearchModelRequestJSON, expectedElasticsearchQueryJSON) + testElasticSearchResponse(filtersAggs, expectedElasticsearchQueryJSON) }) }) } diff --git a/pkg/tsdb/elasticsearch/response_parser.go b/pkg/tsdb/elasticsearch/response_parser.go index 01b8cb1d235..24d5ebebfc4 100644 --- a/pkg/tsdb/elasticsearch/response_parser.go +++ b/pkg/tsdb/elasticsearch/response_parser.go @@ -40,27 +40,26 @@ func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{ } if depth == maxDepth { - if aggDef.Get("type").MustString() == "date_histogram" { + if aggDef.Type == "date_histogram" { err = rp.processMetrics(esAgg, target, series, props) if err != nil { return err } } else { - return fmt.Errorf("not support type:%s", aggDef.Get("type").MustString()) + return fmt.Errorf("not support type:%s", aggDef.Type) } } else { for i, b := range esAgg.Get("buckets").MustArray() { - field := aggDef.Get("field").MustString() bucket := simplejson.NewFromAny(b) newProps := props if key, err := bucket.Get("key").String(); err == nil { - newProps[field] = key + newProps[aggDef.Field] = key } else { props["filter"] = strconv.Itoa(i) } if key, err := bucket.Get("key_as_string").String(); err == nil { - props[field] = key + props[aggDef.Field] = key } rp.processBuckets(bucket.MustMap(), target, series, newProps, depth+1) } @@ -72,17 +71,12 @@ func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{ } func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, target *Query, series *[]*tsdb.TimeSeries, props map[string]string) error { - for _, v := range target.Metrics { - metric := simplejson.NewFromAny(v) - if metric.Get("hide").MustBool(false) { + for _, metric := range target.Metrics { + if metric.Hide { continue } - metricId := metric.Get("id").MustString() - metricField := metric.Get("field").MustString() - metricType := metric.Get("type").MustString() - - switch metricType { + switch metric.Type { case "count": newSeries := tsdb.TimeSeries{} for _, v := range esAgg.Get("buckets").MustArray() { @@ -102,16 +96,16 @@ func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, ta } firstBucket := simplejson.NewFromAny(buckets[0]) - percentiles := firstBucket.GetPath(metricId, "values").MustMap() + percentiles := firstBucket.GetPath(metric.ID, "values").MustMap() for percentileName := range percentiles { newSeries := tsdb.TimeSeries{} newSeries.Tags = props newSeries.Tags["metric"] = "p" + percentileName - newSeries.Tags["field"] = metricField + newSeries.Tags["field"] = metric.Field for _, v := range buckets { bucket := simplejson.NewFromAny(v) - value := castToNullFloat(bucket.GetPath(metricId, "values", percentileName)) + value := castToNullFloat(bucket.GetPath(metric.ID, "values", percentileName)) key := castToNullFloat(bucket.Get("key")) newSeries.Points = append(newSeries.Points, tsdb.TimePoint{value, key}) } @@ -120,20 +114,20 @@ func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, ta default: newSeries := tsdb.TimeSeries{} newSeries.Tags = props - newSeries.Tags["metric"] = metricType - newSeries.Tags["field"] = metricField + newSeries.Tags["metric"] = metric.Type + newSeries.Tags["field"] = metric.Field for _, v := range esAgg.Get("buckets").MustArray() { bucket := simplejson.NewFromAny(v) key := castToNullFloat(bucket.Get("key")) - valueObj, err := bucket.Get(metricId).Map() + valueObj, err := bucket.Get(metric.ID).Map() if err != nil { break } var value null.Float if _, ok := valueObj["normalized_value"]; ok { - value = castToNullFloat(bucket.GetPath(metricId, "normalized_value")) + value = castToNullFloat(bucket.GetPath(metric.ID, "normalized_value")) } else { - value = castToNullFloat(bucket.GetPath(metricId, "value")) + value = castToNullFloat(bucket.GetPath(metric.ID, "value")) } newSeries.Points = append(newSeries.Points, tsdb.TimePoint{value, key}) } @@ -196,10 +190,9 @@ func (rp *ElasticsearchResponseParser) getSeriesName(series *tsdb.TimeSeries, ta // todo, if field and pipelineAgg if field != "" && isPipelineAgg(metricType) { found := false - for _, targetMetricI := range target.Metrics { - targetMetric := simplejson.NewFromAny(targetMetricI) - if targetMetric.Get("id").MustString() == field { - metricName += " " + describeMetric(targetMetric.Get("type").MustString(), field) + for _, metric := range target.Metrics { + if metric.ID == field { + metricName += " " + describeMetric(metric.Type, field) found = true } } @@ -255,11 +248,10 @@ func castToNullFloat(j *simplejson.Json) null.Float { return null.NewFloat(0, false) } -func findAgg(target *Query, aggId string) (*simplejson.Json, error) { +func findAgg(target *Query, aggId string) (*BucketAgg, error) { for _, v := range target.BucketAggs { - aggDef := simplejson.NewFromAny(v) - if aggId == aggDef.Get("id").MustString() { - return aggDef, nil + if aggId == v.ID { + return v, nil } } return nil, errors.New("can't found aggDef, aggID:" + aggId) From 4050fce2205f9fdb6c217eae49686730cabd92c7 Mon Sep 17 00:00:00 2001 From: wph95 Date: Wed, 28 Mar 2018 12:35:05 +0800 Subject: [PATCH 08/87] add response_parser test --- pkg/tsdb/elasticsearch/response_parser.go | 9 +- .../elasticsearch/response_parser_test.go | 109 ++++++++++++++++++ 2 files changed, 116 insertions(+), 2 deletions(-) create mode 100644 pkg/tsdb/elasticsearch/response_parser_test.go diff --git a/pkg/tsdb/elasticsearch/response_parser.go b/pkg/tsdb/elasticsearch/response_parser.go index 24d5ebebfc4..ec7d2f9eb08 100644 --- a/pkg/tsdb/elasticsearch/response_parser.go +++ b/pkg/tsdb/elasticsearch/response_parser.go @@ -30,6 +30,7 @@ func (rp *ElasticsearchResponseParser) getTimeSeries() *tsdb.QueryResult { } func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{}, target *Query, series *[]*tsdb.TimeSeries, props map[string]string, depth int) error { + var err error maxDepth := len(target.BucketAggs) - 1 for aggId, v := range aggs { @@ -113,7 +114,11 @@ func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, ta } default: newSeries := tsdb.TimeSeries{} - newSeries.Tags = props + newSeries.Tags = map[string]string{} + for k, v := range props { + newSeries.Tags[k] = v + } + newSeries.Tags["metric"] = metric.Type newSeries.Tags["field"] = metric.Field for _, v := range esAgg.Get("buckets").MustArray() { @@ -121,7 +126,7 @@ func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, ta key := castToNullFloat(bucket.Get("key")) valueObj, err := bucket.Get(metric.ID).Map() if err != nil { - break + continue } var value null.Float if _, ok := valueObj["normalized_value"]; ok { diff --git a/pkg/tsdb/elasticsearch/response_parser_test.go b/pkg/tsdb/elasticsearch/response_parser_test.go new file mode 100644 index 00000000000..c5b877c1925 --- /dev/null +++ b/pkg/tsdb/elasticsearch/response_parser_test.go @@ -0,0 +1,109 @@ +package elasticsearch + +import ( + "encoding/json" + "github.com/grafana/grafana/pkg/tsdb" + . "github.com/smartystreets/goconvey/convey" + "testing" +) + +func testElasticsearchResponse(body string, target Query) *tsdb.QueryResult { + var responses Responses + err := json.Unmarshal([]byte(body), &responses) + So(err, ShouldBeNil) + + responseParser := ElasticsearchResponseParser{responses.Responses, []*Query{&target}} + return responseParser.getTimeSeries() +} + +func TestElasticSearchResponseParser(t *testing.T) { + Convey("Elasticsearch Response query testing", t, func() { + Convey("Build test average metric with moving average", func() { + responses := `{ + "responses": [ + { + "took": 1, + "timed_out": false, + "_shards": { + "total": 5, + "successful": 5, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": 4500, + "max_score": 0, + "hits": [] + }, + "aggregations": { + "2": { + "buckets": [ + { + "1": { + "value": null + }, + "key_as_string": "1522205880000", + "key": 1522205880000, + "doc_count": 0 + }, + { + "1": { + "value": 10 + }, + "key_as_string": "1522205940000", + "key": 1522205940000, + "doc_count": 300 + }, + { + "1": { + "value": 10 + }, + "3": { + "value": 20 + }, + "key_as_string": "1522206000000", + "key": 1522206000000, + "doc_count": 300 + }, + { + "1": { + "value": 10 + }, + "3": { + "value": 20 + }, + "key_as_string": "1522206060000", + "key": 1522206060000, + "doc_count": 300 + } + ] + } + }, + "status": 200 + } + ] +} +` + res := testElasticsearchResponse(responses, avgWithMovingAvg) + So(len(res.Series), ShouldEqual, 2) + So(res.Series[0].Name, ShouldEqual, "Average value") + So(len(res.Series[0].Points), ShouldEqual, 4) + for i, p := range res.Series[0].Points { + if i == 0 { + So(p[0].Valid, ShouldBeFalse) + } else { + So(p[0].Float64, ShouldEqual, 10) + } + So(p[1].Float64, ShouldEqual, 1522205880000+60000*i) + } + + So(res.Series[1].Name, ShouldEqual, "Moving Average Average 1") + So(len(res.Series[1].Points), ShouldEqual, 2) + + for _, p := range res.Series[1].Points { + So(p[0].Float64, ShouldEqual, 20) + } + + }) + }) +} From 2e67e3ba633534e3218697726ad18b81eb11e53f Mon Sep 17 00:00:00 2001 From: Alexandre Georges Date: Sun, 20 May 2018 14:07:40 +0200 Subject: [PATCH 09/87] Added Swiss franc currency --- public/app/core/utils/kbn.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/public/app/core/utils/kbn.ts b/public/app/core/utils/kbn.ts index ff92bb5c77a..7102c78fa9a 100644 --- a/public/app/core/utils/kbn.ts +++ b/public/app/core/utils/kbn.ts @@ -448,6 +448,7 @@ kbn.valueFormats.currencyISK = kbn.formatBuilders.currency('kr'); kbn.valueFormats.currencyNOK = kbn.formatBuilders.currency('kr'); kbn.valueFormats.currencySEK = kbn.formatBuilders.currency('kr'); kbn.valueFormats.currencyCZK = kbn.formatBuilders.currency('czk'); +kbn.valueFormats.currencyCHF = kbn.formatBuilders.currency('CHF'); // Data (Binary) kbn.valueFormats.bits = kbn.formatBuilders.binarySIPrefix('b'); @@ -873,6 +874,7 @@ kbn.getUnitFormats = function() { { text: 'Norwegian Krone (kr)', value: 'currencyNOK' }, { text: 'Swedish Krona (kr)', value: 'currencySEK' }, { text: 'Czech koruna (czk)', value: 'currencyCZK' }, + { text: 'Swiss franc (CHF)', value: 'currencyCHF' }, ], }, { From 0cfdd726f777a6e913e1f2d8fbe8f4ece1dbf584 Mon Sep 17 00:00:00 2001 From: mammuthus Date: Sun, 20 May 2018 21:28:53 +0300 Subject: [PATCH 10/87] Singlestat value: vertical alignment fix There is a problem with vertical alignment of Singlestat value - it's a bit lower then it has to be. This hack fix it. --- public/sass/components/_panel_singlestat.scss | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/public/sass/components/_panel_singlestat.scss b/public/sass/components/_panel_singlestat.scss index 33a956a0244..c84234bde9f 100644 --- a/public/sass/components/_panel_singlestat.scss +++ b/public/sass/components/_panel_singlestat.scss @@ -10,10 +10,15 @@ display: table-cell; vertical-align: middle; text-align: center; - position: relative; z-index: 1; font-size: 3em; font-weight: bold; + margin: 0; + position: absolute; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); + padding-bottom: 10px; } .singlestat-panel-prefix { From 77400cef08b3c868bea9d4106c7fcbb82d3d0e65 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 23 May 2018 14:36:41 +0200 Subject: [PATCH 11/87] elasticsearch: refactor and cleanup Move time series query logic to specific file. Remove model parser and move to time series query file, adds parser test. --- pkg/tsdb/elasticsearch/elasticsearch.go | 105 +--------- pkg/tsdb/elasticsearch/model_parser.go | 153 --------------- pkg/tsdb/elasticsearch/model_parser_test.go | 49 ----- pkg/tsdb/elasticsearch/query.go | 71 ++++++- pkg/tsdb/elasticsearch/query_test.go | 49 ++++- .../elasticsearch/response_parser_test.go | 3 +- pkg/tsdb/elasticsearch/time_series_query.go | 182 ++++++++++++++++++ .../elasticsearch/time_series_query_test.go | 118 ++++++++++++ 8 files changed, 421 insertions(+), 309 deletions(-) delete mode 100644 pkg/tsdb/elasticsearch/model_parser.go delete mode 100644 pkg/tsdb/elasticsearch/model_parser_test.go create mode 100644 pkg/tsdb/elasticsearch/time_series_query.go create mode 100644 pkg/tsdb/elasticsearch/time_series_query_test.go diff --git a/pkg/tsdb/elasticsearch/elasticsearch.go b/pkg/tsdb/elasticsearch/elasticsearch.go index 0ce9eca0972..abf25feac06 100644 --- a/pkg/tsdb/elasticsearch/elasticsearch.go +++ b/pkg/tsdb/elasticsearch/elasticsearch.go @@ -1,27 +1,20 @@ package elasticsearch import ( - "bytes" "context" - "encoding/json" - "errors" "fmt" - "github.com/grafana/grafana/pkg/log" - "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/setting" - "github.com/grafana/grafana/pkg/tsdb" - "golang.org/x/net/context/ctxhttp" "net/http" "net/url" "path" "strings" "time" + + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/tsdb" ) -type ElasticsearchExecutor struct { - QueryParser *ElasticSearchQueryParser - Transport *http.Transport -} +type ElasticsearchExecutor struct{} var ( glog log.Logger @@ -29,14 +22,7 @@ var ( ) func NewElasticsearchExecutor(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) { - transport, err := dsInfo.GetHttpTransport() - if err != nil { - return nil, err - } - - return &ElasticsearchExecutor{ - Transport: transport, - }, nil + return &ElasticsearchExecutor{}, nil } func init() { @@ -46,84 +32,11 @@ func init() { } func (e *ElasticsearchExecutor) Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) { - result := &tsdb.Response{} - result.Results = make(map[string]*tsdb.QueryResult) - - queries, err := e.getQuery(dsInfo, tsdbQuery) - if err != nil { - return nil, err + if len(tsdbQuery.Queries) == 0 { + return nil, fmt.Errorf("query contains no queries") } - buff := bytes.Buffer{} - for _, q := range queries { - s, err := q.Build(tsdbQuery, dsInfo) - if err != nil { - return nil, err - } - buff.WriteString(s) - } - payload := buff.String() - - if setting.Env == setting.DEV { - glog.Debug("Elasticsearch playload", "raw playload", payload) - } - glog.Info("Elasticsearch playload", "raw playload", payload) - - req, err := e.createRequest(dsInfo, payload) - if err != nil { - return nil, err - } - - httpClient, err := dsInfo.GetHttpClient() - if err != nil { - return nil, err - } - - resp, err := ctxhttp.Do(ctx, httpClient, req) - if err != nil { - return nil, err - } - - if resp.StatusCode/100 != 2 { - return nil, fmt.Errorf("elasticsearch returned statuscode invalid status code: %v", resp.Status) - } - - var responses Responses - dec := json.NewDecoder(resp.Body) - defer resp.Body.Close() - dec.UseNumber() - err = dec.Decode(&responses) - if err != nil { - return nil, err - } - - for _, res := range responses.Responses { - if res.Err != nil { - return nil, errors.New(res.getErrMsg()) - } - } - responseParser := ElasticsearchResponseParser{responses.Responses, queries} - queryRes := responseParser.getTimeSeries() - result.Results["A"] = queryRes - return result, nil -} - -func (e *ElasticsearchExecutor) getQuery(dsInfo *models.DataSource, context *tsdb.TsdbQuery) ([]*Query, error) { - queries := make([]*Query, 0) - if len(context.Queries) == 0 { - return nil, fmt.Errorf("query request contains no queries") - } - for _, v := range context.Queries { - - query, err := e.QueryParser.Parse(v.Model, dsInfo) - if err != nil { - return nil, err - } - queries = append(queries, query) - - } - return queries, nil - + return e.executeTimeSeriesQuery(ctx, dsInfo, tsdbQuery) } func (e *ElasticsearchExecutor) createRequest(dsInfo *models.DataSource, query string) (*http.Request, error) { diff --git a/pkg/tsdb/elasticsearch/model_parser.go b/pkg/tsdb/elasticsearch/model_parser.go deleted file mode 100644 index 5d94aebef1a..00000000000 --- a/pkg/tsdb/elasticsearch/model_parser.go +++ /dev/null @@ -1,153 +0,0 @@ -package elasticsearch - -import ( - "fmt" - "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/tsdb" - "github.com/leibowitz/moment" - "strings" - "time" -) - -type ElasticSearchQueryParser struct { -} - -func (qp *ElasticSearchQueryParser) Parse(model *simplejson.Json, dsInfo *models.DataSource) (*Query, error) { - //payload := bytes.Buffer{} - //queryHeader := qp.getQueryHeader() - timeField, err := model.Get("timeField").String() - if err != nil { - return nil, err - } - rawQuery := model.Get("query").MustString() - bucketAggs, err := qp.parseBucketAggs(model) - if err != nil { - return nil, err - } - metrics, err := qp.parseMetrics(model) - if err != nil { - return nil, err - } - alias := model.Get("alias").MustString("") - parsedInterval, err := tsdb.GetIntervalFrom(dsInfo, model, time.Millisecond) - if err != nil { - return nil, err - } - - return &Query{timeField, - rawQuery, - bucketAggs, - metrics, - alias, - parsedInterval}, nil -} - -func (qp *ElasticSearchQueryParser) parseBucketAggs(model *simplejson.Json) ([]*BucketAgg, error) { - var err error - var result []*BucketAgg - for _, t := range model.Get("bucketAggs").MustArray() { - aggJson := simplejson.NewFromAny(t) - agg := &BucketAgg{} - - agg.Type, err = aggJson.Get("type").String() - if err != nil { - return nil, err - } - - agg.ID, err = aggJson.Get("id").String() - if err != nil { - return nil, err - } - - agg.Field = aggJson.Get("field").MustString() - agg.Settings = simplejson.NewFromAny(aggJson.Get("settings").MustMap()) - - result = append(result, agg) - } - return result, nil -} - -func (qp *ElasticSearchQueryParser) parseMetrics(model *simplejson.Json) ([]*Metric, error) { - var err error - var result []*Metric - for _, t := range model.Get("metrics").MustArray() { - metricJson := simplejson.NewFromAny(t) - metric := &Metric{} - - metric.Field = metricJson.Get("field").MustString() - metric.Hide = metricJson.Get("hide").MustBool(false) - metric.ID, err = metricJson.Get("id").String() - if err != nil { - return nil, err - } - - metric.PipelineAggregate = metricJson.Get("pipelineAgg").MustString() - metric.Settings = simplejson.NewFromAny(metricJson.Get("settings").MustMap()) - - metric.Type, err = metricJson.Get("type").String() - if err != nil { - return nil, err - } - - result = append(result, metric) - } - return result, nil -} -func getRequestHeader(timeRange *tsdb.TimeRange, dsInfo *models.DataSource) *QueryHeader { - var header QueryHeader - esVersion := dsInfo.JsonData.Get("esVersion").MustInt() - - searchType := "query_then_fetch" - if esVersion < 5 { - searchType = "count" - } - header.SearchType = searchType - header.IgnoreUnavailable = true - header.Index = getIndexList(dsInfo.Database, dsInfo.JsonData.Get("interval").MustString(), timeRange) - - if esVersion >= 56 { - header.MaxConcurrentShardRequests = dsInfo.JsonData.Get("maxConcurrentShardRequests").MustInt() - } - return &header -} - -func getIndexList(pattern string, interval string, timeRange *tsdb.TimeRange) string { - if interval == "" { - return pattern - } - - var indexes []string - indexParts := strings.Split(strings.TrimLeft(pattern, "["), "]") - indexBase := indexParts[0] - if len(indexParts) <= 1 { - return pattern - } - - indexDateFormat := indexParts[1] - - start := moment.NewMoment(timeRange.MustGetFrom()) - end := moment.NewMoment(timeRange.MustGetTo()) - - indexes = append(indexes, fmt.Sprintf("%s%s", indexBase, start.Format(indexDateFormat))) - for start.IsBefore(*end) { - switch interval { - case "Hourly": - start = start.AddHours(1) - - case "Daily": - start = start.AddDay() - - case "Weekly": - start = start.AddWeeks(1) - - case "Monthly": - start = start.AddMonths(1) - - case "Yearly": - start = start.AddYears(1) - } - indexes = append(indexes, fmt.Sprintf("%s%s", indexBase, start.Format(indexDateFormat))) - } - return strings.Join(indexes, ",") -} diff --git a/pkg/tsdb/elasticsearch/model_parser_test.go b/pkg/tsdb/elasticsearch/model_parser_test.go deleted file mode 100644 index aa7336fb69b..00000000000 --- a/pkg/tsdb/elasticsearch/model_parser_test.go +++ /dev/null @@ -1,49 +0,0 @@ -package elasticsearch - -import ( - "github.com/grafana/grafana/pkg/tsdb" - . "github.com/smartystreets/goconvey/convey" - "strconv" - "strings" - "testing" -) - -func makeTime(hour int) string { - //unixtime 1500000000 == 2017-07-14T02:40:00+00:00 - return strconv.Itoa((1500000000 + hour*60*60) * 1000) -} - -func getIndexListByTime(pattern string, interval string, hour int) string { - timeRange := &tsdb.TimeRange{ - From: makeTime(0), - To: makeTime(hour), - } - return getIndexList(pattern, interval, timeRange) -} - -func TestElasticsearchGetIndexList(t *testing.T) { - Convey("Test Elasticsearch getIndex ", t, func() { - - Convey("Parse Interval Formats", func() { - So(getIndexListByTime("[logstash-]YYYY.MM.DD", "Daily", 48), - ShouldEqual, "logstash-2017.07.14,logstash-2017.07.15,logstash-2017.07.16") - - So(len(strings.Split(getIndexListByTime("[logstash-]YYYY.MM.DD.HH", "Hourly", 3), ",")), - ShouldEqual, 4) - - So(getIndexListByTime("[logstash-]YYYY.W", "Weekly", 100), - ShouldEqual, "logstash-2017.28,logstash-2017.29") - - So(getIndexListByTime("[logstash-]YYYY.MM", "Monthly", 700), - ShouldEqual, "logstash-2017.07,logstash-2017.08") - - So(getIndexListByTime("[logstash-]YYYY", "Yearly", 10000), - ShouldEqual, "logstash-2017,logstash-2018,logstash-2019") - }) - - Convey("No Interval", func() { - index := getIndexListByTime("logstash-test", "", 1) - So(index, ShouldEqual, "logstash-test") - }) - }) -} diff --git a/pkg/tsdb/elasticsearch/query.go b/pkg/tsdb/elasticsearch/query.go index a63529df2df..123f8dd5667 100644 --- a/pkg/tsdb/elasticsearch/query.go +++ b/pkg/tsdb/elasticsearch/query.go @@ -5,12 +5,14 @@ import ( "encoding/json" "errors" "fmt" - "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/tsdb" "strconv" "strings" "time" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/tsdb" + "github.com/leibowitz/moment" ) var rangeFilterSetting = RangeFilterSetting{Gte: "$timeFrom", @@ -22,14 +24,12 @@ type Query struct { RawQuery string `json:"query"` BucketAggs []*BucketAgg `json:"bucketAggs"` Metrics []*Metric `json:"metrics"` - Alias string `json:"Alias"` + Alias string `json:"alias"` Interval time.Duration } func (q *Query) Build(queryContext *tsdb.TsdbQuery, dsInfo *models.DataSource) (string, error) { var req Request - payload := bytes.Buffer{} - req.Size = 0 q.renderReqQuery(&req) @@ -45,6 +45,7 @@ func (q *Query) Build(queryContext *tsdb.TsdbQuery, dsInfo *models.DataSource) ( reqBytes, err := json.Marshal(req) reqHeader := getRequestHeader(queryContext.TimeRange, dsInfo) + payload := bytes.Buffer{} payload.WriteString(reqHeader.String() + "\n") payload.WriteString(string(reqBytes) + "\n") return q.renderTemplate(payload.String(), queryContext) @@ -235,3 +236,61 @@ func (q *Query) renderTemplate(payload string, queryContext *tsdb.TsdbQuery) (st payload = strings.Replace(payload, "$__interval", interval.Text, -1) return payload, nil } + +func getRequestHeader(timeRange *tsdb.TimeRange, dsInfo *models.DataSource) *QueryHeader { + var header QueryHeader + esVersion := dsInfo.JsonData.Get("esVersion").MustInt() + + searchType := "query_then_fetch" + if esVersion < 5 { + searchType = "count" + } + header.SearchType = searchType + header.IgnoreUnavailable = true + header.Index = getIndexList(dsInfo.Database, dsInfo.JsonData.Get("interval").MustString(), timeRange) + + if esVersion >= 56 { + header.MaxConcurrentShardRequests = dsInfo.JsonData.Get("maxConcurrentShardRequests").MustInt() + } + return &header +} + +func getIndexList(pattern string, interval string, timeRange *tsdb.TimeRange) string { + if interval == "" { + return pattern + } + + var indexes []string + indexParts := strings.Split(strings.TrimLeft(pattern, "["), "]") + indexBase := indexParts[0] + if len(indexParts) <= 1 { + return pattern + } + + indexDateFormat := indexParts[1] + + start := moment.NewMoment(timeRange.MustGetFrom()) + end := moment.NewMoment(timeRange.MustGetTo()) + + indexes = append(indexes, fmt.Sprintf("%s%s", indexBase, start.Format(indexDateFormat))) + for start.IsBefore(*end) { + switch interval { + case "Hourly": + start = start.AddHours(1) + + case "Daily": + start = start.AddDay() + + case "Weekly": + start = start.AddWeeks(1) + + case "Monthly": + start = start.AddMonths(1) + + case "Yearly": + start = start.AddYears(1) + } + indexes = append(indexes, fmt.Sprintf("%s%s", indexBase, start.Format(indexDateFormat))) + } + return strings.Join(indexes, ",") +} diff --git a/pkg/tsdb/elasticsearch/query_test.go b/pkg/tsdb/elasticsearch/query_test.go index aecca9f4734..1ce6e5ac7bb 100644 --- a/pkg/tsdb/elasticsearch/query_test.go +++ b/pkg/tsdb/elasticsearch/query_test.go @@ -3,14 +3,15 @@ package elasticsearch import ( "encoding/json" "fmt" - "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/tsdb" - . "github.com/smartystreets/goconvey/convey" "reflect" "strconv" "strings" "testing" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/tsdb" + . "github.com/smartystreets/goconvey/convey" ) func testElasticSearchResponse(query Query, expectedElasticSearchRequestJSON string) { @@ -254,3 +255,43 @@ func TestElasticSearchQueryBuilder(t *testing.T) { }) }) } + +func makeTime(hour int) string { + //unixtime 1500000000 == 2017-07-14T02:40:00+00:00 + return strconv.Itoa((1500000000 + hour*60*60) * 1000) +} + +func getIndexListByTime(pattern string, interval string, hour int) string { + timeRange := &tsdb.TimeRange{ + From: makeTime(0), + To: makeTime(hour), + } + return getIndexList(pattern, interval, timeRange) +} + +func TestElasticsearchGetIndexList(t *testing.T) { + Convey("Test Elasticsearch getIndex ", t, func() { + + Convey("Parse Interval Formats", func() { + So(getIndexListByTime("[logstash-]YYYY.MM.DD", "Daily", 48), + ShouldEqual, "logstash-2017.07.14,logstash-2017.07.15,logstash-2017.07.16") + + So(len(strings.Split(getIndexListByTime("[logstash-]YYYY.MM.DD.HH", "Hourly", 3), ",")), + ShouldEqual, 4) + + So(getIndexListByTime("[logstash-]YYYY.W", "Weekly", 100), + ShouldEqual, "logstash-2017.28,logstash-2017.29") + + So(getIndexListByTime("[logstash-]YYYY.MM", "Monthly", 700), + ShouldEqual, "logstash-2017.07,logstash-2017.08") + + So(getIndexListByTime("[logstash-]YYYY", "Yearly", 10000), + ShouldEqual, "logstash-2017,logstash-2018,logstash-2019") + }) + + Convey("No Interval", func() { + index := getIndexListByTime("logstash-test", "", 1) + So(index, ShouldEqual, "logstash-test") + }) + }) +} diff --git a/pkg/tsdb/elasticsearch/response_parser_test.go b/pkg/tsdb/elasticsearch/response_parser_test.go index c5b877c1925..1df2c4551ae 100644 --- a/pkg/tsdb/elasticsearch/response_parser_test.go +++ b/pkg/tsdb/elasticsearch/response_parser_test.go @@ -2,9 +2,10 @@ package elasticsearch import ( "encoding/json" + "testing" + "github.com/grafana/grafana/pkg/tsdb" . "github.com/smartystreets/goconvey/convey" - "testing" ) func testElasticsearchResponse(body string, target Query) *tsdb.QueryResult { diff --git a/pkg/tsdb/elasticsearch/time_series_query.go b/pkg/tsdb/elasticsearch/time_series_query.go new file mode 100644 index 00000000000..af8f61eb144 --- /dev/null +++ b/pkg/tsdb/elasticsearch/time_series_query.go @@ -0,0 +1,182 @@ +package elasticsearch + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/tsdb" + "golang.org/x/net/context/ctxhttp" +) + +type timeSeriesQuery struct { + queries []*Query +} + +func (e *ElasticsearchExecutor) executeTimeSeriesQuery(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) { + result := &tsdb.Response{} + result.Results = make(map[string]*tsdb.QueryResult) + + tsQueryParser := newTimeSeriesQueryParser(dsInfo) + query, err := tsQueryParser.parse(tsdbQuery) + if err != nil { + return nil, err + } + + buff := bytes.Buffer{} + for _, q := range query.queries { + s, err := q.Build(tsdbQuery, dsInfo) + if err != nil { + return nil, err + } + buff.WriteString(s) + } + payload := buff.String() + + if setting.Env == setting.DEV { + glog.Debug("Elasticsearch playload", "raw playload", payload) + } + glog.Info("Elasticsearch playload", "raw playload", payload) + + req, err := e.createRequest(dsInfo, payload) + if err != nil { + return nil, err + } + + httpClient, err := dsInfo.GetHttpClient() + if err != nil { + return nil, err + } + + resp, err := ctxhttp.Do(ctx, httpClient, req) + if err != nil { + return nil, err + } + + if resp.StatusCode/100 != 2 { + return nil, fmt.Errorf("elasticsearch returned statuscode invalid status code: %v", resp.Status) + } + + var responses Responses + defer resp.Body.Close() + dec := json.NewDecoder(resp.Body) + dec.UseNumber() + err = dec.Decode(&responses) + if err != nil { + return nil, err + } + + for _, res := range responses.Responses { + if res.Err != nil { + return nil, errors.New(res.getErrMsg()) + } + } + responseParser := ElasticsearchResponseParser{responses.Responses, query.queries} + queryRes := responseParser.getTimeSeries() + result.Results["A"] = queryRes + return result, nil +} + +type timeSeriesQueryParser struct { + ds *models.DataSource +} + +func newTimeSeriesQueryParser(ds *models.DataSource) *timeSeriesQueryParser { + return &timeSeriesQueryParser{ + ds: ds, + } +} + +func (p *timeSeriesQueryParser) parse(tsdbQuery *tsdb.TsdbQuery) (*timeSeriesQuery, error) { + queries := make([]*Query, 0) + for _, q := range tsdbQuery.Queries { + model := q.Model + timeField, err := model.Get("timeField").String() + if err != nil { + return nil, err + } + rawQuery := model.Get("query").MustString() + bucketAggs, err := p.parseBucketAggs(model) + if err != nil { + return nil, err + } + metrics, err := p.parseMetrics(model) + if err != nil { + return nil, err + } + alias := model.Get("alias").MustString("") + parsedInterval, err := tsdb.GetIntervalFrom(p.ds, model, time.Millisecond) + if err != nil { + return nil, err + } + + queries = append(queries, &Query{ + TimeField: timeField, + RawQuery: rawQuery, + BucketAggs: bucketAggs, + Metrics: metrics, + Alias: alias, + Interval: parsedInterval, + }) + } + + return &timeSeriesQuery{queries: queries}, nil +} + +func (p *timeSeriesQueryParser) parseBucketAggs(model *simplejson.Json) ([]*BucketAgg, error) { + var err error + var result []*BucketAgg + for _, t := range model.Get("bucketAggs").MustArray() { + aggJson := simplejson.NewFromAny(t) + agg := &BucketAgg{} + + agg.Type, err = aggJson.Get("type").String() + if err != nil { + return nil, err + } + + agg.ID, err = aggJson.Get("id").String() + if err != nil { + return nil, err + } + + agg.Field = aggJson.Get("field").MustString() + agg.Settings = simplejson.NewFromAny(aggJson.Get("settings").MustMap()) + + result = append(result, agg) + } + return result, nil +} + +func (p *timeSeriesQueryParser) parseMetrics(model *simplejson.Json) ([]*Metric, error) { + var err error + var result []*Metric + for _, t := range model.Get("metrics").MustArray() { + metricJSON := simplejson.NewFromAny(t) + metric := &Metric{} + + metric.Field = metricJSON.Get("field").MustString() + metric.Hide = metricJSON.Get("hide").MustBool(false) + metric.ID, err = metricJSON.Get("id").String() + if err != nil { + return nil, err + } + + metric.PipelineAggregate = metricJSON.Get("pipelineAgg").MustString() + metric.Settings = simplejson.NewFromAny(metricJSON.Get("settings").MustMap()) + + metric.Type, err = metricJSON.Get("type").String() + if err != nil { + return nil, err + } + + result = append(result, metric) + } + return result, nil +} diff --git a/pkg/tsdb/elasticsearch/time_series_query_test.go b/pkg/tsdb/elasticsearch/time_series_query_test.go new file mode 100644 index 00000000000..4950bc811de --- /dev/null +++ b/pkg/tsdb/elasticsearch/time_series_query_test.go @@ -0,0 +1,118 @@ +package elasticsearch + +import ( + "testing" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/tsdb" + . "github.com/smartystreets/goconvey/convey" +) + +func TestTimeSeriesQueryParser(t *testing.T) { + Convey("Test time series query parser", t, func() { + ds := &models.DataSource{} + p := newTimeSeriesQueryParser(ds) + + Convey("Should be able to parse query", func() { + json, err := simplejson.NewJson([]byte(`{ + "timeField": "@timestamp", + "query": "@metric:cpu", + "alias": "{{@hostname}} {{metric}}", + "metrics": [ + { + "field": "@value", + "id": "1", + "meta": {}, + "settings": { + "percents": [ + "90" + ] + }, + "type": "percentiles" + }, + { + "type": "count", + "field": "select field", + "id": "4", + "settings": {}, + "meta": {} + } + ], + "bucketAggs": [ + { + "fake": true, + "field": "@hostname", + "id": "3", + "settings": { + "min_doc_count": 1, + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "5m", + "min_doc_count": 0, + "trimEdges": 0 + }, + "type": "date_histogram" + } + ] + }`)) + So(err, ShouldBeNil) + tsdbQuery := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + DataSource: ds, + Model: json, + }, + }, + } + tsQuery, err := p.parse(tsdbQuery) + So(err, ShouldBeNil) + So(tsQuery.queries, ShouldHaveLength, 1) + + q := tsQuery.queries[0] + + So(q.TimeField, ShouldEqual, "@timestamp") + So(q.RawQuery, ShouldEqual, "@metric:cpu") + So(q.Alias, ShouldEqual, "{{@hostname}} {{metric}}") + + So(q.Metrics, ShouldHaveLength, 2) + So(q.Metrics[0].Field, ShouldEqual, "@value") + So(q.Metrics[0].ID, ShouldEqual, "1") + So(q.Metrics[0].Type, ShouldEqual, "percentiles") + So(q.Metrics[0].Hide, ShouldBeFalse) + So(q.Metrics[0].PipelineAggregate, ShouldEqual, "") + So(q.Metrics[0].Settings.Get("percents").MustStringArray()[0], ShouldEqual, "90") + + So(q.Metrics[1].Field, ShouldEqual, "select field") + So(q.Metrics[1].ID, ShouldEqual, "4") + So(q.Metrics[1].Type, ShouldEqual, "count") + So(q.Metrics[1].Hide, ShouldBeFalse) + So(q.Metrics[1].PipelineAggregate, ShouldEqual, "") + So(q.Metrics[1].Settings.MustMap(), ShouldBeEmpty) + + So(q.BucketAggs, ShouldHaveLength, 2) + So(q.BucketAggs[0].Field, ShouldEqual, "@hostname") + So(q.BucketAggs[0].ID, ShouldEqual, "3") + So(q.BucketAggs[0].Type, ShouldEqual, "terms") + So(q.BucketAggs[0].Settings.Get("min_doc_count").MustInt64(), ShouldEqual, 1) + So(q.BucketAggs[0].Settings.Get("order").MustString(), ShouldEqual, "desc") + So(q.BucketAggs[0].Settings.Get("orderBy").MustString(), ShouldEqual, "_term") + So(q.BucketAggs[0].Settings.Get("size").MustString(), ShouldEqual, "10") + + So(q.BucketAggs[1].Field, ShouldEqual, "@timestamp") + So(q.BucketAggs[1].ID, ShouldEqual, "2") + So(q.BucketAggs[1].Type, ShouldEqual, "date_histogram") + So(q.BucketAggs[1].Settings.Get("interval").MustString(), ShouldEqual, "5m") + So(q.BucketAggs[1].Settings.Get("min_doc_count").MustInt64(), ShouldEqual, 0) + So(q.BucketAggs[1].Settings.Get("trimEdges").MustInt64(), ShouldEqual, 0) + }) + }) +} From e171ed89102ff136f48488f38c98e71e03a77813 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 23 May 2018 14:59:12 +0200 Subject: [PATCH 12/87] elasticsearch: new simple client for communicating with elasticsearch Handles minor differences of es 2, 5 and 5.6. Implements index pattern logic. Exposes builders for building search requests. --- pkg/tsdb/elasticsearch/client/client.go | 286 +++++++++++ pkg/tsdb/elasticsearch/client/client_test.go | 215 ++++++++ .../elasticsearch/client/index_pattern.go | 312 ++++++++++++ .../client/index_pattern_test.go | 244 +++++++++ pkg/tsdb/elasticsearch/client/models.go | 304 +++++++++++ .../elasticsearch/client/search_request.go | 446 +++++++++++++++++ .../client/search_request_test.go | 471 ++++++++++++++++++ 7 files changed, 2278 insertions(+) create mode 100644 pkg/tsdb/elasticsearch/client/client.go create mode 100644 pkg/tsdb/elasticsearch/client/client_test.go create mode 100644 pkg/tsdb/elasticsearch/client/index_pattern.go create mode 100644 pkg/tsdb/elasticsearch/client/index_pattern_test.go create mode 100644 pkg/tsdb/elasticsearch/client/models.go create mode 100644 pkg/tsdb/elasticsearch/client/search_request.go create mode 100644 pkg/tsdb/elasticsearch/client/search_request_test.go diff --git a/pkg/tsdb/elasticsearch/client/client.go b/pkg/tsdb/elasticsearch/client/client.go new file mode 100644 index 00000000000..3dae343bd37 --- /dev/null +++ b/pkg/tsdb/elasticsearch/client/client.go @@ -0,0 +1,286 @@ +package es + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "path" + "strings" + "time" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/tsdb" + + "github.com/grafana/grafana/pkg/models" + "golang.org/x/net/context/ctxhttp" +) + +const loggerName = "tsdb.elasticsearch.client" + +var ( + clientLog = log.New(loggerName) + intervalCalculator = tsdb.NewIntervalCalculator(&tsdb.IntervalOptions{MinInterval: 15 * time.Second}) +) + +// Client represents a client which can interact with elasticsearch api +type Client interface { + GetVersion() int + GetTimeField() string + GetMinInterval(queryInterval string) (time.Duration, error) + ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearchResponse, error) + MultiSearch() *MultiSearchRequestBuilder +} + +// NewClient creates a new elasticsearch client +var NewClient = func(ctx context.Context, ds *models.DataSource, timeRange *tsdb.TimeRange) (Client, error) { + version, err := ds.JsonData.Get("esVersion").Int() + if err != nil { + return nil, fmt.Errorf("eleasticsearch version is required, err=%v", err) + } + + timeField, err := ds.JsonData.Get("timeField").String() + if err != nil { + return nil, fmt.Errorf("eleasticsearch time field name is required, err=%v", err) + } + + indexInterval := ds.JsonData.Get("interval").MustString() + ip, err := newIndexPattern(indexInterval, ds.Database) + if err != nil { + return nil, err + } + + indices, err := ip.GetIndices(timeRange) + if err != nil { + return nil, err + } + + bc := &baseClientImpl{ + ctx: ctx, + ds: ds, + version: version, + timeField: timeField, + indices: indices, + } + + clientLog.Debug("Creating new client", "version", version, "timeField", timeField, "indices", strings.Join(indices, ", ")) + + switch version { + case 2: + return newV2Client(bc) + case 5: + return newV5Client(bc) + case 56: + return newV56Client(bc) + } + + return nil, fmt.Errorf("elasticsearch version=%d is not supported", version) +} + +type baseClient interface { + Client + getSettings() *simplejson.Json + executeBatchRequest(uriPath string, requests []*multiRequest) (*http.Response, error) + executeRequest(method, uriPath string, body []byte) (*http.Response, error) + createMultiSearchRequests(searchRequests []*SearchRequest) []*multiRequest +} + +type baseClientImpl struct { + ctx context.Context + ds *models.DataSource + version int + timeField string + indices []string +} + +func (c *baseClientImpl) GetVersion() int { + return c.version +} + +func (c *baseClientImpl) GetTimeField() string { + return c.timeField +} + +func (c *baseClientImpl) GetMinInterval(queryInterval string) (time.Duration, error) { + return tsdb.GetIntervalFrom(c.ds, simplejson.NewFromAny(map[string]string{ + "interval": queryInterval, + }), 15*time.Second) +} + +func (c *baseClientImpl) getSettings() *simplejson.Json { + return c.ds.JsonData +} + +type multiRequest struct { + header map[string]interface{} + body interface{} +} + +func (c *baseClientImpl) executeBatchRequest(uriPath string, requests []*multiRequest) (*http.Response, error) { + payload := bytes.Buffer{} + for _, r := range requests { + reqHeader, err := json.Marshal(r.header) + if err != nil { + return nil, err + } + payload.WriteString(string(reqHeader) + "\n") + + reqBody, err := json.Marshal(r.body) + if err != nil { + return nil, err + } + payload.WriteString(string(reqBody) + "\n") + } + + return c.executeRequest(http.MethodPost, uriPath, payload.Bytes()) +} + +func (c *baseClientImpl) executeRequest(method, uriPath string, body []byte) (*http.Response, error) { + u, _ := url.Parse(c.ds.Url) + u.Path = path.Join(u.Path, uriPath) + + var req *http.Request + var err error + if method == http.MethodPost { + req, err = http.NewRequest(http.MethodPost, u.String(), bytes.NewBuffer(body)) + } else { + req, err = http.NewRequest(http.MethodGet, u.String(), nil) + } + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", "Grafana") + req.Header.Set("Content-Type", "application/json") + + if c.ds.BasicAuth { + clientLog.Debug("Request configured to use basic authentication") + req.SetBasicAuth(c.ds.BasicAuthUser, c.ds.BasicAuthPassword) + } + + if !c.ds.BasicAuth && c.ds.User != "" { + clientLog.Debug("Request configured to use basic authentication") + req.SetBasicAuth(c.ds.User, c.ds.Password) + } + + httpClient, err := c.ds.GetHttpClient() + if err != nil { + return nil, err + } + + if method == http.MethodPost { + clientLog.Debug("Executing request", "url", req.URL.String(), "method", method) + } else { + clientLog.Debug("Executing request", "url", req.URL.String(), "method", method) + } + + return ctxhttp.Do(c.ctx, httpClient, req) +} + +func (c *baseClientImpl) ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearchResponse, error) { + multiRequests := c.createMultiSearchRequests(r.Requests) + res, err := c.executeBatchRequest("_msearch", multiRequests) + if err != nil { + return nil, err + } + + var msr MultiSearchResponse + defer res.Body.Close() + dec := json.NewDecoder(res.Body) + err = dec.Decode(&msr) + if err != nil { + return nil, err + } + + clientLog.Debug("Received multisearch response", "code", res.StatusCode, "status", res.Status, "content-length", res.ContentLength) + + msr.status = res.StatusCode + + return &msr, nil +} + +func (c *baseClientImpl) createMultiSearchRequests(searchRequests []*SearchRequest) []*multiRequest { + multiRequests := []*multiRequest{} + + for _, searchReq := range searchRequests { + multiRequests = append(multiRequests, &multiRequest{ + header: map[string]interface{}{ + "search_type": "query_then_fetch", + "ignore_unavailable": true, + "index": strings.Join(c.indices, ","), + }, + body: searchReq, + }) + } + + return multiRequests +} + +type v2Client struct { + baseClient +} + +func newV2Client(bc baseClient) (*v2Client, error) { + c := v2Client{ + baseClient: bc, + } + + return &c, nil +} + +func (c *v2Client) createMultiSearchRequests(searchRequests []*SearchRequest) []*multiRequest { + multiRequests := c.baseClient.createMultiSearchRequests(searchRequests) + + for _, mr := range multiRequests { + mr.header["search_type"] = "count" + } + + return multiRequests +} + +type v5Client struct { + baseClient +} + +func newV5Client(bc baseClient) (*v5Client, error) { + c := v5Client{ + baseClient: bc, + } + + return &c, nil +} + +type v56Client struct { + *v5Client + maxConcurrentShardRequests int +} + +func newV56Client(bc baseClient) (*v56Client, error) { + v5Client := v5Client{ + baseClient: bc, + } + maxConcurrentShardRequests := bc.getSettings().Get("maxConcurrentShardRequests").MustInt(256) + + c := v56Client{ + v5Client: &v5Client, + maxConcurrentShardRequests: maxConcurrentShardRequests, + } + + return &c, nil +} + +func (c *v56Client) createMultiSearchRequests(searchRequests []*SearchRequest) []*multiRequest { + multiRequests := c.v5Client.createMultiSearchRequests(searchRequests) + + for _, mr := range multiRequests { + mr.header["max_concurrent_shard_requests"] = c.maxConcurrentShardRequests + } + + return multiRequests +} + +func (c *baseClientImpl) MultiSearch() *MultiSearchRequestBuilder { + return NewMultiSearchRequestBuilder(c.GetVersion()) +} diff --git a/pkg/tsdb/elasticsearch/client/client_test.go b/pkg/tsdb/elasticsearch/client/client_test.go new file mode 100644 index 00000000000..d557ceb28b1 --- /dev/null +++ b/pkg/tsdb/elasticsearch/client/client_test.go @@ -0,0 +1,215 @@ +package es + +import ( + "net/http" + "testing" + + "github.com/grafana/grafana/pkg/components/simplejson" + + "github.com/grafana/grafana/pkg/models" + . "github.com/smartystreets/goconvey/convey" +) + +func TestClient(t *testing.T) { + Convey("Test elasticsearch client", t, func() { + Convey("NewClient", func() { + Convey("When no version set should return error", func() { + ds := &models.DataSource{ + JsonData: simplejson.NewFromAny(make(map[string]interface{})), + } + + _, err := NewClient(nil, ds, nil) + So(err, ShouldNotBeNil) + }) + + Convey("When no time field name set should return error", func() { + ds := &models.DataSource{ + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 5, + }), + } + + _, err := NewClient(nil, ds, nil) + So(err, ShouldNotBeNil) + }) + + Convey("When unspported version set should return error", func() { + ds := &models.DataSource{ + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 6, + "timeField": "@timestamp", + }), + } + + _, err := NewClient(nil, ds, nil) + So(err, ShouldNotBeNil) + }) + + Convey("When version 2 should return v2 client", func() { + ds := &models.DataSource{ + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 2, + "timeField": "@timestamp", + }), + } + + c, err := NewClient(nil, ds, nil) + So(err, ShouldBeNil) + So(c.GetVersion(), ShouldEqual, 2) + }) + + Convey("When version 5 should return v5 client", func() { + ds := &models.DataSource{ + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 5, + "timeField": "@timestamp", + }), + } + + c, err := NewClient(nil, ds, nil) + So(err, ShouldBeNil) + So(c.GetVersion(), ShouldEqual, 5) + }) + + Convey("When version 56 should return v5.6 client", func() { + ds := &models.DataSource{ + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 56, + "timeField": "@timestamp", + }), + } + + c, err := NewClient(nil, ds, nil) + So(err, ShouldBeNil) + So(c.GetVersion(), ShouldEqual, 56) + }) + }) + + Convey("v2", func() { + ds := &models.DataSource{ + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 2, + }), + } + + c, err := newV2Client(newFakeBaseClient(ds, []string{"test-*"})) + So(err, ShouldBeNil) + So(c, ShouldNotBeNil) + + Convey("When creating multisearch requests should have correct headers", func() { + multiRequests := c.createMultiSearchRequests([]*SearchRequest{ + {Index: "test-*"}, + }) + So(multiRequests, ShouldHaveLength, 1) + header := multiRequests[0].header + So(header, ShouldHaveLength, 3) + So(header["index"], ShouldEqual, "test-*") + So(header["ignore_unavailable"], ShouldEqual, true) + So(header["search_type"], ShouldEqual, "count") + }) + }) + + Convey("v5", func() { + ds := &models.DataSource{ + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 5, + }), + } + + c, err := newV5Client(newFakeBaseClient(ds, []string{"test-*"})) + So(err, ShouldBeNil) + So(c, ShouldNotBeNil) + + Convey("When creating multisearch requests should have correct headers", func() { + multiRequests := c.createMultiSearchRequests([]*SearchRequest{ + {Index: "test-*"}, + }) + So(multiRequests, ShouldHaveLength, 1) + header := multiRequests[0].header + So(header, ShouldHaveLength, 3) + So(header["index"], ShouldEqual, "test-*") + So(header["ignore_unavailable"], ShouldEqual, true) + So(header["search_type"], ShouldEqual, "query_then_fetch") + }) + }) + + Convey("v5.6", func() { + Convey("With default settings", func() { + ds := models.DataSource{ + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 56, + }), + } + + c, err := newV56Client(newFakeBaseClient(&ds, []string{"test-*"})) + So(err, ShouldBeNil) + So(c, ShouldNotBeNil) + + Convey("When creating multisearch requests should have correct headers", func() { + multiRequests := c.createMultiSearchRequests([]*SearchRequest{ + {Index: "test-*"}, + }) + So(multiRequests, ShouldHaveLength, 1) + header := multiRequests[0].header + So(header, ShouldHaveLength, 4) + So(header["index"], ShouldEqual, "test-*") + So(header["ignore_unavailable"], ShouldEqual, true) + So(header["search_type"], ShouldEqual, "query_then_fetch") + So(header["max_concurrent_shard_requests"], ShouldEqual, 256) + }) + }) + + Convey("With custom settings", func() { + ds := models.DataSource{ + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 56, + "maxConcurrentShardRequests": 100, + }), + } + + c, err := newV56Client(newFakeBaseClient(&ds, []string{"test-*"})) + So(err, ShouldBeNil) + So(c, ShouldNotBeNil) + Convey("When creating multisearch requests should have correct headers", func() { + multiRequests := c.createMultiSearchRequests([]*SearchRequest{ + {Index: "test-*"}, + }) + So(multiRequests, ShouldHaveLength, 1) + header := multiRequests[0].header + So(header, ShouldHaveLength, 4) + So(header["index"], ShouldEqual, "test-*") + So(header["ignore_unavailable"], ShouldEqual, true) + So(header["search_type"], ShouldEqual, "query_then_fetch") + So(header["max_concurrent_shard_requests"], ShouldEqual, 100) + }) + }) + }) + }) +} + +type fakeBaseClient struct { + *baseClientImpl + ds *models.DataSource +} + +func newFakeBaseClient(ds *models.DataSource, indices []string) baseClient { + return &fakeBaseClient{ + baseClientImpl: &baseClientImpl{ + ds: ds, + indices: indices, + }, + ds: ds, + } +} + +func (c *fakeBaseClient) executeBatchRequest(uriPath string, requests []*multiRequest) (*http.Response, error) { + return nil, nil +} + +func (c *fakeBaseClient) executeRequest(method, uriPath string, body []byte) (*http.Response, error) { + return nil, nil +} + +func (c *fakeBaseClient) executeMultisearch(searchRequests []*SearchRequest) ([]*SearchResponse, error) { + return nil, nil +} diff --git a/pkg/tsdb/elasticsearch/client/index_pattern.go b/pkg/tsdb/elasticsearch/client/index_pattern.go new file mode 100644 index 00000000000..8391e902ea4 --- /dev/null +++ b/pkg/tsdb/elasticsearch/client/index_pattern.go @@ -0,0 +1,312 @@ +package es + +import ( + "fmt" + "regexp" + "strings" + "time" + + "github.com/grafana/grafana/pkg/tsdb" +) + +const ( + noInterval = "" + intervalHourly = "hourly" + intervalDaily = "daily" + intervalWeekly = "weekly" + intervalMonthly = "monthly" + intervalYearly = "yearly" +) + +type indexPattern interface { + GetIndices(timeRange *tsdb.TimeRange) ([]string, error) +} + +var newIndexPattern = func(interval string, pattern string) (indexPattern, error) { + if interval == noInterval { + return &staticIndexPattern{indexName: pattern}, nil + } + + return newDynamicIndexPattern(interval, pattern) +} + +type staticIndexPattern struct { + indexName string +} + +func (ip *staticIndexPattern) GetIndices(timeRange *tsdb.TimeRange) ([]string, error) { + return []string{ip.indexName}, nil +} + +type intervalGenerator interface { + Generate(from, to time.Time) []time.Time +} + +type dynamicIndexPattern struct { + interval string + pattern string + intervalGenerator intervalGenerator +} + +func newDynamicIndexPattern(interval, pattern string) (*dynamicIndexPattern, error) { + var generator intervalGenerator + + switch strings.ToLower(interval) { + case intervalHourly: + generator = &hourlyInterval{} + case intervalDaily: + generator = &dailyInterval{} + case intervalWeekly: + generator = &weeklyInterval{} + case intervalMonthly: + generator = &monthlyInterval{} + case intervalYearly: + generator = &yearlyInterval{} + default: + return nil, fmt.Errorf("unsupported interval '%s'", interval) + } + + return &dynamicIndexPattern{ + interval: interval, + pattern: pattern, + intervalGenerator: generator, + }, nil +} + +func (ip *dynamicIndexPattern) GetIndices(timeRange *tsdb.TimeRange) ([]string, error) { + from := timeRange.GetFromAsTimeUTC() + to := timeRange.GetToAsTimeUTC() + intervals := ip.intervalGenerator.Generate(from, to) + indices := make([]string, 0) + + for _, t := range intervals { + indices = append(indices, formatDate(t, ip.pattern)) + } + + return indices, nil +} + +type hourlyInterval struct{} + +func (i *hourlyInterval) Generate(from, to time.Time) []time.Time { + intervals := []time.Time{} + start := time.Date(from.Year(), from.Month(), from.Day(), from.Hour(), 0, 0, 0, time.UTC) + end := time.Date(to.Year(), to.Month(), to.Day(), to.Hour(), 0, 0, 0, time.UTC) + + intervals = append(intervals, start) + + for start.Before(end) { + start = start.Add(time.Hour) + intervals = append(intervals, start) + } + + return intervals +} + +type dailyInterval struct{} + +func (i *dailyInterval) Generate(from, to time.Time) []time.Time { + intervals := []time.Time{} + start := time.Date(from.Year(), from.Month(), from.Day(), 0, 0, 0, 0, time.UTC) + end := time.Date(to.Year(), to.Month(), to.Day(), 0, 0, 0, 0, time.UTC) + + intervals = append(intervals, start) + + for start.Before(end) { + start = start.Add(24 * time.Hour) + intervals = append(intervals, start) + } + + return intervals +} + +type weeklyInterval struct{} + +func (i *weeklyInterval) Generate(from, to time.Time) []time.Time { + intervals := []time.Time{} + start := time.Date(from.Year(), from.Month(), from.Day(), 0, 0, 0, 0, time.UTC) + end := time.Date(to.Year(), to.Month(), to.Day(), 0, 0, 0, 0, time.UTC) + + for start.Weekday() != time.Monday { + start = start.Add(-24 * time.Hour) + } + + for end.Weekday() != time.Monday { + end = end.Add(-24 * time.Hour) + } + + year, week := start.ISOWeek() + intervals = append(intervals, start) + + for start.Before(end) { + start = start.Add(24 * time.Hour) + nextYear, nextWeek := start.ISOWeek() + if nextYear != year || nextWeek != week { + intervals = append(intervals, start) + } + year = nextYear + week = nextWeek + } + + return intervals +} + +type monthlyInterval struct{} + +func (i *monthlyInterval) Generate(from, to time.Time) []time.Time { + intervals := []time.Time{} + start := time.Date(from.Year(), from.Month(), 1, 0, 0, 0, 0, time.UTC) + end := time.Date(to.Year(), to.Month(), 1, 0, 0, 0, 0, time.UTC) + + month := start.Month() + intervals = append(intervals, start) + + for start.Before(end) { + start = start.Add(24 * time.Hour) + nextMonth := start.Month() + if nextMonth != month { + intervals = append(intervals, start) + } + month = nextMonth + } + + return intervals +} + +type yearlyInterval struct{} + +func (i *yearlyInterval) Generate(from, to time.Time) []time.Time { + intervals := []time.Time{} + start := time.Date(from.Year(), 1, 1, 0, 0, 0, 0, time.UTC) + end := time.Date(to.Year(), 1, 1, 0, 0, 0, 0, time.UTC) + + year := start.Year() + intervals = append(intervals, start) + + for start.Before(end) { + start = start.Add(24 * time.Hour) + nextYear := start.Year() + if nextYear != year { + intervals = append(intervals, start) + } + year = nextYear + } + + return intervals +} + +var datePatternRegex = regexp.MustCompile("(LT|LL?L?L?|l{1,4}|Mo|MM?M?M?|Do|DDDo|DD?D?D?|ddd?d?|do?|w[o|w]?|W[o|W]?|YYYYY|YYYY|YY|gg(ggg?)?|GG(GGG?)?|e|E|a|A|hh?|HH?|mm?|ss?|SS?S?|X|zz?|ZZ?|Q)") + +var datePatternReplacements = map[string]string{ + "M": "1", // stdNumMonth 1 2 ... 11 12 + "MM": "01", // stdZeroMonth 01 02 ... 11 12 + "MMM": "Jan", // stdMonth Jan Feb ... Nov Dec + "MMMM": "January", // stdLongMonth January February ... November December + "D": "2", // stdDay 1 2 ... 30 30 + "DD": "02", // stdZeroDay 01 02 ... 30 31 + "DDD": "", // Day of the year 1 2 ... 364 365 + "DDDD": "", // Day of the year 001 002 ... 364 365 @todo**** + "d": "", // Numeric representation of day of the week 0 1 ... 5 6 + "dd": "Mon", // ***Su Mo ... Fr Sa @todo + "ddd": "Mon", // Sun Mon ... Fri Sat + "dddd": "Monday", // stdLongWeekDay Sunday Monday ... Friday Saturday + "e": "", // Numeric representation of day of the week 0 1 ... 5 6 @todo + "E": "", // ISO-8601 numeric representation of the day of the week (added in PHP 5.1.0) 1 2 ... 6 7 @todo + "w": "", // 1 2 ... 52 53 + "ww": "", // ***01 02 ... 52 53 @todo + "W": "", // 1 2 ... 52 53 + "WW": "", // ***01 02 ... 52 53 @todo + "YY": "06", // stdYear 70 71 ... 29 30 + "YYYY": "2006", // stdLongYear 1970 1971 ... 2029 2030 + "gg": "", // ISO-8601 year number 70 71 ... 29 30 + "gggg": "", // ***1970 1971 ... 2029 2030 + "GG": "", //70 71 ... 29 30 + "GGGG": "", // ***1970 1971 ... 2029 2030 + "Q": "", // 1, 2, 3, 4 + "A": "PM", // stdPM AM PM + "a": "pm", // stdpm am pm + "H": "", // stdHour 0 1 ... 22 23 + "HH": "15", // 00 01 ... 22 23 + "h": "3", // stdHour12 1 2 ... 11 12 + "hh": "03", // stdZeroHour12 01 02 ... 11 12 + "m": "4", // stdZeroMinute 0 1 ... 58 59 + "mm": "04", // stdZeroMinute 00 01 ... 58 59 + "s": "5", // stdSecond 0 1 ... 58 59 + "ss": "05", // stdZeroSecond ***00 01 ... 58 59 + "z": "MST", //EST CST ... MST PST + "zz": "MST", //EST CST ... MST PST + "Z": "Z07:00", // stdNumColonTZ -07:00 -06:00 ... +06:00 +07:00 + "ZZ": "-0700", // stdNumTZ -0700 -0600 ... +0600 +0700 + "X": "", // Seconds since unix epoch 1360013296 + "LT": "3:04 PM", // 8:30 PM + "L": "01/02/2006", //09/04/1986 + "l": "1/2/2006", //9/4/1986 + "ll": "Jan 2 2006", //Sep 4 1986 + "lll": "Jan 2 2006 3:04 PM", //Sep 4 1986 8:30 PM + "llll": "Mon, Jan 2 2006 3:04 PM", //Thu, Sep 4 1986 8:30 PM +} + +func formatDate(t time.Time, pattern string) string { + var datePattern string + parts := strings.Split(strings.TrimLeft(pattern, "["), "]") + base := parts[0] + if len(parts) == 2 { + datePattern = parts[1] + } else { + datePattern = base + base = "" + } + + formatted := t.Format(patternToLayout(datePattern)) + + if strings.Contains(formatted, "", fmt.Sprintf("%d", isoYear), -1) + formatted = strings.Replace(formatted, "", isoYearShort, -1) + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", isoWeek), -1) + + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", t.Unix()), -1) + + day := t.Weekday() + dayOfWeekIso := int(day) + if day == time.Sunday { + dayOfWeekIso = 7 + } + + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", day), -1) + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", dayOfWeekIso), -1) + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", t.YearDay()), -1) + + quarter := 4 + + switch t.Month() { + case time.January, time.February, time.March: + quarter = 1 + case time.April, time.May, time.June: + quarter = 2 + case time.July, time.August, time.September: + quarter = 3 + } + + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", quarter), -1) + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", t.Hour()), -1) + } + + return base + formatted +} + +func patternToLayout(pattern string) string { + var match [][]string + if match = datePatternRegex.FindAllStringSubmatch(pattern, -1); match == nil { + return pattern + } + + for i := range match { + if replace, ok := datePatternReplacements[match[i][0]]; ok { + pattern = strings.Replace(pattern, match[i][0], replace, 1) + } + } + + return pattern +} diff --git a/pkg/tsdb/elasticsearch/client/index_pattern_test.go b/pkg/tsdb/elasticsearch/client/index_pattern_test.go new file mode 100644 index 00000000000..3bd823d8c87 --- /dev/null +++ b/pkg/tsdb/elasticsearch/client/index_pattern_test.go @@ -0,0 +1,244 @@ +package es + +import ( + "fmt" + "testing" + "time" + + "github.com/grafana/grafana/pkg/tsdb" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestIndexPattern(t *testing.T) { + Convey("Static index patterns", t, func() { + indexPatternScenario(noInterval, "data-*", nil, func(indices []string) { + So(indices, ShouldHaveLength, 1) + So(indices[0], ShouldEqual, "data-*") + }) + + indexPatternScenario(noInterval, "es-index-name", nil, func(indices []string) { + So(indices, ShouldHaveLength, 1) + So(indices[0], ShouldEqual, "es-index-name") + }) + }) + + Convey("Dynamic index patterns", t, func() { + from := fmt.Sprintf("%d", time.Date(2018, 5, 15, 17, 50, 0, 0, time.UTC).UnixNano()/int64(time.Millisecond)) + to := fmt.Sprintf("%d", time.Date(2018, 5, 15, 17, 55, 0, 0, time.UTC).UnixNano()/int64(time.Millisecond)) + + indexPatternScenario(intervalHourly, "[data-]YYYY.MM.DD.HH", tsdb.NewTimeRange(from, to), func(indices []string) { + //So(indices, ShouldHaveLength, 1) + So(indices[0], ShouldEqual, "data-2018.05.15.17") + }) + + indexPatternScenario(intervalDaily, "[data-]YYYY.MM.DD", tsdb.NewTimeRange(from, to), func(indices []string) { + So(indices, ShouldHaveLength, 1) + So(indices[0], ShouldEqual, "data-2018.05.15") + }) + + indexPatternScenario(intervalWeekly, "[data-]GGGG.WW", tsdb.NewTimeRange(from, to), func(indices []string) { + So(indices, ShouldHaveLength, 1) + So(indices[0], ShouldEqual, "data-2018.20") + }) + + indexPatternScenario(intervalMonthly, "[data-]YYYY.MM", tsdb.NewTimeRange(from, to), func(indices []string) { + So(indices, ShouldHaveLength, 1) + So(indices[0], ShouldEqual, "data-2018.05") + }) + + indexPatternScenario(intervalYearly, "[data-]YYYY", tsdb.NewTimeRange(from, to), func(indices []string) { + So(indices, ShouldHaveLength, 1) + So(indices[0], ShouldEqual, "data-2018") + }) + }) + + Convey("Hourly interval", t, func() { + Convey("Should return 1 interval", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 1, 1, 23, 6, 0, 0, time.UTC) + intervals := (&hourlyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 1) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 23, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 2 intervals", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 1, 2, 0, 6, 0, 0, time.UTC) + intervals := (&hourlyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 2) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 23, 0, 0, 0, time.UTC)) + So(intervals[1], ShouldEqual, time.Date(2018, 1, 2, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 10 intervals", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 1, 2, 8, 6, 0, 0, time.UTC) + intervals := (&hourlyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 10) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 23, 0, 0, 0, time.UTC)) + So(intervals[4], ShouldEqual, time.Date(2018, 1, 2, 3, 0, 0, 0, time.UTC)) + So(intervals[9], ShouldEqual, time.Date(2018, 1, 2, 8, 0, 0, 0, time.UTC)) + }) + }) + + Convey("Daily interval", t, func() { + Convey("Should return 1 day", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 1, 1, 23, 6, 0, 0, time.UTC) + intervals := (&dailyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 1) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 2 days", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 1, 2, 0, 6, 0, 0, time.UTC) + intervals := (&dailyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 2) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + So(intervals[1], ShouldEqual, time.Date(2018, 1, 2, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 32 days", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 2, 1, 8, 6, 0, 0, time.UTC) + intervals := (&dailyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 32) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + So(intervals[30], ShouldEqual, time.Date(2018, 1, 31, 0, 0, 0, 0, time.UTC)) + So(intervals[31], ShouldEqual, time.Date(2018, 2, 1, 0, 0, 0, 0, time.UTC)) + }) + }) + + Convey("Weekly interval", t, func() { + Convey("Should return 1 week (1)", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 1, 1, 23, 6, 0, 0, time.UTC) + intervals := (&weeklyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 1) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 1 week (2)", func() { + from := time.Date(2017, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2017, 1, 1, 23, 6, 0, 0, time.UTC) + intervals := (&weeklyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 1) + So(intervals[0], ShouldEqual, time.Date(2016, 12, 26, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 2 weeks (1)", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 1, 10, 23, 6, 0, 0, time.UTC) + intervals := (&weeklyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 2) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + So(intervals[1], ShouldEqual, time.Date(2018, 1, 8, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 2 weeks (2)", func() { + from := time.Date(2017, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2017, 1, 8, 23, 6, 0, 0, time.UTC) + intervals := (&weeklyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 2) + So(intervals[0], ShouldEqual, time.Date(2016, 12, 26, 0, 0, 0, 0, time.UTC)) + So(intervals[1], ShouldEqual, time.Date(2017, 1, 2, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 3 weeks (1)", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 1, 21, 23, 6, 0, 0, time.UTC) + intervals := (&weeklyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 3) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + So(intervals[1], ShouldEqual, time.Date(2018, 1, 8, 0, 0, 0, 0, time.UTC)) + So(intervals[2], ShouldEqual, time.Date(2018, 1, 15, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 3 weeks (2)", func() { + from := time.Date(2017, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2017, 1, 9, 23, 6, 0, 0, time.UTC) + intervals := (&weeklyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 3) + So(intervals[0], ShouldEqual, time.Date(2016, 12, 26, 0, 0, 0, 0, time.UTC)) + So(intervals[1], ShouldEqual, time.Date(2017, 1, 2, 0, 0, 0, 0, time.UTC)) + So(intervals[2], ShouldEqual, time.Date(2017, 1, 9, 0, 0, 0, 0, time.UTC)) + }) + }) + + Convey("Monthly interval", t, func() { + Convey("Should return 1 month", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 1, 1, 23, 6, 0, 0, time.UTC) + intervals := (&monthlyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 1) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 2 months", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 2, 2, 0, 6, 0, 0, time.UTC) + intervals := (&monthlyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 2) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + So(intervals[1], ShouldEqual, time.Date(2018, 2, 1, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 14 months", func() { + from := time.Date(2017, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 2, 1, 8, 6, 0, 0, time.UTC) + intervals := (&monthlyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 14) + So(intervals[0], ShouldEqual, time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC)) + So(intervals[13], ShouldEqual, time.Date(2018, 2, 1, 0, 0, 0, 0, time.UTC)) + }) + }) + + Convey("Yearly interval", t, func() { + Convey("Should return 1 year (hour diff)", func() { + from := time.Date(2018, 2, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 2, 1, 23, 6, 0, 0, time.UTC) + intervals := (&yearlyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 1) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 1 year (month diff)", func() { + from := time.Date(2018, 2, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 12, 31, 23, 59, 59, 0, time.UTC) + intervals := (&yearlyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 1) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 2 years", func() { + from := time.Date(2018, 2, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2019, 1, 1, 23, 59, 59, 0, time.UTC) + intervals := (&yearlyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 2) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + So(intervals[1], ShouldEqual, time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 5 years", func() { + from := time.Date(2014, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 11, 1, 23, 59, 59, 0, time.UTC) + intervals := (&yearlyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 5) + So(intervals[0], ShouldEqual, time.Date(2014, 1, 1, 0, 0, 0, 0, time.UTC)) + So(intervals[4], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + }) + }) +} + +func indexPatternScenario(interval string, pattern string, timeRange *tsdb.TimeRange, fn func(indices []string)) { + Convey(fmt.Sprintf("Index pattern (interval=%s, index=%s", interval, pattern), func() { + ip, err := newIndexPattern(interval, pattern) + So(err, ShouldBeNil) + So(ip, ShouldNotBeNil) + indices, err := ip.GetIndices(timeRange) + So(err, ShouldBeNil) + fn(indices) + }) +} diff --git a/pkg/tsdb/elasticsearch/client/models.go b/pkg/tsdb/elasticsearch/client/models.go new file mode 100644 index 00000000000..3c86dcce825 --- /dev/null +++ b/pkg/tsdb/elasticsearch/client/models.go @@ -0,0 +1,304 @@ +package es + +import ( + "encoding/json" +) + +// SearchRequest represents a search request +type SearchRequest struct { + Index string + Size int + Sort map[string]interface{} + Query *Query + Aggs AggArray + CustomProps map[string]interface{} +} + +// MarshalJSON returns the JSON encoding of the request. +func (r *SearchRequest) MarshalJSON() ([]byte, error) { + root := make(map[string]interface{}) + + root["size"] = r.Size + if len(r.Sort) > 0 { + root["sort"] = r.Sort + } + + for key, value := range r.CustomProps { + root[key] = value + } + + root["query"] = r.Query + + if len(r.Aggs) > 0 { + root["aggs"] = r.Aggs + } + + return json.Marshal(root) +} + +// SearchResponseHits represents search response hits +type SearchResponseHits struct { + Hits []map[string]interface{} + Total int64 +} + +// SearchResponse represents a search response +type SearchResponse struct { + Error map[string]interface{} `json:"error"` + Aggregations map[string]interface{} `json:"aggregations"` + Hits *SearchResponseHits `json:"hits"` +} + +// func (r *Response) getErrMsg() string { +// var msg bytes.Buffer +// errJson := simplejson.NewFromAny(r.Err) +// errType, err := errJson.Get("type").String() +// if err == nil { +// msg.WriteString(fmt.Sprintf("type:%s", errType)) +// } + +// reason, err := errJson.Get("type").String() +// if err == nil { +// msg.WriteString(fmt.Sprintf("reason:%s", reason)) +// } +// return msg.String() +// } + +// MultiSearchRequest represents a multi search request +type MultiSearchRequest struct { + Requests []*SearchRequest +} + +// MultiSearchResponse represents a multi search response +type MultiSearchResponse struct { + status int `json:"status,omitempty"` + Responses []*SearchResponse `json:"responses"` +} + +// Query represents a query +type Query struct { + Bool *BoolQuery `json:"bool"` +} + +// BoolQuery represents a bool query +type BoolQuery struct { + Filters []Filter +} + +// NewBoolQuery create a new bool query +func NewBoolQuery() *BoolQuery { + return &BoolQuery{Filters: make([]Filter, 0)} +} + +// MarshalJSON returns the JSON encoding of the boolean query. +func (q *BoolQuery) MarshalJSON() ([]byte, error) { + root := make(map[string]interface{}) + + if len(q.Filters) > 0 { + if len(q.Filters) == 1 { + root["filter"] = q.Filters[0] + } else { + root["filter"] = q.Filters + } + } + return json.Marshal(root) +} + +// Filter represents a search filter +type Filter interface{} + +// QueryStringFilter represents a query string search filter +type QueryStringFilter struct { + Filter + Query string + AnalyzeWildcard bool +} + +// MarshalJSON returns the JSON encoding of the query string filter. +func (f *QueryStringFilter) MarshalJSON() ([]byte, error) { + root := map[string]interface{}{ + "query_string": map[string]interface{}{ + "query": f.Query, + "analyze_wildcard": f.AnalyzeWildcard, + }, + } + + return json.Marshal(root) +} + +// RangeFilter represents a range search filter +type RangeFilter struct { + Filter + Key string + Gte string + Lte string + Format string +} + +// DateFormatEpochMS represents a date format of epoch milliseconds (epoch_millis) +const DateFormatEpochMS = "epoch_millis" + +// MarshalJSON returns the JSON encoding of the query string filter. +func (f *RangeFilter) MarshalJSON() ([]byte, error) { + root := map[string]map[string]map[string]interface{}{ + "range": { + f.Key: { + "lte": f.Lte, + "gte": f.Gte, + }, + }, + } + + if f.Format != "" { + root["range"][f.Key]["format"] = f.Format + } + + return json.Marshal(root) +} + +// Aggregation represents an aggregation +type Aggregation interface{} + +// Agg represents a key and aggregation +type Agg struct { + Key string + Aggregation *aggContainer +} + +// MarshalJSON returns the JSON encoding of the agg +func (a *Agg) MarshalJSON() ([]byte, error) { + root := map[string]interface{}{ + a.Key: a.Aggregation, + } + + return json.Marshal(root) +} + +// AggArray represents a collection of key/aggregation pairs +type AggArray []*Agg + +// MarshalJSON returns the JSON encoding of the agg +func (a AggArray) MarshalJSON() ([]byte, error) { + aggsMap := make(map[string]Aggregation) + + for _, subAgg := range a { + aggsMap[subAgg.Key] = subAgg.Aggregation + } + + return json.Marshal(aggsMap) +} + +type aggContainer struct { + Type string + Aggregation Aggregation + Aggs AggArray +} + +// MarshalJSON returns the JSON encoding of the aggregation container +func (a *aggContainer) MarshalJSON() ([]byte, error) { + root := map[string]interface{}{ + a.Type: a.Aggregation, + } + + if len(a.Aggs) > 0 { + root["aggs"] = a.Aggs + } + + return json.Marshal(root) +} + +type aggDef struct { + key string + aggregation *aggContainer + builders []AggBuilder +} + +func newAggDef(key string, aggregation *aggContainer) *aggDef { + return &aggDef{ + key: key, + aggregation: aggregation, + builders: make([]AggBuilder, 0), + } +} + +// HistogramAgg represents a histogram aggregation +type HistogramAgg struct { + Interval int `json:"interval,omitempty"` + Field string `json:"field"` + MinDocCount int `json:"min_doc_count"` + Missing *int `json:"missing,omitempty"` +} + +// DateHistogramAgg represents a date histogram aggregation +type DateHistogramAgg struct { + Field string `json:"field"` + Interval string `json:"interval,omitempty"` + MinDocCount int `json:"min_doc_count"` + Missing *string `json:"missing,omitempty"` + ExtendedBounds *ExtendedBounds `json:"extended_bounds"` + Format string `json:"format"` +} + +// FiltersAggregation represents a filters aggregation +type FiltersAggregation struct { + Filters map[string]interface{} `json:"filters"` +} + +// TermsAggregation represents a terms aggregation +type TermsAggregation struct { + Field string `json:"field"` + Size int `json:"size"` + Order map[string]interface{} `json:"order"` + MinDocCount *int `json:"min_doc_count,omitempty"` + Missing *string `json:"missing,omitempty"` +} + +// ExtendedBounds represents extended bounds +type ExtendedBounds struct { + Min string `json:"min"` + Max string `json:"max"` +} + +// GeoHashGridAggregation represents a geo hash grid aggregation +type GeoHashGridAggregation struct { + Field string `json:"field"` + Precision int `json:"precision"` +} + +// MetricAggregation represents a metric aggregation +type MetricAggregation struct { + Field string + Settings map[string]interface{} +} + +// MarshalJSON returns the JSON encoding of the metric aggregation +func (a *MetricAggregation) MarshalJSON() ([]byte, error) { + root := map[string]interface{}{ + "field": a.Field, + } + + for k, v := range a.Settings { + root[k] = v + } + + return json.Marshal(root) +} + +// PipelineAggregation represents a metric aggregation +type PipelineAggregation struct { + BucketPath string + Settings map[string]interface{} +} + +// MarshalJSON returns the JSON encoding of the pipeline aggregation +func (a *PipelineAggregation) MarshalJSON() ([]byte, error) { + root := map[string]interface{}{ + "bucket_path": a.BucketPath, + } + + for k, v := range a.Settings { + root[k] = v + } + + return json.Marshal(root) +} diff --git a/pkg/tsdb/elasticsearch/client/search_request.go b/pkg/tsdb/elasticsearch/client/search_request.go new file mode 100644 index 00000000000..a582d8ec247 --- /dev/null +++ b/pkg/tsdb/elasticsearch/client/search_request.go @@ -0,0 +1,446 @@ +package es + +import ( + "strings" +) + +// SearchRequestBuilder represents a builder which can build a search request +type SearchRequestBuilder struct { + version int + index string + size int + sort map[string]interface{} + queryBuilder *QueryBuilder + aggBuilders []AggBuilder + customProps map[string]interface{} +} + +// NewSearchRequestBuilder create a new search request builder +func NewSearchRequestBuilder(version int) *SearchRequestBuilder { + builder := &SearchRequestBuilder{ + version: version, + sort: make(map[string]interface{}), + customProps: make(map[string]interface{}), + aggBuilders: make([]AggBuilder, 0), + } + return builder +} + +// Build builds and return a search request +func (b *SearchRequestBuilder) Build() (*SearchRequest, error) { + sr := SearchRequest{ + Index: b.index, + Size: b.size, + Sort: b.sort, + CustomProps: b.customProps, + } + + if b.queryBuilder != nil { + q, err := b.queryBuilder.Build() + if err != nil { + return nil, err + } + sr.Query = q + } + + if len(b.aggBuilders) > 0 { + sr.Aggs = make(AggArray, 0) + + for _, ab := range b.aggBuilders { + aggArray, err := ab.Build() + if err != nil { + return nil, err + } + for _, agg := range aggArray { + sr.Aggs = append(sr.Aggs, agg) + } + } + } + + return &sr, nil +} + +// Size sets the size of the search request +func (b *SearchRequestBuilder) Size(size int) *SearchRequestBuilder { + b.size = size + return b +} + +// SortDesc adds a sort to the search request +func (b *SearchRequestBuilder) SortDesc(field, unmappedType string) *SearchRequestBuilder { + props := map[string]string{ + "order": "desc", + } + + if unmappedType != "" { + props["unmapped_type"] = unmappedType + } + + b.sort[field] = props + + return b +} + +// AddDocValueField adds a doc value field to the search request +func (b *SearchRequestBuilder) AddDocValueField(field string) *SearchRequestBuilder { + // fields field not supported on version >= 5 + if b.version < 5 { + b.customProps["fields"] = []string{"*", "_source"} + } + + b.customProps["script_fields"] = make(map[string]interface{}) + + if b.version < 5 { + b.customProps["fielddata_fields"] = []string{field} + } else { + b.customProps["docvalue_fields"] = []string{field} + } + + return b +} + +// Query creates and return a query builder +func (b *SearchRequestBuilder) Query() *QueryBuilder { + if b.queryBuilder == nil { + b.queryBuilder = NewQueryBuilder() + } + return b.queryBuilder +} + +// Agg initaite and returns a new aggregation builder +func (b *SearchRequestBuilder) Agg() AggBuilder { + aggBuilder := newAggBuilder() + b.aggBuilders = append(b.aggBuilders, aggBuilder) + return aggBuilder +} + +// MultiSearchRequestBuilder represents a builder which can build a multi search request +type MultiSearchRequestBuilder struct { + version int + requestBuilders []*SearchRequestBuilder +} + +// NewMultiSearchRequestBuilder creates a new multi search request builder +func NewMultiSearchRequestBuilder(version int) *MultiSearchRequestBuilder { + return &MultiSearchRequestBuilder{ + version: version, + } +} + +// Search initiates and returns a new search request builder +func (m *MultiSearchRequestBuilder) Search() *SearchRequestBuilder { + b := NewSearchRequestBuilder(m.version) + m.requestBuilders = append(m.requestBuilders, b) + return b +} + +// Build builds and return a multi search request +func (m *MultiSearchRequestBuilder) Build() (*MultiSearchRequest, error) { + requests := []*SearchRequest{} + for _, sb := range m.requestBuilders { + searchRequest, err := sb.Build() + if err != nil { + return nil, err + } + requests = append(requests, searchRequest) + } + + return &MultiSearchRequest{ + Requests: requests, + }, nil +} + +// QueryBuilder represents a query builder +type QueryBuilder struct { + boolQueryBuilder *BoolQueryBuilder +} + +// NewQueryBuilder create a new query builder +func NewQueryBuilder() *QueryBuilder { + return &QueryBuilder{} +} + +// Build builds and return a query builder +func (b *QueryBuilder) Build() (*Query, error) { + q := Query{} + + if b.boolQueryBuilder != nil { + b, err := b.boolQueryBuilder.Build() + if err != nil { + return nil, err + } + q.Bool = b + } + + return &q, nil +} + +// Bool creates and return a query builder +func (b *QueryBuilder) Bool() *BoolQueryBuilder { + if b.boolQueryBuilder == nil { + b.boolQueryBuilder = NewBoolQueryBuilder() + } + return b.boolQueryBuilder +} + +// BoolQueryBuilder represents a bool query builder +type BoolQueryBuilder struct { + filterQueryBuilder *FilterQueryBuilder +} + +// NewBoolQueryBuilder create a new bool query builder +func NewBoolQueryBuilder() *BoolQueryBuilder { + return &BoolQueryBuilder{} +} + +// Filter creates and return a filter query builder +func (b *BoolQueryBuilder) Filter() *FilterQueryBuilder { + if b.filterQueryBuilder == nil { + b.filterQueryBuilder = NewFilterQueryBuilder() + } + return b.filterQueryBuilder +} + +// Build builds and return a bool query builder +func (b *BoolQueryBuilder) Build() (*BoolQuery, error) { + boolQuery := BoolQuery{} + + if b.filterQueryBuilder != nil { + filters, err := b.filterQueryBuilder.Build() + if err != nil { + return nil, err + } + boolQuery.Filters = filters + } + + return &boolQuery, nil +} + +// FilterQueryBuilder represents a filter query builder +type FilterQueryBuilder struct { + filters []Filter +} + +// NewFilterQueryBuilder creates a new filter query builder +func NewFilterQueryBuilder() *FilterQueryBuilder { + return &FilterQueryBuilder{ + filters: make([]Filter, 0), + } +} + +// Build builds and return a filter query builder +func (b *FilterQueryBuilder) Build() ([]Filter, error) { + return b.filters, nil +} + +// AddDateRangeFilter adds a new time range filter +func (b *FilterQueryBuilder) AddDateRangeFilter(timeField, lte, gte, format string) *FilterQueryBuilder { + b.filters = append(b.filters, &RangeFilter{ + Key: timeField, + Lte: lte, + Gte: gte, + Format: format, + }) + return b +} + +// AddQueryStringFilter adds a new query string filter +func (b *FilterQueryBuilder) AddQueryStringFilter(querystring string, analyseWildcard bool) *FilterQueryBuilder { + if len(strings.TrimSpace(querystring)) == 0 { + return b + } + + b.filters = append(b.filters, &QueryStringFilter{ + Query: querystring, + AnalyzeWildcard: analyseWildcard, + }) + return b +} + +// AggBuilder represents an aggregation builder +type AggBuilder interface { + Histogram(key, field string, fn func(a *HistogramAgg, b AggBuilder)) AggBuilder + DateHistogram(key, field string, fn func(a *DateHistogramAgg, b AggBuilder)) AggBuilder + Terms(key, field string, fn func(a *TermsAggregation, b AggBuilder)) AggBuilder + Filters(key string, fn func(a *FiltersAggregation, b AggBuilder)) AggBuilder + GeoHashGrid(key, field string, fn func(a *GeoHashGridAggregation, b AggBuilder)) AggBuilder + Metric(key, metricType, field string, fn func(a *MetricAggregation)) AggBuilder + Pipeline(key, pipelineType, bucketPath string, fn func(a *PipelineAggregation)) AggBuilder + Build() (AggArray, error) +} + +type aggBuilderImpl struct { + AggBuilder + aggDefs []*aggDef +} + +func newAggBuilder() *aggBuilderImpl { + return &aggBuilderImpl{ + aggDefs: make([]*aggDef, 0), + } +} + +func (b *aggBuilderImpl) Build() (AggArray, error) { + aggs := make(AggArray, 0) + + for _, aggDef := range b.aggDefs { + agg := &Agg{ + Key: aggDef.key, + Aggregation: aggDef.aggregation, + } + + for _, cb := range aggDef.builders { + childAggs, err := cb.Build() + if err != nil { + return nil, err + } + + for _, childAgg := range childAggs { + agg.Aggregation.Aggs = append(agg.Aggregation.Aggs, childAgg) + } + } + + aggs = append(aggs, agg) + } + + return aggs, nil +} + +func (b *aggBuilderImpl) Histogram(key, field string, fn func(a *HistogramAgg, b AggBuilder)) AggBuilder { + innerAgg := &HistogramAgg{ + Field: field, + } + aggDef := newAggDef(key, &aggContainer{ + Type: "histogram", + Aggregation: innerAgg, + }) + + if fn != nil { + builder := newAggBuilder() + aggDef.builders = append(aggDef.builders, builder) + fn(innerAgg, builder) + } + + b.aggDefs = append(b.aggDefs, aggDef) + + return b +} + +func (b *aggBuilderImpl) DateHistogram(key, field string, fn func(a *DateHistogramAgg, b AggBuilder)) AggBuilder { + innerAgg := &DateHistogramAgg{ + Field: field, + } + aggDef := newAggDef(key, &aggContainer{ + Type: "date_histogram", + Aggregation: innerAgg, + }) + + if fn != nil { + builder := newAggBuilder() + aggDef.builders = append(aggDef.builders, builder) + fn(innerAgg, builder) + } + + b.aggDefs = append(b.aggDefs, aggDef) + + return b +} + +func (b *aggBuilderImpl) Terms(key, field string, fn func(a *TermsAggregation, b AggBuilder)) AggBuilder { + innerAgg := &TermsAggregation{ + Field: field, + Order: make(map[string]interface{}), + } + aggDef := newAggDef(key, &aggContainer{ + Type: "terms", + Aggregation: innerAgg, + }) + + if fn != nil { + builder := newAggBuilder() + aggDef.builders = append(aggDef.builders, builder) + fn(innerAgg, builder) + } + + b.aggDefs = append(b.aggDefs, aggDef) + + return b +} + +func (b *aggBuilderImpl) Filters(key string, fn func(a *FiltersAggregation, b AggBuilder)) AggBuilder { + innerAgg := &FiltersAggregation{ + Filters: make(map[string]interface{}), + } + aggDef := newAggDef(key, &aggContainer{ + Type: "filters", + Aggregation: innerAgg, + }) + if fn != nil { + builder := newAggBuilder() + aggDef.builders = append(aggDef.builders, builder) + fn(innerAgg, builder) + } + + b.aggDefs = append(b.aggDefs, aggDef) + + return b +} + +func (b *aggBuilderImpl) GeoHashGrid(key, field string, fn func(a *GeoHashGridAggregation, b AggBuilder)) AggBuilder { + innerAgg := &GeoHashGridAggregation{ + Field: field, + Precision: 5, + } + aggDef := newAggDef(key, &aggContainer{ + Type: "geohash_grid", + Aggregation: innerAgg, + }) + + if fn != nil { + builder := newAggBuilder() + aggDef.builders = append(aggDef.builders, builder) + fn(innerAgg, builder) + } + + b.aggDefs = append(b.aggDefs, aggDef) + + return b +} + +func (b *aggBuilderImpl) Metric(key, metricType, field string, fn func(a *MetricAggregation)) AggBuilder { + innerAgg := &MetricAggregation{ + Field: field, + Settings: make(map[string]interface{}), + } + aggDef := newAggDef(key, &aggContainer{ + Type: metricType, + Aggregation: innerAgg, + }) + + if fn != nil { + fn(innerAgg) + } + + b.aggDefs = append(b.aggDefs, aggDef) + + return b +} + +func (b *aggBuilderImpl) Pipeline(key, pipelineType, bucketPath string, fn func(a *PipelineAggregation)) AggBuilder { + innerAgg := &PipelineAggregation{ + BucketPath: bucketPath, + Settings: make(map[string]interface{}), + } + aggDef := newAggDef(key, &aggContainer{ + Type: pipelineType, + Aggregation: innerAgg, + }) + + if fn != nil { + fn(innerAgg) + } + + b.aggDefs = append(b.aggDefs, aggDef) + + return b +} diff --git a/pkg/tsdb/elasticsearch/client/search_request_test.go b/pkg/tsdb/elasticsearch/client/search_request_test.go new file mode 100644 index 00000000000..d93f8826442 --- /dev/null +++ b/pkg/tsdb/elasticsearch/client/search_request_test.go @@ -0,0 +1,471 @@ +package es + +import ( + "encoding/json" + "testing" + + "github.com/grafana/grafana/pkg/components/simplejson" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestSearchRequest(t *testing.T) { + Convey("Test elasticsearch search request", t, func() { + timeField := "@timestamp" + Convey("Given new search request builder for es version 5", func() { + b := NewSearchRequestBuilder(5) + + Convey("When building search request", func() { + sr, err := b.Build() + So(err, ShouldBeNil) + + Convey("Should have size of zero", func() { + So(sr.Size, ShouldEqual, 0) + }) + + Convey("Should have no sorting", func() { + So(sr.Sort, ShouldHaveLength, 0) + }) + + Convey("When marshal to JSON should generate correct json", func() { + body, err := json.Marshal(sr) + So(err, ShouldBeNil) + json, err := simplejson.NewJson([]byte(body)) + So(err, ShouldBeNil) + So(json.Get("size").MustInt(500), ShouldEqual, 0) + So(json.Get("sort").Interface(), ShouldBeNil) + So(json.Get("aggs").Interface(), ShouldBeNil) + So(json.Get("query").Interface(), ShouldBeNil) + }) + }) + + Convey("When adding size, sort, filters", func() { + b.Size(200) + b.SortDesc(timeField, "boolean") + filters := b.Query().Bool().Filter() + filters.AddDateRangeFilter(timeField, "$timeTo", "$timeFrom", DateFormatEpochMS) + filters.AddQueryStringFilter("test", true) + + Convey("When building search request", func() { + sr, err := b.Build() + So(err, ShouldBeNil) + + Convey("Should have correct size", func() { + So(sr.Size, ShouldEqual, 200) + }) + + Convey("Should have correct sorting", func() { + sort, ok := sr.Sort[timeField].(map[string]string) + So(ok, ShouldBeTrue) + So(sort["order"], ShouldEqual, "desc") + So(sort["unmapped_type"], ShouldEqual, "boolean") + }) + + Convey("Should have range filter", func() { + f, ok := sr.Query.Bool.Filters[0].(*RangeFilter) + So(ok, ShouldBeTrue) + So(f.Gte, ShouldEqual, "$timeFrom") + So(f.Lte, ShouldEqual, "$timeTo") + So(f.Format, ShouldEqual, "epoch_millis") + }) + + Convey("Should have query string filter", func() { + f, ok := sr.Query.Bool.Filters[1].(*QueryStringFilter) + So(ok, ShouldBeTrue) + So(f.Query, ShouldEqual, "test") + So(f.AnalyzeWildcard, ShouldBeTrue) + }) + + Convey("When marshal to JSON should generate correct json", func() { + body, err := json.Marshal(sr) + So(err, ShouldBeNil) + json, err := simplejson.NewJson([]byte(body)) + So(err, ShouldBeNil) + So(json.Get("size").MustInt(0), ShouldEqual, 200) + + sort := json.GetPath("sort", timeField) + So(sort.Get("order").MustString(), ShouldEqual, "desc") + So(sort.Get("unmapped_type").MustString(), ShouldEqual, "boolean") + + timeRangeFilter := json.GetPath("query", "bool", "filter").GetIndex(0).Get("range").Get(timeField) + So(timeRangeFilter.Get("gte").MustString(""), ShouldEqual, "$timeFrom") + So(timeRangeFilter.Get("lte").MustString(""), ShouldEqual, "$timeTo") + So(timeRangeFilter.Get("format").MustString(""), ShouldEqual, DateFormatEpochMS) + + queryStringFilter := json.GetPath("query", "bool", "filter").GetIndex(1).Get("query_string") + So(queryStringFilter.Get("analyze_wildcard").MustBool(false), ShouldEqual, true) + So(queryStringFilter.Get("query").MustString(""), ShouldEqual, "test") + }) + }) + }) + + Convey("When adding doc value field", func() { + b.AddDocValueField(timeField) + + Convey("should set correct props", func() { + So(b.customProps["fields"], ShouldBeNil) + + scriptFields, ok := b.customProps["script_fields"].(map[string]interface{}) + So(ok, ShouldBeTrue) + So(scriptFields, ShouldHaveLength, 0) + + docValueFields, ok := b.customProps["docvalue_fields"].([]string) + So(ok, ShouldBeTrue) + So(docValueFields, ShouldHaveLength, 1) + So(docValueFields[0], ShouldEqual, timeField) + }) + + Convey("When building search request", func() { + sr, err := b.Build() + So(err, ShouldBeNil) + + Convey("When marshal to JSON should generate correct json", func() { + body, err := json.Marshal(sr) + So(err, ShouldBeNil) + json, err := simplejson.NewJson([]byte(body)) + So(err, ShouldBeNil) + + scriptFields, err := json.Get("script_fields").Map() + So(err, ShouldBeNil) + So(scriptFields, ShouldHaveLength, 0) + + _, err = json.Get("fields").StringArray() + So(err, ShouldNotBeNil) + + docValueFields, err := json.Get("docvalue_fields").StringArray() + So(err, ShouldBeNil) + So(docValueFields, ShouldHaveLength, 1) + So(docValueFields[0], ShouldEqual, timeField) + }) + }) + }) + + Convey("and adding multiple top level aggs", func() { + aggBuilder := b.Agg() + aggBuilder.Terms("1", "@hostname", nil) + aggBuilder.DateHistogram("2", "@timestamp", nil) + + Convey("When building search request", func() { + sr, err := b.Build() + So(err, ShouldBeNil) + + Convey("Should have 2 top level aggs", func() { + aggs := sr.Aggs + So(aggs, ShouldHaveLength, 2) + So(aggs[0].Key, ShouldEqual, "1") + So(aggs[0].Aggregation.Type, ShouldEqual, "terms") + So(aggs[1].Key, ShouldEqual, "2") + So(aggs[1].Aggregation.Type, ShouldEqual, "date_histogram") + }) + + Convey("When marshal to JSON should generate correct json", func() { + body, err := json.Marshal(sr) + So(err, ShouldBeNil) + json, err := simplejson.NewJson([]byte(body)) + So(err, ShouldBeNil) + + So(json.Get("aggs").MustMap(), ShouldHaveLength, 2) + So(json.GetPath("aggs", "1", "terms", "field").MustString(), ShouldEqual, "@hostname") + So(json.GetPath("aggs", "2", "date_histogram", "field").MustString(), ShouldEqual, "@timestamp") + }) + }) + }) + + Convey("and adding top level agg with child agg", func() { + aggBuilder := b.Agg() + aggBuilder.Terms("1", "@hostname", func(a *TermsAggregation, ib AggBuilder) { + ib.DateHistogram("2", "@timestamp", nil) + }) + + Convey("When building search request", func() { + sr, err := b.Build() + So(err, ShouldBeNil) + + Convey("Should have 1 top level agg and one child agg", func() { + aggs := sr.Aggs + So(aggs, ShouldHaveLength, 1) + + topAgg := aggs[0] + So(topAgg.Key, ShouldEqual, "1") + So(topAgg.Aggregation.Type, ShouldEqual, "terms") + So(topAgg.Aggregation.Aggs, ShouldHaveLength, 1) + + childAgg := aggs[0].Aggregation.Aggs[0] + So(childAgg.Key, ShouldEqual, "2") + So(childAgg.Aggregation.Type, ShouldEqual, "date_histogram") + }) + + Convey("When marshal to JSON should generate correct json", func() { + body, err := json.Marshal(sr) + So(err, ShouldBeNil) + json, err := simplejson.NewJson([]byte(body)) + So(err, ShouldBeNil) + + So(json.Get("aggs").MustMap(), ShouldHaveLength, 1) + firstLevelAgg := json.GetPath("aggs", "1") + secondLevelAgg := firstLevelAgg.GetPath("aggs", "2") + So(firstLevelAgg.GetPath("terms", "field").MustString(), ShouldEqual, "@hostname") + So(secondLevelAgg.GetPath("date_histogram", "field").MustString(), ShouldEqual, "@timestamp") + }) + }) + }) + + Convey("and adding two top level aggs with child agg", func() { + aggBuilder := b.Agg() + aggBuilder.Histogram("1", "@hostname", func(a *HistogramAgg, ib AggBuilder) { + ib.DateHistogram("2", "@timestamp", nil) + }) + aggBuilder.Filters("3", func(a *FiltersAggregation, ib AggBuilder) { + ib.Terms("4", "@test", nil) + }) + + Convey("When building search request", func() { + sr, err := b.Build() + So(err, ShouldBeNil) + + Convey("Should have 2 top level aggs with one child agg each", func() { + aggs := sr.Aggs + So(aggs, ShouldHaveLength, 2) + + topAggOne := aggs[0] + So(topAggOne.Key, ShouldEqual, "1") + So(topAggOne.Aggregation.Type, ShouldEqual, "histogram") + So(topAggOne.Aggregation.Aggs, ShouldHaveLength, 1) + + topAggOnechildAgg := topAggOne.Aggregation.Aggs[0] + So(topAggOnechildAgg.Key, ShouldEqual, "2") + So(topAggOnechildAgg.Aggregation.Type, ShouldEqual, "date_histogram") + + topAggTwo := aggs[1] + So(topAggTwo.Key, ShouldEqual, "3") + So(topAggTwo.Aggregation.Type, ShouldEqual, "filters") + So(topAggTwo.Aggregation.Aggs, ShouldHaveLength, 1) + + topAggTwochildAgg := topAggTwo.Aggregation.Aggs[0] + So(topAggTwochildAgg.Key, ShouldEqual, "4") + So(topAggTwochildAgg.Aggregation.Type, ShouldEqual, "terms") + }) + + Convey("When marshal to JSON should generate correct json", func() { + body, err := json.Marshal(sr) + So(err, ShouldBeNil) + json, err := simplejson.NewJson([]byte(body)) + So(err, ShouldBeNil) + + topAggOne := json.GetPath("aggs", "1") + So(topAggOne.GetPath("histogram", "field").MustString(), ShouldEqual, "@hostname") + topAggOnechildAgg := topAggOne.GetPath("aggs", "2") + So(topAggOnechildAgg.GetPath("date_histogram", "field").MustString(), ShouldEqual, "@timestamp") + + topAggTwo := json.GetPath("aggs", "3") + topAggTwochildAgg := topAggTwo.GetPath("aggs", "4") + So(topAggTwo.GetPath("filters").MustArray(), ShouldHaveLength, 0) + So(topAggTwochildAgg.GetPath("terms", "field").MustString(), ShouldEqual, "@test") + }) + }) + }) + + Convey("and adding top level agg with child agg with child agg", func() { + aggBuilder := b.Agg() + aggBuilder.Terms("1", "@hostname", func(a *TermsAggregation, ib AggBuilder) { + ib.Terms("2", "@app", func(a *TermsAggregation, ib AggBuilder) { + ib.DateHistogram("3", "@timestamp", nil) + }) + }) + + Convey("When building search request", func() { + sr, err := b.Build() + So(err, ShouldBeNil) + + Convey("Should have 1 top level agg with one child having a child", func() { + aggs := sr.Aggs + So(aggs, ShouldHaveLength, 1) + + topAgg := aggs[0] + So(topAgg.Key, ShouldEqual, "1") + So(topAgg.Aggregation.Type, ShouldEqual, "terms") + So(topAgg.Aggregation.Aggs, ShouldHaveLength, 1) + + childAgg := topAgg.Aggregation.Aggs[0] + So(childAgg.Key, ShouldEqual, "2") + So(childAgg.Aggregation.Type, ShouldEqual, "terms") + + childChildAgg := childAgg.Aggregation.Aggs[0] + So(childChildAgg.Key, ShouldEqual, "3") + So(childChildAgg.Aggregation.Type, ShouldEqual, "date_histogram") + }) + + Convey("When marshal to JSON should generate correct json", func() { + body, err := json.Marshal(sr) + So(err, ShouldBeNil) + json, err := simplejson.NewJson([]byte(body)) + So(err, ShouldBeNil) + + topAgg := json.GetPath("aggs", "1") + So(topAgg.GetPath("terms", "field").MustString(), ShouldEqual, "@hostname") + + childAgg := topAgg.GetPath("aggs", "2") + So(childAgg.GetPath("terms", "field").MustString(), ShouldEqual, "@app") + + childChildAgg := childAgg.GetPath("aggs", "3") + So(childChildAgg.GetPath("date_histogram", "field").MustString(), ShouldEqual, "@timestamp") + }) + }) + }) + + Convey("and adding bucket and metric aggs", func() { + aggBuilder := b.Agg() + aggBuilder.Terms("1", "@hostname", func(a *TermsAggregation, ib AggBuilder) { + ib.Terms("2", "@app", func(a *TermsAggregation, ib AggBuilder) { + ib.Metric("4", "avg", "@value", nil) + ib.DateHistogram("3", "@timestamp", func(a *DateHistogramAgg, ib AggBuilder) { + ib.Metric("4", "avg", "@value", nil) + ib.Metric("5", "max", "@value", nil) + }) + }) + }) + + Convey("When building search request", func() { + sr, err := b.Build() + So(err, ShouldBeNil) + + Convey("Should have 1 top level agg with one child having a child", func() { + aggs := sr.Aggs + So(aggs, ShouldHaveLength, 1) + + topAgg := aggs[0] + So(topAgg.Key, ShouldEqual, "1") + So(topAgg.Aggregation.Type, ShouldEqual, "terms") + So(topAgg.Aggregation.Aggs, ShouldHaveLength, 1) + + childAgg := topAgg.Aggregation.Aggs[0] + So(childAgg.Key, ShouldEqual, "2") + So(childAgg.Aggregation.Type, ShouldEqual, "terms") + + childChildOneAgg := childAgg.Aggregation.Aggs[0] + So(childChildOneAgg.Key, ShouldEqual, "4") + So(childChildOneAgg.Aggregation.Type, ShouldEqual, "avg") + + childChildTwoAgg := childAgg.Aggregation.Aggs[1] + So(childChildTwoAgg.Key, ShouldEqual, "3") + So(childChildTwoAgg.Aggregation.Type, ShouldEqual, "date_histogram") + + childChildTwoChildOneAgg := childChildTwoAgg.Aggregation.Aggs[0] + So(childChildTwoChildOneAgg.Key, ShouldEqual, "4") + So(childChildTwoChildOneAgg.Aggregation.Type, ShouldEqual, "avg") + + childChildTwoChildTwoAgg := childChildTwoAgg.Aggregation.Aggs[1] + So(childChildTwoChildTwoAgg.Key, ShouldEqual, "5") + So(childChildTwoChildTwoAgg.Aggregation.Type, ShouldEqual, "max") + }) + + Convey("When marshal to JSON should generate correct json", func() { + body, err := json.Marshal(sr) + So(err, ShouldBeNil) + json, err := simplejson.NewJson([]byte(body)) + So(err, ShouldBeNil) + + termsAgg := json.GetPath("aggs", "1") + So(termsAgg.GetPath("terms", "field").MustString(), ShouldEqual, "@hostname") + + termsAggTwo := termsAgg.GetPath("aggs", "2") + So(termsAggTwo.GetPath("terms", "field").MustString(), ShouldEqual, "@app") + + termsAggTwoAvg := termsAggTwo.GetPath("aggs", "4") + So(termsAggTwoAvg.GetPath("avg", "field").MustString(), ShouldEqual, "@value") + + dateHistAgg := termsAggTwo.GetPath("aggs", "3") + So(dateHistAgg.GetPath("date_histogram", "field").MustString(), ShouldEqual, "@timestamp") + + avgAgg := dateHistAgg.GetPath("aggs", "4") + So(avgAgg.GetPath("avg", "field").MustString(), ShouldEqual, "@value") + + maxAgg := dateHistAgg.GetPath("aggs", "5") + So(maxAgg.GetPath("max", "field").MustString(), ShouldEqual, "@value") + }) + }) + }) + }) + + Convey("Given new search request builder for es version 2", func() { + b := NewSearchRequestBuilder(2) + + Convey("When adding doc value field", func() { + b.AddDocValueField(timeField) + + Convey("should set correct props", func() { + fields, ok := b.customProps["fields"].([]string) + So(ok, ShouldBeTrue) + So(fields, ShouldHaveLength, 2) + So(fields[0], ShouldEqual, "*") + So(fields[1], ShouldEqual, "_source") + + scriptFields, ok := b.customProps["script_fields"].(map[string]interface{}) + So(ok, ShouldBeTrue) + So(scriptFields, ShouldHaveLength, 0) + + fieldDataFields, ok := b.customProps["fielddata_fields"].([]string) + So(ok, ShouldBeTrue) + So(fieldDataFields, ShouldHaveLength, 1) + So(fieldDataFields[0], ShouldEqual, timeField) + }) + + Convey("When building search request", func() { + sr, err := b.Build() + So(err, ShouldBeNil) + + Convey("When marshal to JSON should generate correct json", func() { + body, err := json.Marshal(sr) + So(err, ShouldBeNil) + json, err := simplejson.NewJson([]byte(body)) + So(err, ShouldBeNil) + + scriptFields, err := json.Get("script_fields").Map() + So(err, ShouldBeNil) + So(scriptFields, ShouldHaveLength, 0) + + fields, err := json.Get("fields").StringArray() + So(err, ShouldBeNil) + So(fields, ShouldHaveLength, 2) + So(fields[0], ShouldEqual, "*") + So(fields[1], ShouldEqual, "_source") + + fieldDataFields, err := json.Get("fielddata_fields").StringArray() + So(err, ShouldBeNil) + So(fieldDataFields, ShouldHaveLength, 1) + So(fieldDataFields[0], ShouldEqual, timeField) + }) + }) + }) + }) + }) +} + +func TestMultiSearchRequest(t *testing.T) { + Convey("Test elasticsearch multi search request", t, func() { + Convey("Given new multi search request builder", func() { + b := NewMultiSearchRequestBuilder(0) + + Convey("When adding one search request", func() { + b.Search() + + Convey("When building search request should contain one search request", func() { + mr, err := b.Build() + So(err, ShouldBeNil) + So(mr.Requests, ShouldHaveLength, 1) + }) + }) + + Convey("When adding two search requests", func() { + b.Search() + b.Search() + + Convey("When building search request should contain two search requests", func() { + mr, err := b.Build() + So(err, ShouldBeNil) + So(mr.Requests, ShouldHaveLength, 2) + }) + }) + }) + }) +} From 4840adff00019b3372ec5840eb34b08822d1f82f Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 23 May 2018 15:09:58 +0200 Subject: [PATCH 13/87] elasticsearch: refactor query handling and use new es simple client Removes moment dependency. Adds response parser tests (based on frontend tests). Adds time series query tests (based on frontend tests). Fixes various issues related to response parsing and building search request queries. Added support for extended stats metrics and geo hash grid aggregations. --- Gopkg.lock | 8 +- Gopkg.toml | 4 - pkg/tsdb/elasticsearch/elasticsearch.go | 34 +- pkg/tsdb/elasticsearch/elasticsearch_test.go | 121 -- pkg/tsdb/elasticsearch/models.go | 150 +-- pkg/tsdb/elasticsearch/query.go | 296 ---- pkg/tsdb/elasticsearch/query_def.go | 43 - pkg/tsdb/elasticsearch/query_test.go | 297 ----- pkg/tsdb/elasticsearch/response_parser.go | 381 +++++- .../elasticsearch/response_parser_test.go | 960 +++++++++++-- pkg/tsdb/elasticsearch/time_series_query.go | 300 +++-- .../elasticsearch/time_series_query_test.go | 518 ++++++- vendor/github.com/leibowitz/moment/diff.go | 75 -- vendor/github.com/leibowitz/moment/moment.go | 1185 ----------------- .../leibowitz/moment/moment_parser.go | 100 -- .../github.com/leibowitz/moment/parse_day.go | 32 - .../leibowitz/moment/strftime_parser.go | 68 - 17 files changed, 1966 insertions(+), 2606 deletions(-) delete mode 100644 pkg/tsdb/elasticsearch/elasticsearch_test.go delete mode 100644 pkg/tsdb/elasticsearch/query.go delete mode 100644 pkg/tsdb/elasticsearch/query_def.go delete mode 100644 pkg/tsdb/elasticsearch/query_test.go delete mode 100644 vendor/github.com/leibowitz/moment/diff.go delete mode 100644 vendor/github.com/leibowitz/moment/moment.go delete mode 100644 vendor/github.com/leibowitz/moment/moment_parser.go delete mode 100644 vendor/github.com/leibowitz/moment/parse_day.go delete mode 100644 vendor/github.com/leibowitz/moment/strftime_parser.go diff --git a/Gopkg.lock b/Gopkg.lock index 65548818ca3..41fc92313d1 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -308,12 +308,6 @@ packages = ["."] revision = "7cafcd837844e784b526369c9bce262804aebc60" -[[projects]] - branch = "master" - name = "github.com/leibowitz/moment" - packages = ["."] - revision = "8548108dcca204a1110b99e5fec966817499fe84" - [[projects]] branch = "master" name = "github.com/lib/pq" @@ -667,6 +661,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "4039f122ac5dd045948e003eb7a74c8864df1759b25147f1b2e2e8ad7a8414d6" + inputs-digest = "bd54a1a836599d90b36d4ac1af56d716ef9ca5be4865e217bddd49e3d32a1997" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 98665ab7310..a9f79c402df 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -199,7 +199,3 @@ ignored = [ [[constraint]] name = "github.com/denisenkom/go-mssqldb" revision = "270bc3860bb94dd3a3ffd047377d746c5e276726" - -[[constraint]] - branch = "master" - name = "github.com/leibowitz/moment" diff --git a/pkg/tsdb/elasticsearch/elasticsearch.go b/pkg/tsdb/elasticsearch/elasticsearch.go index abf25feac06..857b847f0f9 100644 --- a/pkg/tsdb/elasticsearch/elasticsearch.go +++ b/pkg/tsdb/elasticsearch/elasticsearch.go @@ -3,17 +3,14 @@ package elasticsearch import ( "context" "fmt" - "net/http" - "net/url" - "path" - "strings" - "time" "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/tsdb" + "github.com/grafana/grafana/pkg/tsdb/elasticsearch/client" ) +// ElasticsearchExecutor represents a handler for handling elasticsearch datasource request type ElasticsearchExecutor struct{} var ( @@ -21,43 +18,28 @@ var ( intervalCalculator tsdb.IntervalCalculator ) +// NewElasticsearchExecutor creates a new elasticsearch executor func NewElasticsearchExecutor(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) { return &ElasticsearchExecutor{}, nil } func init() { glog = log.New("tsdb.elasticsearch") + intervalCalculator = tsdb.NewIntervalCalculator(nil) tsdb.RegisterTsdbQueryEndpoint("elasticsearch", NewElasticsearchExecutor) - intervalCalculator = tsdb.NewIntervalCalculator(&tsdb.IntervalOptions{MinInterval: time.Millisecond * 1}) } +// Query handles an elasticsearch datasource request func (e *ElasticsearchExecutor) Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) { if len(tsdbQuery.Queries) == 0 { return nil, fmt.Errorf("query contains no queries") } - return e.executeTimeSeriesQuery(ctx, dsInfo, tsdbQuery) -} - -func (e *ElasticsearchExecutor) createRequest(dsInfo *models.DataSource, query string) (*http.Request, error) { - u, _ := url.Parse(dsInfo.Url) - u.Path = path.Join(u.Path, "_msearch") - req, err := http.NewRequest(http.MethodPost, u.String(), strings.NewReader(query)) + client, err := es.NewClient(ctx, dsInfo, tsdbQuery.TimeRange) if err != nil { return nil, err } - req.Header.Set("User-Agent", "Grafana") - req.Header.Set("Content-Type", "application/json") - if dsInfo.BasicAuth { - req.SetBasicAuth(dsInfo.BasicAuthUser, dsInfo.BasicAuthPassword) - } - - if !dsInfo.BasicAuth && dsInfo.User != "" { - req.SetBasicAuth(dsInfo.User, dsInfo.Password) - } - - glog.Debug("Elasticsearch request", "url", req.URL.String()) - glog.Debug("Elasticsearch request", "body", query) - return req, nil + query := newTimeSeriesQuery(client, tsdbQuery, intervalCalculator) + return query.execute() } diff --git a/pkg/tsdb/elasticsearch/elasticsearch_test.go b/pkg/tsdb/elasticsearch/elasticsearch_test.go deleted file mode 100644 index ad905299166..00000000000 --- a/pkg/tsdb/elasticsearch/elasticsearch_test.go +++ /dev/null @@ -1,121 +0,0 @@ -package elasticsearch - -import ( - "github.com/grafana/grafana/pkg/components/simplejson" - "time" -) - -var avgWithMovingAvg = Query{ - TimeField: "timestamp", - RawQuery: "(test:query) AND (name:sample)", - Interval: time.Millisecond, - BucketAggs: []*BucketAgg{{ - Field: "timestamp", - ID: "2", - Type: "date_histogram", - Settings: simplejson.NewFromAny(map[string]interface{}{ - "interval": "auto", - "min_doc_count": 0, - "trimEdges": 0, - }), - }}, - Metrics: []*Metric{{ - Field: "value", - ID: "1", - Type: "avg", - Settings: simplejson.NewFromAny(map[string]interface{}{ - "script": map[string]string{ - "inline": "_value * 2", - }, - }), - }, { - Field: "1", - ID: "3", - Type: "moving_avg", - PipelineAggregate: "1", - Settings: simplejson.NewFromAny(map[string]interface{}{ - "minimize": false, - "model": "simple", - "window": 5, - }), - }}, -} - -var wildcardsAndQuotes = Query{ - TimeField: "timestamp", - RawQuery: "scope:$location.leagueconnect.api AND name:*CreateRegistration AND name:\"*.201-responses.rate\"", - Interval: time.Millisecond, - BucketAggs: []*BucketAgg{{ - Field: "timestamp", - ID: "2", - Type: "date_histogram", - Settings: simplejson.NewFromAny(map[string]interface{}{}), - }}, - Metrics: []*Metric{{ - Field: "value", - ID: "1", - Type: "sum", - Settings: simplejson.NewFromAny(map[string]interface{}{}), - }}, -} -var termAggs = Query{ - TimeField: "timestamp", - RawQuery: "(scope:*.hmp.metricsd) AND (name_raw:builtin.general.*_instance_count)", - Interval: time.Millisecond, - BucketAggs: []*BucketAgg{{ - Field: "name_raw", - ID: "4", - Type: "terms", - Settings: simplejson.NewFromAny(map[string]interface{}{ - "order": "desc", - "orderBy": "_term", - "size": "10", - }), - }, { - Field: "timestamp", - ID: "2", - Type: "date_histogram", - Settings: simplejson.NewFromAny(map[string]interface{}{ - "interval": "auto", - "min_doc_count": 0, - "trimEdges": 0, - }), - }}, - Metrics: []*Metric{{ - Field: "value", - ID: "1", - Type: "sum", - Settings: simplejson.NewFromAny(map[string]interface{}{}), - }}, -} - -var filtersAggs = Query{ - TimeField: "time", - RawQuery: "*", - Interval: time.Millisecond, - BucketAggs: []*BucketAgg{{ - ID: "3", - Type: "filters", - Settings: simplejson.NewFromAny(map[string]interface{}{ - "filters": []interface{}{ - map[string]interface{}{"label": "hello", "query": "host:\"67.65.185.232\""}, - }, - }), - }, { - Field: "timestamp", - ID: "2", - Type: "date_histogram", - Settings: simplejson.NewFromAny(map[string]interface{}{ - "interval": "auto", - "min_doc_count": 0, - "trimEdges": 0, - }), - }}, - Metrics: []*Metric{{ - Field: "bytesSent", - ID: "1", - Type: "count", - PipelineAggregate: "select metric", - Settings: simplejson.NewFromAny(map[string]interface{}{}), - }}, -} diff --git a/pkg/tsdb/elasticsearch/models.go b/pkg/tsdb/elasticsearch/models.go index 9cf295cbd0e..b3fdee95b91 100644 --- a/pkg/tsdb/elasticsearch/models.go +++ b/pkg/tsdb/elasticsearch/models.go @@ -1,12 +1,21 @@ package elasticsearch import ( - "bytes" - "encoding/json" - "fmt" "github.com/grafana/grafana/pkg/components/simplejson" ) +// Query represents the time series query model of the datasource +type Query struct { + TimeField string `json:"timeField"` + RawQuery string `json:"query"` + BucketAggs []*BucketAgg `json:"bucketAggs"` + Metrics []*MetricAgg `json:"metrics"` + Alias string `json:"alias"` + Interval string + RefID string +} + +// BucketAgg represents a bucket aggregation of the time series query model of the datasource type BucketAgg struct { Field string `json:"field"` ID string `json:"id"` @@ -14,120 +23,55 @@ type BucketAgg struct { Type string `jsons:"type"` } -type Metric struct { +// MetricAgg represents a metric aggregation of the time series query model of the datasource +type MetricAgg struct { Field string `json:"field"` Hide bool `json:"hide"` ID string `json:"id"` PipelineAggregate string `json:"pipelineAgg"` Settings *simplejson.Json `json:"settings"` + Meta *simplejson.Json `json:"meta"` Type string `json:"type"` } -type QueryHeader struct { - SearchType string `json:"search_type"` - IgnoreUnavailable bool `json:"ignore_unavailable"` - Index interface{} `json:"index"` - MaxConcurrentShardRequests int `json:"max_concurrent_shard_requests,omitempty"` +var metricAggType = map[string]string{ + "count": "Count", + "avg": "Average", + "sum": "Sum", + "max": "Max", + "min": "Min", + "extended_stats": "Extended Stats", + "percentiles": "Percentiles", + "cardinality": "Unique Count", + "moving_avg": "Moving Average", + "derivative": "Derivative", + "raw_document": "Raw Document", } -func (q *QueryHeader) String() string { - r, _ := json.Marshal(q) - return string(r) +var extendedStats = map[string]string{ + "avg": "Avg", + "min": "Min", + "max": "Max", + "sum": "Sum", + "count": "Count", + "std_deviation": "Std Dev", + "std_deviation_bounds_upper": "Std Dev Upper", + "std_deviation_bounds_lower": "Std Dev Lower", } -type Request struct { - Query map[string]interface{} `json:"query"` - Aggs Aggs `json:"aggs"` - Size int `json:"size"` +var pipelineAggType = map[string]string{ + "moving_avg": "moving_avg", + "derivative": "derivative", } -type Aggs map[string]interface{} - -type HistogramAgg struct { - Interval string `json:"interval,omitempty"` - Field string `json:"field"` - MinDocCount int `json:"min_doc_count"` - Missing string `json:"missing,omitempty"` -} - -type DateHistogramAgg struct { - HistogramAgg - ExtendedBounds ExtendedBounds `json:"extended_bounds"` - Format string `json:"format"` -} - -type FiltersAgg struct { - Filters map[string]interface{} `json:"filters"` -} - -type TermsAgg struct { - Field string `json:"field"` - Size int `json:"size"` - Order map[string]interface{} `json:"order"` - Missing string `json:"missing,omitempty"` -} - -type TermsAggWrap struct { - Terms TermsAgg `json:"terms"` - Aggs Aggs `json:"aggs"` -} - -type ExtendedBounds struct { - Min string `json:"min"` - Max string `json:"max"` -} - -type RangeFilter struct { - Range map[string]RangeFilterSetting `json:"range"` -} -type RangeFilterSetting struct { - Gte string `json:"gte"` - Lte string `json:"lte"` - Format string `json:"format"` -} - -func newRangeFilter(field string, rangeFilterSetting RangeFilterSetting) *RangeFilter { - return &RangeFilter{ - map[string]RangeFilterSetting{field: rangeFilterSetting}} -} - -type QueryStringFilter struct { - QueryString QueryStringFilterSetting `json:"query_string"` -} -type QueryStringFilterSetting struct { - AnalyzeWildcard bool `json:"analyze_wildcard"` - Query string `json:"query"` -} - -func newQueryStringFilter(analyzeWildcard bool, query string) *QueryStringFilter { - return &QueryStringFilter{QueryStringFilterSetting{AnalyzeWildcard: analyzeWildcard, Query: query}} -} - -type BoolQuery struct { - Filter []interface{} `json:"filter"` -} - -type Responses struct { - Responses []Response `json:"responses"` -} - -type Response struct { - Status int `json:"status"` - Err map[string]interface{} `json:"error"` - Aggregations map[string]interface{} `json:"aggregations"` -} - -func (r *Response) getErrMsg() string { - var msg bytes.Buffer - errJson := simplejson.NewFromAny(r.Err) - errType, err := errJson.Get("type").String() - if err == nil { - msg.WriteString(fmt.Sprintf("type:%s", errType)) +func isPipelineAgg(metricType string) bool { + if _, ok := pipelineAggType[metricType]; ok { + return true } + return false +} - reason, err := errJson.Get("type").String() - if err == nil { - msg.WriteString(fmt.Sprintf("reason:%s", reason)) - } - return msg.String() +func describeMetric(metricType, field string) string { + text := metricAggType[metricType] + return text + " " + field } diff --git a/pkg/tsdb/elasticsearch/query.go b/pkg/tsdb/elasticsearch/query.go deleted file mode 100644 index 123f8dd5667..00000000000 --- a/pkg/tsdb/elasticsearch/query.go +++ /dev/null @@ -1,296 +0,0 @@ -package elasticsearch - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "strconv" - "strings" - "time" - - "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/tsdb" - "github.com/leibowitz/moment" -) - -var rangeFilterSetting = RangeFilterSetting{Gte: "$timeFrom", - Lte: "$timeTo", - Format: "epoch_millis"} - -type Query struct { - TimeField string `json:"timeField"` - RawQuery string `json:"query"` - BucketAggs []*BucketAgg `json:"bucketAggs"` - Metrics []*Metric `json:"metrics"` - Alias string `json:"alias"` - Interval time.Duration -} - -func (q *Query) Build(queryContext *tsdb.TsdbQuery, dsInfo *models.DataSource) (string, error) { - var req Request - req.Size = 0 - q.renderReqQuery(&req) - - // handle document query - if q.isRawDocumentQuery() { - return "", errors.New("alert not support Raw_Document") - } - - err := q.parseAggs(&req) - if err != nil { - return "", err - } - - reqBytes, err := json.Marshal(req) - reqHeader := getRequestHeader(queryContext.TimeRange, dsInfo) - payload := bytes.Buffer{} - payload.WriteString(reqHeader.String() + "\n") - payload.WriteString(string(reqBytes) + "\n") - return q.renderTemplate(payload.String(), queryContext) -} - -func (q *Query) isRawDocumentQuery() bool { - if len(q.BucketAggs) == 0 { - if len(q.Metrics) > 0 { - metric := simplejson.NewFromAny(q.Metrics[0]) - if metric.Get("type").MustString("") == "raw_document" { - return true - } - } - } - return false -} - -func (q *Query) renderReqQuery(req *Request) { - req.Query = make(map[string]interface{}) - boolQuery := BoolQuery{} - boolQuery.Filter = append(boolQuery.Filter, newRangeFilter(q.TimeField, rangeFilterSetting)) - boolQuery.Filter = append(boolQuery.Filter, newQueryStringFilter(true, q.RawQuery)) - req.Query["bool"] = boolQuery -} - -func (q *Query) parseAggs(req *Request) error { - aggs := make(Aggs) - nestedAggs := aggs - for _, agg := range q.BucketAggs { - esAggs := make(Aggs) - switch agg.Type { - case "date_histogram": - esAggs["date_histogram"] = q.getDateHistogramAgg(agg) - case "histogram": - esAggs["histogram"] = q.getHistogramAgg(agg) - case "filters": - esAggs["filters"] = q.getFilters(agg) - case "terms": - terms := q.getTerms(agg) - esAggs["terms"] = terms.Terms - esAggs["aggs"] = terms.Aggs - case "geohash_grid": - return errors.New("alert not support Geo_Hash_Grid") - } - - if _, ok := nestedAggs["aggs"]; !ok { - nestedAggs["aggs"] = make(Aggs) - } - - if aggs, ok := (nestedAggs["aggs"]).(Aggs); ok { - aggs[agg.ID] = esAggs - } - nestedAggs = esAggs - - } - nestedAggs["aggs"] = make(Aggs) - - for _, metric := range q.Metrics { - subAgg := make(Aggs) - - if metric.Type == "count" { - continue - } - settings := metric.Settings.MustMap(make(map[string]interface{})) - - if isPipelineAgg(metric.Type) { - if _, err := strconv.Atoi(metric.PipelineAggregate); err == nil { - settings["buckets_path"] = metric.PipelineAggregate - } else { - continue - } - - } else { - settings["field"] = metric.Field - } - - subAgg[metric.Type] = settings - nestedAggs["aggs"].(Aggs)[metric.ID] = subAgg - } - req.Aggs = aggs["aggs"].(Aggs) - return nil -} - -func (q *Query) getDateHistogramAgg(target *BucketAgg) *DateHistogramAgg { - agg := &DateHistogramAgg{} - interval, err := target.Settings.Get("interval").String() - if err == nil { - agg.Interval = interval - } - agg.Field = q.TimeField - agg.MinDocCount = target.Settings.Get("min_doc_count").MustInt(0) - agg.ExtendedBounds = ExtendedBounds{"$timeFrom", "$timeTo"} - agg.Format = "epoch_millis" - - if agg.Interval == "auto" { - agg.Interval = "$__interval" - } - - missing, err := target.Settings.Get("missing").String() - if err == nil { - agg.Missing = missing - } - return agg -} - -func (q *Query) getHistogramAgg(target *BucketAgg) *HistogramAgg { - agg := &HistogramAgg{} - interval, err := target.Settings.Get("interval").String() - if err == nil { - agg.Interval = interval - } - - if target.Field != "" { - agg.Field = target.Field - } - agg.MinDocCount = target.Settings.Get("min_doc_count").MustInt(0) - missing, err := target.Settings.Get("missing").String() - if err == nil { - agg.Missing = missing - } - return agg -} - -func (q *Query) getFilters(target *BucketAgg) *FiltersAgg { - agg := &FiltersAgg{} - agg.Filters = map[string]interface{}{} - for _, filter := range target.Settings.Get("filters").MustArray() { - filterJson := simplejson.NewFromAny(filter) - query := filterJson.Get("query").MustString("") - label := filterJson.Get("label").MustString("") - if label == "" { - label = query - } - - agg.Filters[label] = newQueryStringFilter(true, query) - } - return agg -} - -func (q *Query) getTerms(target *BucketAgg) *TermsAggWrap { - agg := &TermsAggWrap{Aggs: make(Aggs)} - agg.Terms.Field = target.Field - if len(target.Settings.MustMap()) == 0 { - return agg - } - sizeStr := target.Settings.Get("size").MustString("") - size, err := strconv.Atoi(sizeStr) - if err != nil { - size = 500 - } - agg.Terms.Size = size - orderBy, err := target.Settings.Get("orderBy").String() - if err == nil { - agg.Terms.Order = make(map[string]interface{}) - agg.Terms.Order[orderBy] = target.Settings.Get("order").MustString("") - if _, err := strconv.Atoi(orderBy); err != nil { - for _, metricI := range q.Metrics { - metric := simplejson.NewFromAny(metricI) - metricId := metric.Get("id").MustString() - if metricId == orderBy { - subAggs := make(Aggs) - metricField := metric.Get("field").MustString() - metricType := metric.Get("type").MustString() - subAggs[metricType] = map[string]string{"field": metricField} - agg.Aggs = make(Aggs) - agg.Aggs[metricId] = subAggs - break - } - } - } - } - - missing, err := target.Settings.Get("missing").String() - if err == nil { - agg.Terms.Missing = missing - } - - return agg -} - -func (q *Query) renderTemplate(payload string, queryContext *tsdb.TsdbQuery) (string, error) { - timeRange := queryContext.TimeRange - interval := intervalCalculator.Calculate(timeRange, q.Interval) - payload = strings.Replace(payload, "$timeFrom", fmt.Sprintf("%d", timeRange.GetFromAsMsEpoch()), -1) - payload = strings.Replace(payload, "$timeTo", fmt.Sprintf("%d", timeRange.GetToAsMsEpoch()), -1) - payload = strings.Replace(payload, "$interval", interval.Text, -1) - payload = strings.Replace(payload, "$__interval_ms", strconv.FormatInt(interval.Value.Nanoseconds()/int64(time.Millisecond), 10), -1) - payload = strings.Replace(payload, "$__interval", interval.Text, -1) - return payload, nil -} - -func getRequestHeader(timeRange *tsdb.TimeRange, dsInfo *models.DataSource) *QueryHeader { - var header QueryHeader - esVersion := dsInfo.JsonData.Get("esVersion").MustInt() - - searchType := "query_then_fetch" - if esVersion < 5 { - searchType = "count" - } - header.SearchType = searchType - header.IgnoreUnavailable = true - header.Index = getIndexList(dsInfo.Database, dsInfo.JsonData.Get("interval").MustString(), timeRange) - - if esVersion >= 56 { - header.MaxConcurrentShardRequests = dsInfo.JsonData.Get("maxConcurrentShardRequests").MustInt() - } - return &header -} - -func getIndexList(pattern string, interval string, timeRange *tsdb.TimeRange) string { - if interval == "" { - return pattern - } - - var indexes []string - indexParts := strings.Split(strings.TrimLeft(pattern, "["), "]") - indexBase := indexParts[0] - if len(indexParts) <= 1 { - return pattern - } - - indexDateFormat := indexParts[1] - - start := moment.NewMoment(timeRange.MustGetFrom()) - end := moment.NewMoment(timeRange.MustGetTo()) - - indexes = append(indexes, fmt.Sprintf("%s%s", indexBase, start.Format(indexDateFormat))) - for start.IsBefore(*end) { - switch interval { - case "Hourly": - start = start.AddHours(1) - - case "Daily": - start = start.AddDay() - - case "Weekly": - start = start.AddWeeks(1) - - case "Monthly": - start = start.AddMonths(1) - - case "Yearly": - start = start.AddYears(1) - } - indexes = append(indexes, fmt.Sprintf("%s%s", indexBase, start.Format(indexDateFormat))) - } - return strings.Join(indexes, ",") -} diff --git a/pkg/tsdb/elasticsearch/query_def.go b/pkg/tsdb/elasticsearch/query_def.go deleted file mode 100644 index 128e752d97a..00000000000 --- a/pkg/tsdb/elasticsearch/query_def.go +++ /dev/null @@ -1,43 +0,0 @@ -package elasticsearch - -var metricAggType = map[string]string{ - "count": "Count", - "avg": "Average", - "sum": "Sum", - "max": "Max", - "min": "Min", - "extended_stats": "Extended Stats", - "percentiles": "Percentiles", - "cardinality": "Unique Count", - "moving_avg": "Moving Average", - "derivative": "Derivative", - "raw_document": "Raw Document", -} - -var extendedStats = map[string]string{ - "avg": "Avg", - "min": "Min", - "max": "Max", - "sum": "Sum", - "count": "Count", - "std_deviation": "Std Dev", - "std_deviation_bounds_upper": "Std Dev Upper", - "std_deviation_bounds_lower": "Std Dev Lower", -} - -var pipelineOptions = map[string]string{ - "moving_avg": "moving_avg", - "derivative": "derivative", -} - -func isPipelineAgg(metricType string) bool { - if _, ok := pipelineOptions[metricType]; ok { - return true - } - return false -} - -func describeMetric(metricType, field string) string { - text := metricAggType[metricType] - return text + " " + field -} diff --git a/pkg/tsdb/elasticsearch/query_test.go b/pkg/tsdb/elasticsearch/query_test.go deleted file mode 100644 index 1ce6e5ac7bb..00000000000 --- a/pkg/tsdb/elasticsearch/query_test.go +++ /dev/null @@ -1,297 +0,0 @@ -package elasticsearch - -import ( - "encoding/json" - "fmt" - "reflect" - "strconv" - "strings" - "testing" - - "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/tsdb" - . "github.com/smartystreets/goconvey/convey" -) - -func testElasticSearchResponse(query Query, expectedElasticSearchRequestJSON string) { - var queryExpectedJSONInterface, queryJSONInterface interface{} - jsonDate, _ := simplejson.NewJson([]byte(`{"esVersion":2}`)) - dsInfo := &models.DataSource{ - Database: "grafana-test", - JsonData: jsonDate, - } - - testTimeRange := tsdb.NewTimeRange("5m", "now") - - s, err := query.Build(&tsdb.TsdbQuery{TimeRange: testTimeRange}, dsInfo) - So(err, ShouldBeNil) - queryJSON := strings.Split(s, "\n")[1] - err = json.Unmarshal([]byte(queryJSON), &queryJSONInterface) - So(err, ShouldBeNil) - - expectedElasticSearchRequestJSON = strings.Replace( - expectedElasticSearchRequestJSON, - "", - strconv.FormatInt(testTimeRange.GetFromAsMsEpoch(), 10), - -1, - ) - - expectedElasticSearchRequestJSON = strings.Replace( - expectedElasticSearchRequestJSON, - "", - strconv.FormatInt(testTimeRange.GetToAsMsEpoch(), 10), - -1, - ) - - err = json.Unmarshal([]byte(expectedElasticSearchRequestJSON), &queryExpectedJSONInterface) - So(err, ShouldBeNil) - - result := reflect.DeepEqual(queryExpectedJSONInterface, queryJSONInterface) - if !result { - fmt.Printf("ERROR: %s \n != \n %s", expectedElasticSearchRequestJSON, queryJSON) - } - So(result, ShouldBeTrue) -} -func TestElasticSearchQueryBuilder(t *testing.T) { - Convey("Elasticsearch QueryBuilder query testing", t, func() { - Convey("Build test average metric with moving average", func() { - var expectedElasticsearchQueryJSON = ` - { - "size": 0, - "query": { - "bool": { - "filter": [ - { - "range": { - "timestamp": { - "gte": "", - "lte": "", - "format": "epoch_millis" - } - } - }, - { - "query_string": { - "analyze_wildcard": true, - "query": "(test:query) AND (name:sample)" - } - } - ] - } - }, - "aggs": { - "2": { - "date_histogram": { - "interval": "200ms", - "field": "timestamp", - "min_doc_count": 0, - "extended_bounds": { - "min": "", - "max": "" - }, - "format": "epoch_millis" - }, - "aggs": { - "1": { - "avg": { - "field": "value", - "script": { - "inline": "_value * 2" - } - } - }, - "3": { - "moving_avg": { - "buckets_path": "1", - "window": 5, - "model": "simple", - "minimize": false - } - } - } - } - } - }` - - testElasticSearchResponse(avgWithMovingAvg, expectedElasticsearchQueryJSON) - }) - Convey("Test Wildcards and Quotes", func() { - expectedElasticsearchQueryJSON := ` - { - "size": 0, - "query": { - "bool": { - "filter": [ - { - "range": { - "timestamp": { - "gte": "", - "lte": "", - "format": "epoch_millis" - } - } - }, - { - "query_string": { - "analyze_wildcard": true, - "query": "scope:$location.leagueconnect.api AND name:*CreateRegistration AND name:\"*.201-responses.rate\"" - } - } - ] - } - }, - "aggs": { - "2": { - "aggs": { - "1": { - "sum": { - "field": "value" - } - } - }, - "date_histogram": { - "extended_bounds": { - "max": "", - "min": "" - }, - "field": "timestamp", - "format": "epoch_millis", - "min_doc_count": 0 - } - } - } - }` - - testElasticSearchResponse(wildcardsAndQuotes, expectedElasticsearchQueryJSON) - }) - Convey("Test Term Aggregates", func() { - expectedElasticsearchQueryJSON := ` - { - "size": 0, - "query": { - "bool": { - "filter": [ - { - "range": { - "timestamp": { - "gte": "", - "lte": "", - "format": "epoch_millis" - } - } - }, - { - "query_string": { - "analyze_wildcard": true, - "query": "(scope:*.hmp.metricsd) AND (name_raw:builtin.general.*_instance_count)" - } - } - ] - } - }, - "aggs": {"4":{"aggs":{"2":{"aggs":{"1":{"sum":{"field":"value"}}},"date_histogram":{"extended_bounds":{"max":"","min":""},"field":"timestamp","format":"epoch_millis","interval":"200ms","min_doc_count":0}}},"terms":{"field":"name_raw","order":{"_term":"desc"},"size":10}}} - }` - - testElasticSearchResponse(termAggs, expectedElasticsearchQueryJSON) - }) - Convey("Test Filters Aggregates", func() { - expectedElasticsearchQueryJSON := `{ - "size": 0, - "query": { - "bool": { - "filter": [ - { - "range": { - "time": { - "gte": "", - "lte": "", - "format": "epoch_millis" - } - } - }, - { - "query_string": { - "analyze_wildcard": true, - "query": "*" - } - } - ] - } - }, - "aggs": { - "3": { - "filters": { - "filters": { - "hello": { - "query_string": { - "query": "host:\"67.65.185.232\"", - "analyze_wildcard": true - } - } - } - }, - "aggs": { - "2": { - "date_histogram": { - "interval": "200ms", - "field": "time", - "min_doc_count": 0, - "extended_bounds": { - "min": "", - "max": "" - }, - "format": "epoch_millis" - }, - "aggs": {} - } - } - } - } - } - ` - - testElasticSearchResponse(filtersAggs, expectedElasticsearchQueryJSON) - }) - }) -} - -func makeTime(hour int) string { - //unixtime 1500000000 == 2017-07-14T02:40:00+00:00 - return strconv.Itoa((1500000000 + hour*60*60) * 1000) -} - -func getIndexListByTime(pattern string, interval string, hour int) string { - timeRange := &tsdb.TimeRange{ - From: makeTime(0), - To: makeTime(hour), - } - return getIndexList(pattern, interval, timeRange) -} - -func TestElasticsearchGetIndexList(t *testing.T) { - Convey("Test Elasticsearch getIndex ", t, func() { - - Convey("Parse Interval Formats", func() { - So(getIndexListByTime("[logstash-]YYYY.MM.DD", "Daily", 48), - ShouldEqual, "logstash-2017.07.14,logstash-2017.07.15,logstash-2017.07.16") - - So(len(strings.Split(getIndexListByTime("[logstash-]YYYY.MM.DD.HH", "Hourly", 3), ",")), - ShouldEqual, 4) - - So(getIndexListByTime("[logstash-]YYYY.W", "Weekly", 100), - ShouldEqual, "logstash-2017.28,logstash-2017.29") - - So(getIndexListByTime("[logstash-]YYYY.MM", "Monthly", 700), - ShouldEqual, "logstash-2017.07,logstash-2017.08") - - So(getIndexListByTime("[logstash-]YYYY", "Yearly", 10000), - ShouldEqual, "logstash-2017,logstash-2018,logstash-2019") - }) - - Convey("No Interval", func() { - index := getIndexListByTime("logstash-test", "", 1) - So(index, ShouldEqual, "logstash-test") - }) - }) -} diff --git a/pkg/tsdb/elasticsearch/response_parser.go b/pkg/tsdb/elasticsearch/response_parser.go index ec7d2f9eb08..029b2e02142 100644 --- a/pkg/tsdb/elasticsearch/response_parser.go +++ b/pkg/tsdb/elasticsearch/response_parser.go @@ -2,39 +2,79 @@ package elasticsearch import ( "errors" - "fmt" + "regexp" + "sort" + "strconv" + "strings" + "github.com/grafana/grafana/pkg/components/null" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/tsdb" - "regexp" - "strconv" - "strings" + "github.com/grafana/grafana/pkg/tsdb/elasticsearch/client" ) -type ElasticsearchResponseParser struct { - Responses []Response +type responseParser struct { + Responses []*es.SearchResponse Targets []*Query } -func (rp *ElasticsearchResponseParser) getTimeSeries() *tsdb.QueryResult { - queryRes := tsdb.NewQueryResult() - for i, res := range rp.Responses { - target := rp.Targets[i] - props := make(map[string]string) - series := make([]*tsdb.TimeSeries, 0) - rp.processBuckets(res.Aggregations, target, &series, props, 0) - rp.nameSeries(&series, target) - queryRes.Series = append(queryRes.Series, series...) +var newResponseParser = func(responses []*es.SearchResponse, targets []*Query) *responseParser { + return &responseParser{ + Responses: responses, + Targets: targets, } - return queryRes } -func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{}, target *Query, series *[]*tsdb.TimeSeries, props map[string]string, depth int) error { +func (rp *responseParser) getTimeSeries() (*tsdb.Response, error) { + result := &tsdb.Response{} + result.Results = make(map[string]*tsdb.QueryResult) + if rp.Responses == nil { + return result, nil + } + + for i, res := range rp.Responses { + target := rp.Targets[i] + + if res.Error != nil { + result.Results[target.RefID] = getErrorFromElasticResponse(res) + continue + } + + queryRes := tsdb.NewQueryResult() + props := make(map[string]string) + table := tsdb.Table{ + Columns: make([]tsdb.TableColumn, 0), + Rows: make([]tsdb.RowValues, 0), + } + err := rp.processBuckets(res.Aggregations, target, &queryRes.Series, &table, props, 0) + if err != nil { + return nil, err + } + rp.nameSeries(&queryRes.Series, target) + rp.trimDatapoints(&queryRes.Series, target) + + if len(table.Rows) > 0 { + queryRes.Tables = append(queryRes.Tables, &table) + } + + result.Results[target.RefID] = queryRes + } + return result, nil +} + +func (rp *responseParser) processBuckets(aggs map[string]interface{}, target *Query, series *tsdb.TimeSeriesSlice, table *tsdb.Table, props map[string]string, depth int) error { var err error maxDepth := len(target.BucketAggs) - 1 - for aggId, v := range aggs { - aggDef, _ := findAgg(target, aggId) + + aggIDs := make([]string, 0) + for k := range aggs { + aggIDs = append(aggIDs, k) + } + sort.Strings(aggIDs) + for _, aggID := range aggIDs { + v := aggs[aggID] + aggDef, _ := findAgg(target, aggID) esAgg := simplejson.NewFromAny(v) if aggDef == nil { continue @@ -43,26 +83,50 @@ func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{ if depth == maxDepth { if aggDef.Type == "date_histogram" { err = rp.processMetrics(esAgg, target, series, props) - if err != nil { - return err - } } else { - return fmt.Errorf("not support type:%s", aggDef.Type) + err = rp.processAggregationDocs(esAgg, aggDef, target, table, props) + } + if err != nil { + return err } } else { - for i, b := range esAgg.Get("buckets").MustArray() { + for _, b := range esAgg.Get("buckets").MustArray() { bucket := simplejson.NewFromAny(b) - newProps := props + newProps := make(map[string]string, 0) + + for k, v := range props { + newProps[k] = v + } + if key, err := bucket.Get("key").String(); err == nil { newProps[aggDef.Field] = key - } else { - props["filter"] = strconv.Itoa(i) + } else if key, err := bucket.Get("key").Int64(); err == nil { + newProps[aggDef.Field] = strconv.FormatInt(key, 10) } if key, err := bucket.Get("key_as_string").String(); err == nil { - props[aggDef.Field] = key + newProps[aggDef.Field] = key + } + err = rp.processBuckets(bucket.MustMap(), target, series, table, newProps, depth+1) + if err != nil { + return err + } + } + + for k, v := range esAgg.Get("buckets").MustMap() { + bucket := simplejson.NewFromAny(v) + newProps := make(map[string]string, 0) + + for k, v := range props { + newProps[k] = v + } + + newProps["filter"] = k + + err = rp.processBuckets(bucket.MustMap(), target, series, table, newProps, depth+1) + if err != nil { + return err } - rp.processBuckets(bucket.MustMap(), target, series, newProps, depth+1) } } @@ -71,7 +135,7 @@ func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{ } -func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, target *Query, series *[]*tsdb.TimeSeries, props map[string]string) error { +func (rp *responseParser) processMetrics(esAgg *simplejson.Json, target *Query, series *tsdb.TimeSeriesSlice, props map[string]string) error { for _, metric := range target.Metrics { if metric.Hide { continue @@ -79,14 +143,20 @@ func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, ta switch metric.Type { case "count": - newSeries := tsdb.TimeSeries{} + newSeries := tsdb.TimeSeries{ + Tags: make(map[string]string), + } + for _, v := range esAgg.Get("buckets").MustArray() { bucket := simplejson.NewFromAny(v) value := castToNullFloat(bucket.Get("doc_count")) key := castToNullFloat(bucket.Get("key")) newSeries.Points = append(newSeries.Points, tsdb.TimePoint{value, key}) } - newSeries.Tags = props + + for k, v := range props { + newSeries.Tags[k] = v + } newSeries.Tags["metric"] = "count" *series = append(*series, &newSeries) @@ -99,9 +169,18 @@ func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, ta firstBucket := simplejson.NewFromAny(buckets[0]) percentiles := firstBucket.GetPath(metric.ID, "values").MustMap() - for percentileName := range percentiles { - newSeries := tsdb.TimeSeries{} - newSeries.Tags = props + percentileKeys := make([]string, 0) + for k := range percentiles { + percentileKeys = append(percentileKeys, k) + } + sort.Strings(percentileKeys) + for _, percentileName := range percentileKeys { + newSeries := tsdb.TimeSeries{ + Tags: make(map[string]string), + } + for k, v := range props { + newSeries.Tags[k] = v + } newSeries.Tags["metric"] = "p" + percentileName newSeries.Tags["field"] = metric.Field for _, v := range buckets { @@ -112,9 +191,49 @@ func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, ta } *series = append(*series, &newSeries) } + case "extended_stats": + buckets := esAgg.Get("buckets").MustArray() + + metaKeys := make([]string, 0) + meta := metric.Meta.MustMap() + for k := range meta { + metaKeys = append(metaKeys, k) + } + sort.Strings(metaKeys) + for _, statName := range metaKeys { + v := meta[statName] + if enabled, ok := v.(bool); !ok || !enabled { + continue + } + + newSeries := tsdb.TimeSeries{ + Tags: make(map[string]string), + } + for k, v := range props { + newSeries.Tags[k] = v + } + newSeries.Tags["metric"] = statName + newSeries.Tags["field"] = metric.Field + + for _, v := range buckets { + bucket := simplejson.NewFromAny(v) + key := castToNullFloat(bucket.Get("key")) + var value null.Float + if statName == "std_deviation_bounds_upper" { + value = castToNullFloat(bucket.GetPath(metric.ID, "std_deviation_bounds", "upper")) + } else if statName == "std_deviation_bounds_lower" { + value = castToNullFloat(bucket.GetPath(metric.ID, "std_deviation_bounds", "lower")) + } else { + value = castToNullFloat(bucket.GetPath(metric.ID, statName)) + } + newSeries.Points = append(newSeries.Points, tsdb.TimePoint{value, key}) + } + *series = append(*series, &newSeries) + } default: - newSeries := tsdb.TimeSeries{} - newSeries.Tags = map[string]string{} + newSeries := tsdb.TimeSeries{ + Tags: make(map[string]string), + } for k, v := range props { newSeries.Tags[k] = v } @@ -142,7 +261,129 @@ func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, ta return nil } -func (rp *ElasticsearchResponseParser) nameSeries(seriesList *[]*tsdb.TimeSeries, target *Query) { +func (rp *responseParser) processAggregationDocs(esAgg *simplejson.Json, aggDef *BucketAgg, target *Query, table *tsdb.Table, props map[string]string) error { + propKeys := make([]string, 0) + for k := range props { + propKeys = append(propKeys, k) + } + sort.Strings(propKeys) + + if len(table.Columns) == 0 { + for _, propKey := range propKeys { + table.Columns = append(table.Columns, tsdb.TableColumn{Text: propKey}) + } + table.Columns = append(table.Columns, tsdb.TableColumn{Text: aggDef.Field}) + } + + addMetricValue := func(values *tsdb.RowValues, metricName string, value null.Float) { + found := false + for _, c := range table.Columns { + if c.Text == metricName { + found = true + break + } + } + if !found { + table.Columns = append(table.Columns, tsdb.TableColumn{Text: metricName}) + } + *values = append(*values, value) + } + + for _, v := range esAgg.Get("buckets").MustArray() { + bucket := simplejson.NewFromAny(v) + values := make(tsdb.RowValues, 0) + + for _, propKey := range propKeys { + values = append(values, props[propKey]) + } + + if key, err := bucket.Get("key").String(); err == nil { + values = append(values, key) + } else { + values = append(values, castToNullFloat(bucket.Get("key"))) + } + + for _, metric := range target.Metrics { + switch metric.Type { + case "count": + addMetricValue(&values, rp.getMetricName(metric.Type), castToNullFloat(bucket.Get("doc_count"))) + break + case "extended_stats": + metaKeys := make([]string, 0) + meta := metric.Meta.MustMap() + for k := range meta { + metaKeys = append(metaKeys, k) + } + sort.Strings(metaKeys) + for _, statName := range metaKeys { + v := meta[statName] + if enabled, ok := v.(bool); !ok || !enabled { + continue + } + + var value null.Float + if statName == "std_deviation_bounds_upper" { + value = castToNullFloat(bucket.GetPath(metric.ID, "std_deviation_bounds", "upper")) + } else if statName == "std_deviation_bounds_lower" { + value = castToNullFloat(bucket.GetPath(metric.ID, "std_deviation_bounds", "lower")) + } else { + value = castToNullFloat(bucket.GetPath(metric.ID, statName)) + } + + addMetricValue(&values, rp.getMetricName(metric.Type), value) + break + } + default: + metricName := rp.getMetricName(metric.Type) + otherMetrics := make([]*MetricAgg, 0) + + for _, m := range target.Metrics { + if m.Type == metric.Type { + otherMetrics = append(otherMetrics, m) + } + } + + if len(otherMetrics) > 1 { + metricName += " " + metric.Field + } + + addMetricValue(&values, metricName, castToNullFloat(bucket.GetPath(metric.ID, "value"))) + break + } + } + + table.Rows = append(table.Rows, values) + } + + return nil +} + +func (rp *responseParser) trimDatapoints(series *tsdb.TimeSeriesSlice, target *Query) { + var histogram *BucketAgg + for _, bucketAgg := range target.BucketAggs { + if bucketAgg.Type == "date_histogram" { + histogram = bucketAgg + break + } + } + + if histogram == nil { + return + } + + trimEdges, err := histogram.Settings.Get("trimEdges").Int() + if err != nil { + return + } + + for _, s := range *series { + if len(s.Points) > trimEdges*2 { + s.Points = s.Points[trimEdges : len(s.Points)-trimEdges] + } + } +} + +func (rp *responseParser) nameSeries(seriesList *tsdb.TimeSeriesSlice, target *Query) { set := make(map[string]string) for _, v := range *seriesList { if metricType, exists := v.Tags["metric"]; exists { @@ -158,7 +399,9 @@ func (rp *ElasticsearchResponseParser) nameSeries(seriesList *[]*tsdb.TimeSeries } -func (rp *ElasticsearchResponseParser) getSeriesName(series *tsdb.TimeSeries, target *Query, metricTypeCount int) string { +var aliasPatternRegex = regexp.MustCompile(`\{\{([\s\S]+?)\}\}`) + +func (rp *responseParser) getSeriesName(series *tsdb.TimeSeries, target *Query, metricTypeCount int) string { metricType := series.Tags["metric"] metricName := rp.getMetricName(metricType) delete(series.Tags, "metric") @@ -170,27 +413,31 @@ func (rp *ElasticsearchResponseParser) getSeriesName(series *tsdb.TimeSeries, ta } if target.Alias != "" { - var re = regexp.MustCompile(`{{([\s\S]+?)}}`) - for _, match := range re.FindAllString(target.Alias, -1) { - group := match[2 : len(match)-2] + seriesName := target.Alias - if strings.HasPrefix(group, "term ") { - if term, ok := series.Tags["term "]; ok { - strings.Replace(target.Alias, match, term, 1) - } + subMatches := aliasPatternRegex.FindAllStringSubmatch(target.Alias, -1) + for _, subMatch := range subMatches { + group := subMatch[0] + + if len(subMatch) > 1 { + group = subMatch[1] + } + + if strings.Index(group, "term ") == 0 { + seriesName = strings.Replace(seriesName, subMatch[0], series.Tags[group[5:]], 1) } if v, ok := series.Tags[group]; ok { - strings.Replace(target.Alias, match, v, 1) + seriesName = strings.Replace(seriesName, subMatch[0], v, 1) } - - switch group { - case "metric": - strings.Replace(target.Alias, match, metricName, 1) - case "field": - strings.Replace(target.Alias, match, field, 1) + if group == "metric" { + seriesName = strings.Replace(seriesName, subMatch[0], metricName, 1) + } + if group == "field" { + seriesName = strings.Replace(seriesName, subMatch[0], field, 1) } - } + + return seriesName } // todo, if field and pipelineAgg if field != "" && isPipelineAgg(metricType) { @@ -204,7 +451,6 @@ func (rp *ElasticsearchResponseParser) getSeriesName(series *tsdb.TimeSeries, ta if !found { metricName = "Unset" } - } else if field != "" { metricName += " " + field } @@ -226,7 +472,7 @@ func (rp *ElasticsearchResponseParser) getSeriesName(series *tsdb.TimeSeries, ta } -func (rp *ElasticsearchResponseParser) getMetricName(metric string) string { +func (rp *responseParser) getMetricName(metric string) string { if text, ok := metricAggType[metric]; ok { return text } @@ -253,11 +499,28 @@ func castToNullFloat(j *simplejson.Json) null.Float { return null.NewFloat(0, false) } -func findAgg(target *Query, aggId string) (*BucketAgg, error) { +func findAgg(target *Query, aggID string) (*BucketAgg, error) { for _, v := range target.BucketAggs { - if aggId == v.ID { + if aggID == v.ID { return v, nil } } - return nil, errors.New("can't found aggDef, aggID:" + aggId) + return nil, errors.New("can't found aggDef, aggID:" + aggID) +} + +func getErrorFromElasticResponse(response *es.SearchResponse) *tsdb.QueryResult { + result := tsdb.NewQueryResult() + json := simplejson.NewFromAny(response.Error) + reason := json.Get("reason").MustString() + rootCauseReason := json.Get("root_cause").GetIndex(0).Get("reason").MustString() + + if rootCauseReason != "" { + result.ErrorString = rootCauseReason + } else if reason != "" { + result.ErrorString = reason + } else { + result.ErrorString = "Unkown elasticsearch error response" + } + + return result } diff --git a/pkg/tsdb/elasticsearch/response_parser_test.go b/pkg/tsdb/elasticsearch/response_parser_test.go index 1df2c4551ae..b00c14cf946 100644 --- a/pkg/tsdb/elasticsearch/response_parser_test.go +++ b/pkg/tsdb/elasticsearch/response_parser_test.go @@ -2,109 +2,879 @@ package elasticsearch import ( "encoding/json" + "fmt" "testing" + "time" + + "github.com/grafana/grafana/pkg/components/null" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/tsdb/elasticsearch/client" "github.com/grafana/grafana/pkg/tsdb" . "github.com/smartystreets/goconvey/convey" ) -func testElasticsearchResponse(body string, target Query) *tsdb.QueryResult { - var responses Responses - err := json.Unmarshal([]byte(body), &responses) - So(err, ShouldBeNil) - - responseParser := ElasticsearchResponseParser{responses.Responses, []*Query{&target}} - return responseParser.getTimeSeries() -} - -func TestElasticSearchResponseParser(t *testing.T) { - Convey("Elasticsearch Response query testing", t, func() { - Convey("Build test average metric with moving average", func() { - responses := `{ - "responses": [ - { - "took": 1, - "timed_out": false, - "_shards": { - "total": 5, - "successful": 5, - "skipped": 0, - "failed": 0 - }, - "hits": { - "total": 4500, - "max_score": 0, - "hits": [] - }, - "aggregations": { - "2": { - "buckets": [ - { - "1": { - "value": null - }, - "key_as_string": "1522205880000", - "key": 1522205880000, - "doc_count": 0 - }, - { - "1": { - "value": 10 - }, - "key_as_string": "1522205940000", - "key": 1522205940000, - "doc_count": 300 - }, - { - "1": { - "value": 10 - }, - "3": { - "value": 20 - }, - "key_as_string": "1522206000000", - "key": 1522206000000, - "doc_count": 300 - }, - { - "1": { - "value": 10 - }, - "3": { - "value": 20 - }, - "key_as_string": "1522206060000", - "key": 1522206060000, - "doc_count": 300 +func TestResponseParser(t *testing.T) { + Convey("Elasticsearch response parser test", t, func() { + Convey("Simple query and count", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "count", "id": "1" }], + "bucketAggs": [{ "type": "date_histogram", "field": "@timestamp", "id": "2" }] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "2": { + "buckets": [ + { + "doc_count": 10, + "key": 1000 + }, + { + "doc_count": 15, + "key": 2000 + } + ] + } } - ] - } - }, - "status": 200 - } - ] -} -` - res := testElasticsearchResponse(responses, avgWithMovingAvg) - So(len(res.Series), ShouldEqual, 2) - So(res.Series[0].Name, ShouldEqual, "Average value") - So(len(res.Series[0].Points), ShouldEqual, 4) - for i, p := range res.Series[0].Points { - if i == 0 { - So(p[0].Valid, ShouldBeFalse) - } else { - So(p[0].Float64, ShouldEqual, 10) - } - So(p[1].Float64, ShouldEqual, 1522205880000+60000*i) - } - - So(res.Series[1].Name, ShouldEqual, "Moving Average Average 1") - So(len(res.Series[1].Points), ShouldEqual, 2) - - for _, p := range res.Series[1].Points { - So(p[0].Float64, ShouldEqual, 20) - } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Series, ShouldHaveLength, 1) + series := queryRes.Series[0] + So(series.Name, ShouldEqual, "Count") + So(series.Points, ShouldHaveLength, 2) + So(series.Points[0][0].Float64, ShouldEqual, 10) + So(series.Points[0][1].Float64, ShouldEqual, 1000) + So(series.Points[1][0].Float64, ShouldEqual, 15) + So(series.Points[1][1].Float64, ShouldEqual, 2000) }) + + Convey("Simple query count & avg aggregation", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "count", "id": "1" }, {"type": "avg", "field": "value", "id": "2" }], + "bucketAggs": [{ "type": "date_histogram", "field": "@timestamp", "id": "3" }] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "3": { + "buckets": [ + { + "2": { "value": 88 }, + "doc_count": 10, + "key": 1000 + }, + { + "2": { "value": 99 }, + "doc_count": 15, + "key": 2000 + } + ] + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Series, ShouldHaveLength, 2) + seriesOne := queryRes.Series[0] + So(seriesOne.Name, ShouldEqual, "Count") + So(seriesOne.Points, ShouldHaveLength, 2) + So(seriesOne.Points[0][0].Float64, ShouldEqual, 10) + So(seriesOne.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesOne.Points[1][0].Float64, ShouldEqual, 15) + So(seriesOne.Points[1][1].Float64, ShouldEqual, 2000) + + seriesTwo := queryRes.Series[1] + So(seriesTwo.Name, ShouldEqual, "Average value") + So(seriesTwo.Points, ShouldHaveLength, 2) + So(seriesTwo.Points[0][0].Float64, ShouldEqual, 88) + So(seriesTwo.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesTwo.Points[1][0].Float64, ShouldEqual, 99) + So(seriesTwo.Points[1][1].Float64, ShouldEqual, 2000) + }) + + Convey("Single group by query one metric", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "count", "id": "1" }], + "bucketAggs": [ + { "type": "terms", "field": "host", "id": "2" }, + { "type": "date_histogram", "field": "@timestamp", "id": "3" } + ] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "2": { + "buckets": [ + { + "3": { + "buckets": [{ "doc_count": 1, "key": 1000 }, { "doc_count": 3, "key": 2000 }] + }, + "doc_count": 4, + "key": "server1" + }, + { + "3": { + "buckets": [{ "doc_count": 2, "key": 1000 }, { "doc_count": 8, "key": 2000 }] + }, + "doc_count": 10, + "key": "server2" + } + ] + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Series, ShouldHaveLength, 2) + seriesOne := queryRes.Series[0] + So(seriesOne.Name, ShouldEqual, "server1") + So(seriesOne.Points, ShouldHaveLength, 2) + So(seriesOne.Points[0][0].Float64, ShouldEqual, 1) + So(seriesOne.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesOne.Points[1][0].Float64, ShouldEqual, 3) + So(seriesOne.Points[1][1].Float64, ShouldEqual, 2000) + + seriesTwo := queryRes.Series[1] + So(seriesTwo.Name, ShouldEqual, "server2") + So(seriesTwo.Points, ShouldHaveLength, 2) + So(seriesTwo.Points[0][0].Float64, ShouldEqual, 2) + So(seriesTwo.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesTwo.Points[1][0].Float64, ShouldEqual, 8) + So(seriesTwo.Points[1][1].Float64, ShouldEqual, 2000) + }) + + Convey("Single group by query two metrics", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "count", "id": "1" }, { "type": "avg", "field": "@value", "id": "4" }], + "bucketAggs": [ + { "type": "terms", "field": "host", "id": "2" }, + { "type": "date_histogram", "field": "@timestamp", "id": "3" } + ] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "2": { + "buckets": [ + { + "3": { + "buckets": [ + { "4": { "value": 10 }, "doc_count": 1, "key": 1000 }, + { "4": { "value": 12 }, "doc_count": 3, "key": 2000 } + ] + }, + "doc_count": 4, + "key": "server1" + }, + { + "3": { + "buckets": [ + { "4": { "value": 20 }, "doc_count": 1, "key": 1000 }, + { "4": { "value": 32 }, "doc_count": 3, "key": 2000 } + ] + }, + "doc_count": 10, + "key": "server2" + } + ] + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Series, ShouldHaveLength, 4) + seriesOne := queryRes.Series[0] + So(seriesOne.Name, ShouldEqual, "server1 Count") + So(seriesOne.Points, ShouldHaveLength, 2) + So(seriesOne.Points[0][0].Float64, ShouldEqual, 1) + So(seriesOne.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesOne.Points[1][0].Float64, ShouldEqual, 3) + So(seriesOne.Points[1][1].Float64, ShouldEqual, 2000) + + seriesTwo := queryRes.Series[1] + So(seriesTwo.Name, ShouldEqual, "server1 Average @value") + So(seriesTwo.Points, ShouldHaveLength, 2) + So(seriesTwo.Points[0][0].Float64, ShouldEqual, 10) + So(seriesTwo.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesTwo.Points[1][0].Float64, ShouldEqual, 12) + So(seriesTwo.Points[1][1].Float64, ShouldEqual, 2000) + + seriesThree := queryRes.Series[2] + So(seriesThree.Name, ShouldEqual, "server2 Count") + So(seriesThree.Points, ShouldHaveLength, 2) + So(seriesThree.Points[0][0].Float64, ShouldEqual, 1) + So(seriesThree.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesThree.Points[1][0].Float64, ShouldEqual, 3) + So(seriesThree.Points[1][1].Float64, ShouldEqual, 2000) + + seriesFour := queryRes.Series[3] + So(seriesFour.Name, ShouldEqual, "server2 Average @value") + So(seriesFour.Points, ShouldHaveLength, 2) + So(seriesFour.Points[0][0].Float64, ShouldEqual, 20) + So(seriesFour.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesFour.Points[1][0].Float64, ShouldEqual, 32) + So(seriesFour.Points[1][1].Float64, ShouldEqual, 2000) + }) + + Convey("With percentiles", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "percentiles", "settings": { "percents": [75, 90] }, "id": "1" }], + "bucketAggs": [{ "type": "date_histogram", "field": "@timestamp", "id": "3" }] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "3": { + "buckets": [ + { + "1": { "values": { "75": 3.3, "90": 5.5 } }, + "doc_count": 10, + "key": 1000 + }, + { + "1": { "values": { "75": 2.3, "90": 4.5 } }, + "doc_count": 15, + "key": 2000 + } + ] + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Series, ShouldHaveLength, 2) + seriesOne := queryRes.Series[0] + So(seriesOne.Name, ShouldEqual, "p75") + So(seriesOne.Points, ShouldHaveLength, 2) + So(seriesOne.Points[0][0].Float64, ShouldEqual, 3.3) + So(seriesOne.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesOne.Points[1][0].Float64, ShouldEqual, 2.3) + So(seriesOne.Points[1][1].Float64, ShouldEqual, 2000) + + seriesTwo := queryRes.Series[1] + So(seriesTwo.Name, ShouldEqual, "p90") + So(seriesTwo.Points, ShouldHaveLength, 2) + So(seriesTwo.Points[0][0].Float64, ShouldEqual, 5.5) + So(seriesTwo.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesTwo.Points[1][0].Float64, ShouldEqual, 4.5) + So(seriesTwo.Points[1][1].Float64, ShouldEqual, 2000) + }) + + Convey("With extended stats", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "extended_stats", "meta": { "max": true, "std_deviation_bounds_upper": true, "std_deviation_bounds_lower": true }, "id": "1" }], + "bucketAggs": [ + { "type": "terms", "field": "host", "id": "3" }, + { "type": "date_histogram", "field": "@timestamp", "id": "4" } + ] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "3": { + "buckets": [ + { + "key": "server1", + "4": { + "buckets": [ + { + "1": { + "max": 10.2, + "min": 5.5, + "std_deviation_bounds": { "upper": 3, "lower": -2 } + }, + "doc_count": 10, + "key": 1000 + } + ] + } + }, + { + "key": "server2", + "4": { + "buckets": [ + { + "1": { + "max": 15.5, + "min": 3.4, + "std_deviation_bounds": { "upper": 4, "lower": -1 } + }, + "doc_count": 10, + "key": 1000 + } + ] + } + } + ] + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Series, ShouldHaveLength, 6) + + seriesOne := queryRes.Series[0] + So(seriesOne.Name, ShouldEqual, "server1 Max") + So(seriesOne.Points, ShouldHaveLength, 1) + So(seriesOne.Points[0][0].Float64, ShouldEqual, 10.2) + So(seriesOne.Points[0][1].Float64, ShouldEqual, 1000) + + seriesTwo := queryRes.Series[1] + So(seriesTwo.Name, ShouldEqual, "server1 Std Dev Lower") + So(seriesTwo.Points, ShouldHaveLength, 1) + So(seriesTwo.Points[0][0].Float64, ShouldEqual, -2) + So(seriesTwo.Points[0][1].Float64, ShouldEqual, 1000) + + seriesThree := queryRes.Series[2] + So(seriesThree.Name, ShouldEqual, "server1 Std Dev Upper") + So(seriesThree.Points, ShouldHaveLength, 1) + So(seriesThree.Points[0][0].Float64, ShouldEqual, 3) + So(seriesThree.Points[0][1].Float64, ShouldEqual, 1000) + + seriesFour := queryRes.Series[3] + So(seriesFour.Name, ShouldEqual, "server2 Max") + So(seriesFour.Points, ShouldHaveLength, 1) + So(seriesFour.Points[0][0].Float64, ShouldEqual, 15.5) + So(seriesFour.Points[0][1].Float64, ShouldEqual, 1000) + + seriesFive := queryRes.Series[4] + So(seriesFive.Name, ShouldEqual, "server2 Std Dev Lower") + So(seriesFive.Points, ShouldHaveLength, 1) + So(seriesFive.Points[0][0].Float64, ShouldEqual, -1) + So(seriesFive.Points[0][1].Float64, ShouldEqual, 1000) + + seriesSix := queryRes.Series[5] + So(seriesSix.Name, ShouldEqual, "server2 Std Dev Upper") + So(seriesSix.Points, ShouldHaveLength, 1) + So(seriesSix.Points[0][0].Float64, ShouldEqual, 4) + So(seriesSix.Points[0][1].Float64, ShouldEqual, 1000) + }) + + Convey("Single group by with alias pattern", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "alias": "{{term @host}} {{metric}} and {{not_exist}} {{@host}}", + "metrics": [{ "type": "count", "id": "1" }], + "bucketAggs": [ + { "type": "terms", "field": "@host", "id": "2" }, + { "type": "date_histogram", "field": "@timestamp", "id": "3" } + ] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "2": { + "buckets": [ + { + "3": { + "buckets": [{ "doc_count": 1, "key": 1000 }, { "doc_count": 3, "key": 2000 }] + }, + "doc_count": 4, + "key": "server1" + }, + { + "3": { + "buckets": [{ "doc_count": 2, "key": 1000 }, { "doc_count": 8, "key": 2000 }] + }, + "doc_count": 10, + "key": "server2" + }, + { + "3": { + "buckets": [{ "doc_count": 2, "key": 1000 }, { "doc_count": 8, "key": 2000 }] + }, + "doc_count": 10, + "key": 0 + } + ] + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Series, ShouldHaveLength, 3) + + seriesOne := queryRes.Series[0] + So(seriesOne.Name, ShouldEqual, "server1 Count and {{not_exist}} server1") + So(seriesOne.Points, ShouldHaveLength, 2) + So(seriesOne.Points[0][0].Float64, ShouldEqual, 1) + So(seriesOne.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesOne.Points[1][0].Float64, ShouldEqual, 3) + So(seriesOne.Points[1][1].Float64, ShouldEqual, 2000) + + seriesTwo := queryRes.Series[1] + So(seriesTwo.Name, ShouldEqual, "server2 Count and {{not_exist}} server2") + So(seriesTwo.Points, ShouldHaveLength, 2) + So(seriesTwo.Points[0][0].Float64, ShouldEqual, 2) + So(seriesTwo.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesTwo.Points[1][0].Float64, ShouldEqual, 8) + So(seriesTwo.Points[1][1].Float64, ShouldEqual, 2000) + + seriesThree := queryRes.Series[2] + So(seriesThree.Name, ShouldEqual, "0 Count and {{not_exist}} 0") + So(seriesThree.Points, ShouldHaveLength, 2) + So(seriesThree.Points[0][0].Float64, ShouldEqual, 2) + So(seriesThree.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesThree.Points[1][0].Float64, ShouldEqual, 8) + So(seriesThree.Points[1][1].Float64, ShouldEqual, 2000) + }) + + Convey("Histogram response", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "count", "id": "1" }], + "bucketAggs": [{ "type": "histogram", "field": "bytes", "id": "3" }] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "3": { + "buckets": [{ "doc_count": 1, "key": 1000 }, { "doc_count": 3, "key": 2000 }, { "doc_count": 2, "key": 3000 }] + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Tables, ShouldHaveLength, 1) + + rows := queryRes.Tables[0].Rows + So(rows, ShouldHaveLength, 3) + cols := queryRes.Tables[0].Columns + So(cols, ShouldHaveLength, 2) + + So(cols[0].Text, ShouldEqual, "bytes") + So(cols[1].Text, ShouldEqual, "Count") + + So(rows[0][0].(null.Float).Float64, ShouldEqual, 1000) + So(rows[0][1].(null.Float).Float64, ShouldEqual, 1) + So(rows[1][0].(null.Float).Float64, ShouldEqual, 2000) + So(rows[1][1].(null.Float).Float64, ShouldEqual, 3) + So(rows[2][0].(null.Float).Float64, ShouldEqual, 3000) + So(rows[2][1].(null.Float).Float64, ShouldEqual, 2) + }) + + Convey("With two filters agg", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "count", "id": "1" }], + "bucketAggs": [ + { + "type": "filters", + "id": "2", + "settings": { + "filters": [{ "query": "@metric:cpu" }, { "query": "@metric:logins.count" }] + } + }, + { "type": "date_histogram", "field": "@timestamp", "id": "3" } + ] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "2": { + "buckets": { + "@metric:cpu": { + "3": { + "buckets": [{ "doc_count": 1, "key": 1000 }, { "doc_count": 3, "key": 2000 }] + } + }, + "@metric:logins.count": { + "3": { + "buckets": [{ "doc_count": 2, "key": 1000 }, { "doc_count": 8, "key": 2000 }] + } + } + } + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Series, ShouldHaveLength, 2) + + seriesOne := queryRes.Series[0] + So(seriesOne.Name, ShouldEqual, "@metric:cpu") + So(seriesOne.Points, ShouldHaveLength, 2) + So(seriesOne.Points[0][0].Float64, ShouldEqual, 1) + So(seriesOne.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesOne.Points[1][0].Float64, ShouldEqual, 3) + So(seriesOne.Points[1][1].Float64, ShouldEqual, 2000) + + seriesTwo := queryRes.Series[1] + So(seriesTwo.Name, ShouldEqual, "@metric:logins.count") + So(seriesTwo.Points, ShouldHaveLength, 2) + So(seriesTwo.Points[0][0].Float64, ShouldEqual, 2) + So(seriesTwo.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesTwo.Points[1][0].Float64, ShouldEqual, 8) + So(seriesTwo.Points[1][1].Float64, ShouldEqual, 2000) + }) + + Convey("With dropfirst and last aggregation", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "avg", "id": "1" }, { "type": "count" }], + "bucketAggs": [ + { + "type": "date_histogram", + "field": "@timestamp", + "id": "2", + "settings": { "trimEdges": 1 } + } + ] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "2": { + "buckets": [ + { + "1": { "value": 1000 }, + "key": 1, + "doc_count": 369 + }, + { + "1": { "value": 2000 }, + "key": 2, + "doc_count": 200 + }, + { + "1": { "value": 2000 }, + "key": 3, + "doc_count": 200 + } + ] + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Series, ShouldHaveLength, 2) + + seriesOne := queryRes.Series[0] + So(seriesOne.Name, ShouldEqual, "Average") + So(seriesOne.Points, ShouldHaveLength, 1) + So(seriesOne.Points[0][0].Float64, ShouldEqual, 2000) + So(seriesOne.Points[0][1].Float64, ShouldEqual, 2) + + seriesTwo := queryRes.Series[1] + So(seriesTwo.Name, ShouldEqual, "Count") + So(seriesTwo.Points, ShouldHaveLength, 1) + So(seriesTwo.Points[0][0].Float64, ShouldEqual, 200) + So(seriesTwo.Points[0][1].Float64, ShouldEqual, 2) + }) + + Convey("No group by time", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "avg", "id": "1" }, { "type": "count" }], + "bucketAggs": [{ "type": "terms", "field": "host", "id": "2" }] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "2": { + "buckets": [ + { + "1": { "value": 1000 }, + "key": "server-1", + "doc_count": 369 + }, + { + "1": { "value": 2000 }, + "key": "server-2", + "doc_count": 200 + } + ] + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Tables, ShouldHaveLength, 1) + + rows := queryRes.Tables[0].Rows + So(rows, ShouldHaveLength, 2) + cols := queryRes.Tables[0].Columns + So(cols, ShouldHaveLength, 3) + + So(cols[0].Text, ShouldEqual, "host") + So(cols[1].Text, ShouldEqual, "Average") + So(cols[2].Text, ShouldEqual, "Count") + + So(rows[0][0].(string), ShouldEqual, "server-1") + So(rows[0][1].(null.Float).Float64, ShouldEqual, 1000) + So(rows[0][2].(null.Float).Float64, ShouldEqual, 369) + So(rows[1][0].(string), ShouldEqual, "server-2") + So(rows[1][1].(null.Float).Float64, ShouldEqual, 2000) + So(rows[1][2].(null.Float).Float64, ShouldEqual, 200) + }) + + Convey("Multiple metrics of same type", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "avg", "field": "test", "id": "1" }, { "type": "avg", "field": "test2", "id": "2" }], + "bucketAggs": [{ "type": "terms", "field": "host", "id": "2" }] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "2": { + "buckets": [ + { + "1": { "value": 1000 }, + "2": { "value": 3000 }, + "key": "server-1", + "doc_count": 369 + } + ] + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Tables, ShouldHaveLength, 1) + + rows := queryRes.Tables[0].Rows + So(rows, ShouldHaveLength, 1) + cols := queryRes.Tables[0].Columns + So(cols, ShouldHaveLength, 3) + + So(cols[0].Text, ShouldEqual, "host") + So(cols[1].Text, ShouldEqual, "Average test") + So(cols[2].Text, ShouldEqual, "Average test2") + + So(rows[0][0].(string), ShouldEqual, "server-1") + So(rows[0][1].(null.Float).Float64, ShouldEqual, 1000) + So(rows[0][2].(null.Float).Float64, ShouldEqual, 3000) + }) + + // Convey("Raw documents query", func() { + // targets := map[string]string{ + // "A": `{ + // "timeField": "@timestamp", + // "metrics": [{ "type": "raw_document", "id": "1" }] + // }`, + // } + // response := `{ + // "responses": [ + // { + // "hits": { + // "total": 100, + // "hits": [ + // { + // "_id": "1", + // "_type": "type", + // "_index": "index", + // "_source": { "sourceProp": "asd" }, + // "fields": { "fieldProp": "field" } + // }, + // { + // "_source": { "sourceProp": "asd2" }, + // "fields": { "fieldProp": "field2" } + // } + // ] + // } + // } + // ] + // }` + // rp, err := newResponseParserForTest(targets, response) + // So(err, ShouldBeNil) + // result, err := rp.getTimeSeries() + // So(err, ShouldBeNil) + // So(result.Results, ShouldHaveLength, 1) + + // queryRes := result.Results["A"] + // So(queryRes, ShouldNotBeNil) + // So(queryRes.Tables, ShouldHaveLength, 1) + + // rows := queryRes.Tables[0].Rows + // So(rows, ShouldHaveLength, 1) + // cols := queryRes.Tables[0].Columns + // So(cols, ShouldHaveLength, 3) + + // So(cols[0].Text, ShouldEqual, "host") + // So(cols[1].Text, ShouldEqual, "Average test") + // So(cols[2].Text, ShouldEqual, "Average test2") + + // So(rows[0][0].(string), ShouldEqual, "server-1") + // So(rows[0][1].(null.Float).Float64, ShouldEqual, 1000) + // So(rows[0][2].(null.Float).Float64, ShouldEqual, 3000) + // }) }) } + +func newResponseParserForTest(tsdbQueries map[string]string, responseBody string) (*responseParser, error) { + from := time.Date(2018, 5, 15, 17, 50, 0, 0, time.UTC) + to := time.Date(2018, 5, 15, 17, 55, 0, 0, time.UTC) + fromStr := fmt.Sprintf("%d", from.UnixNano()/int64(time.Millisecond)) + toStr := fmt.Sprintf("%d", to.UnixNano()/int64(time.Millisecond)) + tsdbQuery := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{}, + TimeRange: tsdb.NewTimeRange(fromStr, toStr), + } + + for refID, tsdbQueryBody := range tsdbQueries { + tsdbQueryJSON, err := simplejson.NewJson([]byte(tsdbQueryBody)) + if err != nil { + return nil, err + } + + tsdbQuery.Queries = append(tsdbQuery.Queries, &tsdb.Query{ + Model: tsdbQueryJSON, + RefId: refID, + }) + } + + var response es.MultiSearchResponse + err := json.Unmarshal([]byte(responseBody), &response) + if err != nil { + return nil, err + } + + tsQueryParser := newTimeSeriesQueryParser() + queries, err := tsQueryParser.parse(tsdbQuery) + if err != nil { + return nil, err + } + + return newResponseParser(response.Responses, queries), nil +} diff --git a/pkg/tsdb/elasticsearch/time_series_query.go b/pkg/tsdb/elasticsearch/time_series_query.go index af8f61eb144..ae4af7704fb 100644 --- a/pkg/tsdb/elasticsearch/time_series_query.go +++ b/pkg/tsdb/elasticsearch/time_series_query.go @@ -1,99 +1,246 @@ package elasticsearch import ( - "bytes" - "context" - "encoding/json" - "errors" "fmt" + "strconv" + "strings" "time" "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/tsdb" - "golang.org/x/net/context/ctxhttp" + "github.com/grafana/grafana/pkg/tsdb/elasticsearch/client" ) type timeSeriesQuery struct { - queries []*Query + client es.Client + tsdbQuery *tsdb.TsdbQuery + intervalCalculator tsdb.IntervalCalculator } -func (e *ElasticsearchExecutor) executeTimeSeriesQuery(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) { +var newTimeSeriesQuery = func(client es.Client, tsdbQuery *tsdb.TsdbQuery, intervalCalculator tsdb.IntervalCalculator) *timeSeriesQuery { + return &timeSeriesQuery{ + client: client, + tsdbQuery: tsdbQuery, + intervalCalculator: intervalCalculator, + } +} + +func (e *timeSeriesQuery) execute() (*tsdb.Response, error) { result := &tsdb.Response{} result.Results = make(map[string]*tsdb.QueryResult) - tsQueryParser := newTimeSeriesQueryParser(dsInfo) - query, err := tsQueryParser.parse(tsdbQuery) + tsQueryParser := newTimeSeriesQueryParser() + queries, err := tsQueryParser.parse(e.tsdbQuery) if err != nil { return nil, err } - buff := bytes.Buffer{} - for _, q := range query.queries { - s, err := q.Build(tsdbQuery, dsInfo) + ms := e.client.MultiSearch() + + from := fmt.Sprintf("%d", e.tsdbQuery.TimeRange.GetFromAsMsEpoch()) + to := fmt.Sprintf("%d", e.tsdbQuery.TimeRange.GetToAsMsEpoch()) + + for _, q := range queries { + minInterval, err := e.client.GetMinInterval(q.Interval) if err != nil { return nil, err } - buff.WriteString(s) - } - payload := buff.String() + interval := e.intervalCalculator.Calculate(e.tsdbQuery.TimeRange, minInterval) - if setting.Env == setting.DEV { - glog.Debug("Elasticsearch playload", "raw playload", payload) - } - glog.Info("Elasticsearch playload", "raw playload", payload) + b := ms.Search() + b.Size(0) + filters := b.Query().Bool().Filter() + filters.AddDateRangeFilter(e.client.GetTimeField(), to, from, es.DateFormatEpochMS) - req, err := e.createRequest(dsInfo, payload) - if err != nil { - return nil, err - } + if q.RawQuery != "" { + filters.AddQueryStringFilter(q.RawQuery, true) + } - httpClient, err := dsInfo.GetHttpClient() - if err != nil { - return nil, err - } + if len(q.BucketAggs) == 0 { + if len(q.Metrics) == 0 || q.Metrics[0].Type != "raw_document" { + result.Results[q.RefID] = &tsdb.QueryResult{ + RefId: q.RefID, + Error: fmt.Errorf("invalid query, missing metrics and aggregations"), + ErrorString: "invalid query, missing metrics and aggregations", + } + continue + } + metric := q.Metrics[0] + b.Size(metric.Settings.Get("size").MustInt(500)) + b.SortDesc("@timestamp", "boolean") + b.AddDocValueField("@timestamp") + continue + } - resp, err := ctxhttp.Do(ctx, httpClient, req) - if err != nil { - return nil, err - } + aggBuilder := b.Agg() - if resp.StatusCode/100 != 2 { - return nil, fmt.Errorf("elasticsearch returned statuscode invalid status code: %v", resp.Status) - } + // iterate backwards to create aggregations bottom-down + for _, bucketAgg := range q.BucketAggs { + switch bucketAgg.Type { + case "date_histogram": + aggBuilder = addDateHistogramAgg(aggBuilder, bucketAgg, from, to, interval) + case "histogram": + aggBuilder = addHistogramAgg(aggBuilder, bucketAgg) + case "filters": + aggBuilder = addFiltersAgg(aggBuilder, bucketAgg) + case "terms": + aggBuilder = addTermsAgg(aggBuilder, bucketAgg, q.Metrics) + case "geohash_grid": + aggBuilder = addGeoHashGridAgg(aggBuilder, bucketAgg) + } + } - var responses Responses - defer resp.Body.Close() - dec := json.NewDecoder(resp.Body) - dec.UseNumber() - err = dec.Decode(&responses) - if err != nil { - return nil, err - } + for _, m := range q.Metrics { + if m.Type == "count" { + continue + } - for _, res := range responses.Responses { - if res.Err != nil { - return nil, errors.New(res.getErrMsg()) + if isPipelineAgg(m.Type) { + if _, err := strconv.Atoi(m.PipelineAggregate); err == nil { + aggBuilder.Pipeline(m.ID, m.Type, m.PipelineAggregate, func(a *es.PipelineAggregation) { + a.Settings = m.Settings.MustMap() + }) + } else { + continue + } + } else { + aggBuilder.Metric(m.ID, m.Type, m.Field, func(a *es.MetricAggregation) { + a.Settings = m.Settings.MustMap() + }) + } } } - responseParser := ElasticsearchResponseParser{responses.Responses, query.queries} - queryRes := responseParser.getTimeSeries() - result.Results["A"] = queryRes - return result, nil -} -type timeSeriesQueryParser struct { - ds *models.DataSource -} - -func newTimeSeriesQueryParser(ds *models.DataSource) *timeSeriesQueryParser { - return &timeSeriesQueryParser{ - ds: ds, + req, err := ms.Build() + if err != nil { + return nil, err } + + res, err := e.client.ExecuteMultisearch(req) + if err != nil { + return nil, err + } + + rp := newResponseParser(res.Responses, queries) + return rp.getTimeSeries() } -func (p *timeSeriesQueryParser) parse(tsdbQuery *tsdb.TsdbQuery) (*timeSeriesQuery, error) { +func addDateHistogramAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg, timeFrom, timeTo string, interval tsdb.Interval) es.AggBuilder { + aggBuilder.DateHistogram(bucketAgg.ID, bucketAgg.Field, func(a *es.DateHistogramAgg, b es.AggBuilder) { + a.Interval = bucketAgg.Settings.Get("interval").MustString("auto") + a.MinDocCount = bucketAgg.Settings.Get("min_doc_count").MustInt(0) + a.ExtendedBounds = &es.ExtendedBounds{Min: timeFrom, Max: timeTo} + a.Format = bucketAgg.Settings.Get("format").MustString(es.DateFormatEpochMS) + + if a.Interval == "auto" { + a.Interval = "$__interval" + } + + a.Interval = strings.Replace(a.Interval, "$interval", interval.Text, -1) + a.Interval = strings.Replace(a.Interval, "$__interval_ms", strconv.FormatInt(interval.Value.Nanoseconds()/int64(time.Millisecond), 10), -1) + a.Interval = strings.Replace(a.Interval, "$__interval", interval.Text, -1) + + if missing, err := bucketAgg.Settings.Get("missing").String(); err == nil { + a.Missing = &missing + } + + aggBuilder = b + }) + + return aggBuilder +} + +func addHistogramAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg) es.AggBuilder { + aggBuilder.Histogram(bucketAgg.ID, bucketAgg.Field, func(a *es.HistogramAgg, b es.AggBuilder) { + a.Interval = bucketAgg.Settings.Get("interval").MustInt(1000) + a.MinDocCount = bucketAgg.Settings.Get("min_doc_count").MustInt(0) + + if missing, err := bucketAgg.Settings.Get("missing").Int(); err == nil { + a.Missing = &missing + } + + aggBuilder = b + }) + + return aggBuilder +} + +func addTermsAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg, metrics []*MetricAgg) es.AggBuilder { + aggBuilder.Terms(bucketAgg.ID, bucketAgg.Field, func(a *es.TermsAggregation, b es.AggBuilder) { + if size, err := bucketAgg.Settings.Get("size").Int(); err == nil { + a.Size = size + } else if size, err := bucketAgg.Settings.Get("size").String(); err == nil { + a.Size, err = strconv.Atoi(size) + if err != nil { + a.Size = 500 + } + } else { + a.Size = 500 + } + if minDocCount, err := bucketAgg.Settings.Get("min_doc_count").Int(); err == nil { + a.MinDocCount = &minDocCount + } + if missing, err := bucketAgg.Settings.Get("missing").String(); err == nil { + a.Missing = &missing + } + + if orderBy, err := bucketAgg.Settings.Get("orderBy").String(); err == nil { + a.Order[orderBy] = bucketAgg.Settings.Get("order").MustString("desc") + + if _, err := strconv.Atoi(orderBy); err == nil { + for _, m := range metrics { + if m.ID == orderBy { + b.Metric(m.ID, m.Type, m.Field, nil) + break + } + } + } + } + + aggBuilder = b + }) + + return aggBuilder +} + +func addFiltersAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg) es.AggBuilder { + filters := make(map[string]interface{}) + for _, filter := range bucketAgg.Settings.Get("filters").MustArray() { + json := simplejson.NewFromAny(filter) + query := json.Get("query").MustString() + label := json.Get("label").MustString() + if label == "" { + label = query + } + filters[label] = &es.QueryStringFilter{Query: query, AnalyzeWildcard: true} + } + + if len(filters) > 0 { + aggBuilder.Filters(bucketAgg.ID, func(a *es.FiltersAggregation, b es.AggBuilder) { + a.Filters = filters + aggBuilder = b + }) + } + + return aggBuilder +} + +func addGeoHashGridAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg) es.AggBuilder { + aggBuilder.GeoHashGrid(bucketAgg.ID, bucketAgg.Field, func(a *es.GeoHashGridAggregation, b es.AggBuilder) { + a.Precision = bucketAgg.Settings.Get("precision").MustInt(3) + aggBuilder = b + }) + + return aggBuilder +} + +type timeSeriesQueryParser struct{} + +func newTimeSeriesQueryParser() *timeSeriesQueryParser { + return &timeSeriesQueryParser{} +} + +func (p *timeSeriesQueryParser) parse(tsdbQuery *tsdb.TsdbQuery) ([]*Query, error) { queries := make([]*Query, 0) for _, q := range tsdbQuery.Queries { model := q.Model @@ -111,10 +258,7 @@ func (p *timeSeriesQueryParser) parse(tsdbQuery *tsdb.TsdbQuery) (*timeSeriesQue return nil, err } alias := model.Get("alias").MustString("") - parsedInterval, err := tsdb.GetIntervalFrom(p.ds, model, time.Millisecond) - if err != nil { - return nil, err - } + interval := model.Get("interval").MustString() queries = append(queries, &Query{ TimeField: timeField, @@ -122,54 +266,52 @@ func (p *timeSeriesQueryParser) parse(tsdbQuery *tsdb.TsdbQuery) (*timeSeriesQue BucketAggs: bucketAggs, Metrics: metrics, Alias: alias, - Interval: parsedInterval, + Interval: interval, + RefID: q.RefId, }) } - return &timeSeriesQuery{queries: queries}, nil + return queries, nil } func (p *timeSeriesQueryParser) parseBucketAggs(model *simplejson.Json) ([]*BucketAgg, error) { var err error var result []*BucketAgg for _, t := range model.Get("bucketAggs").MustArray() { - aggJson := simplejson.NewFromAny(t) + aggJSON := simplejson.NewFromAny(t) agg := &BucketAgg{} - agg.Type, err = aggJson.Get("type").String() + agg.Type, err = aggJSON.Get("type").String() if err != nil { return nil, err } - agg.ID, err = aggJson.Get("id").String() + agg.ID, err = aggJSON.Get("id").String() if err != nil { return nil, err } - agg.Field = aggJson.Get("field").MustString() - agg.Settings = simplejson.NewFromAny(aggJson.Get("settings").MustMap()) + agg.Field = aggJSON.Get("field").MustString() + agg.Settings = simplejson.NewFromAny(aggJSON.Get("settings").MustMap()) result = append(result, agg) } return result, nil } -func (p *timeSeriesQueryParser) parseMetrics(model *simplejson.Json) ([]*Metric, error) { +func (p *timeSeriesQueryParser) parseMetrics(model *simplejson.Json) ([]*MetricAgg, error) { var err error - var result []*Metric + var result []*MetricAgg for _, t := range model.Get("metrics").MustArray() { metricJSON := simplejson.NewFromAny(t) - metric := &Metric{} + metric := &MetricAgg{} metric.Field = metricJSON.Get("field").MustString() metric.Hide = metricJSON.Get("hide").MustBool(false) - metric.ID, err = metricJSON.Get("id").String() - if err != nil { - return nil, err - } - + metric.ID = metricJSON.Get("id").MustString() metric.PipelineAggregate = metricJSON.Get("pipelineAgg").MustString() metric.Settings = simplejson.NewFromAny(metricJSON.Get("settings").MustMap()) + metric.Meta = simplejson.NewFromAny(metricJSON.Get("meta").MustMap()) metric.Type, err = metricJSON.Get("type").String() if err != nil { diff --git a/pkg/tsdb/elasticsearch/time_series_query_test.go b/pkg/tsdb/elasticsearch/time_series_query_test.go index 4950bc811de..e2af4de749a 100644 --- a/pkg/tsdb/elasticsearch/time_series_query_test.go +++ b/pkg/tsdb/elasticsearch/time_series_query_test.go @@ -1,21 +1,514 @@ package elasticsearch import ( + "fmt" "testing" + "time" + + "github.com/grafana/grafana/pkg/tsdb/elasticsearch/client" "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/tsdb" . "github.com/smartystreets/goconvey/convey" ) +func TestExecuteTimeSeriesQuery(t *testing.T) { + from := time.Date(2018, 5, 15, 17, 50, 0, 0, time.UTC) + to := time.Date(2018, 5, 15, 17, 55, 0, 0, time.UTC) + fromStr := fmt.Sprintf("%d", from.UnixNano()/int64(time.Millisecond)) + toStr := fmt.Sprintf("%d", to.UnixNano()/int64(time.Millisecond)) + + Convey("Test execute time series query", t, func() { + Convey("With defaults on es 2", func() { + c := newFakeClient(2) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [{ "type": "date_histogram", "field": "@timestamp", "id": "2" }], + "metrics": [{"type": "count", "id": "0" }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + rangeFilter := sr.Query.Bool.Filters[0].(*es.RangeFilter) + So(rangeFilter.Key, ShouldEqual, c.timeField) + So(rangeFilter.Lte, ShouldEqual, toStr) + So(rangeFilter.Gte, ShouldEqual, fromStr) + So(rangeFilter.Format, ShouldEqual, es.DateFormatEpochMS) + So(sr.Aggs[0].Key, ShouldEqual, "2") + dateHistogramAgg := sr.Aggs[0].Aggregation.Aggregation.(*es.DateHistogramAgg) + So(dateHistogramAgg.Field, ShouldEqual, "@timestamp") + So(dateHistogramAgg.ExtendedBounds.Min, ShouldEqual, fromStr) + So(dateHistogramAgg.ExtendedBounds.Max, ShouldEqual, toStr) + }) + + Convey("With defaults on es 5", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [{ "type": "date_histogram", "field": "@timestamp", "id": "2" }], + "metrics": [{"type": "count", "id": "0" }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + So(sr.Query.Bool.Filters[0].(*es.RangeFilter).Key, ShouldEqual, c.timeField) + So(sr.Aggs[0].Key, ShouldEqual, "2") + So(sr.Aggs[0].Aggregation.Aggregation.(*es.DateHistogramAgg).ExtendedBounds.Min, ShouldEqual, fromStr) + So(sr.Aggs[0].Aggregation.Aggregation.(*es.DateHistogramAgg).ExtendedBounds.Max, ShouldEqual, toStr) + }) + + Convey("With multiple bucket aggs", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { "type": "terms", "field": "@host", "id": "2" }, + { "type": "date_histogram", "field": "@timestamp", "id": "3" } + ], + "metrics": [{"type": "count", "id": "1" }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + firstLevel := sr.Aggs[0] + So(firstLevel.Key, ShouldEqual, "2") + So(firstLevel.Aggregation.Aggregation.(*es.TermsAggregation).Field, ShouldEqual, "@host") + secondLevel := firstLevel.Aggregation.Aggs[0] + So(secondLevel.Key, ShouldEqual, "3") + So(secondLevel.Aggregation.Aggregation.(*es.DateHistogramAgg).Field, ShouldEqual, "@timestamp") + }) + + Convey("With select field", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { "type": "date_histogram", "field": "@timestamp", "id": "2" } + ], + "metrics": [{"type": "avg", "field": "@value", "id": "1" }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + firstLevel := sr.Aggs[0] + So(firstLevel.Key, ShouldEqual, "2") + So(firstLevel.Aggregation.Aggregation.(*es.DateHistogramAgg).Field, ShouldEqual, "@timestamp") + secondLevel := firstLevel.Aggregation.Aggs[0] + So(secondLevel.Key, ShouldEqual, "1") + So(secondLevel.Aggregation.Type, ShouldEqual, "avg") + So(secondLevel.Aggregation.Aggregation.(*es.MetricAggregation).Field, ShouldEqual, "@value") + }) + + Convey("With term agg and order by metric agg", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { + "type": "terms", + "field": "@host", + "id": "2", + "settings": { "size": "5", "order": "asc", "orderBy": "5" } + }, + { "type": "date_histogram", "field": "@timestamp", "id": "3" } + ], + "metrics": [ + {"type": "count", "id": "1" }, + {"type": "avg", "field": "@value", "id": "5" } + ] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + avgAggOrderBy := sr.Aggs[0].Aggregation.Aggs[0] + So(avgAggOrderBy.Key, ShouldEqual, "5") + So(avgAggOrderBy.Aggregation.Type, ShouldEqual, "avg") + + avgAgg := sr.Aggs[0].Aggregation.Aggs[1].Aggregation.Aggs[0] + So(avgAgg.Key, ShouldEqual, "5") + So(avgAgg.Aggregation.Type, ShouldEqual, "avg") + }) + + Convey("With metric percentiles", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { "type": "date_histogram", "field": "@timestamp", "id": "3" } + ], + "metrics": [ + { + "id": "1", + "type": "percentiles", + "field": "@load_time", + "settings": { + "percents": [ "1", "2", "3", "4" ] + } + } + ] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + percentilesAgg := sr.Aggs[0].Aggregation.Aggs[0] + So(percentilesAgg.Key, ShouldEqual, "1") + So(percentilesAgg.Aggregation.Type, ShouldEqual, "percentiles") + metricAgg := percentilesAgg.Aggregation.Aggregation.(*es.MetricAggregation) + percents := metricAgg.Settings["percents"].([]interface{}) + So(percents, ShouldHaveLength, 4) + So(percents[0], ShouldEqual, "1") + So(percents[1], ShouldEqual, "2") + So(percents[2], ShouldEqual, "3") + So(percents[3], ShouldEqual, "4") + }) + + Convey("With filters aggs on es 2", func() { + c := newFakeClient(2) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { + "id": "2", + "type": "filters", + "settings": { + "filters": [ { "query": "@metric:cpu" }, { "query": "@metric:logins.count" } ] + } + }, + { "type": "date_histogram", "field": "@timestamp", "id": "4" } + ], + "metrics": [{"type": "count", "id": "1" }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + filtersAgg := sr.Aggs[0] + So(filtersAgg.Key, ShouldEqual, "2") + So(filtersAgg.Aggregation.Type, ShouldEqual, "filters") + fAgg := filtersAgg.Aggregation.Aggregation.(*es.FiltersAggregation) + So(fAgg.Filters["@metric:cpu"].(*es.QueryStringFilter).Query, ShouldEqual, "@metric:cpu") + So(fAgg.Filters["@metric:logins.count"].(*es.QueryStringFilter).Query, ShouldEqual, "@metric:logins.count") + + dateHistogramAgg := sr.Aggs[0].Aggregation.Aggs[0] + So(dateHistogramAgg.Key, ShouldEqual, "4") + So(dateHistogramAgg.Aggregation.Aggregation.(*es.DateHistogramAgg).Field, ShouldEqual, "@timestamp") + }) + + Convey("With filters aggs on es 5", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { + "id": "2", + "type": "filters", + "settings": { + "filters": [ { "query": "@metric:cpu" }, { "query": "@metric:logins.count" } ] + } + }, + { "type": "date_histogram", "field": "@timestamp", "id": "4" } + ], + "metrics": [{"type": "count", "id": "1" }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + filtersAgg := sr.Aggs[0] + So(filtersAgg.Key, ShouldEqual, "2") + So(filtersAgg.Aggregation.Type, ShouldEqual, "filters") + fAgg := filtersAgg.Aggregation.Aggregation.(*es.FiltersAggregation) + So(fAgg.Filters["@metric:cpu"].(*es.QueryStringFilter).Query, ShouldEqual, "@metric:cpu") + So(fAgg.Filters["@metric:logins.count"].(*es.QueryStringFilter).Query, ShouldEqual, "@metric:logins.count") + + dateHistogramAgg := sr.Aggs[0].Aggregation.Aggs[0] + So(dateHistogramAgg.Key, ShouldEqual, "4") + So(dateHistogramAgg.Aggregation.Aggregation.(*es.DateHistogramAgg).Field, ShouldEqual, "@timestamp") + }) + + Convey("With raw document metric", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [], + "metrics": [{ "id": "1", "type": "raw_document", "settings": {} }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + So(sr.Size, ShouldEqual, 500) + }) + + Convey("With raw document metric size set", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [], + "metrics": [{ "id": "1", "type": "raw_document", "settings": { "size": 1337 } }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + So(sr.Size, ShouldEqual, 1337) + }) + + Convey("With date histogram agg", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { + "id": "2", + "type": "date_histogram", + "field": "@timestamp", + "settings": { "interval": "auto", "min_doc_count": 2 } + } + ], + "metrics": [{"type": "count", "id": "1" }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + firstLevel := sr.Aggs[0] + So(firstLevel.Key, ShouldEqual, "2") + So(firstLevel.Aggregation.Type, ShouldEqual, "date_histogram") + hAgg := firstLevel.Aggregation.Aggregation.(*es.DateHistogramAgg) + So(hAgg.Field, ShouldEqual, "@timestamp") + So(hAgg.Interval, ShouldEqual, "15s") + So(hAgg.MinDocCount, ShouldEqual, 2) + }) + + Convey("With histogram agg", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { + "id": "3", + "type": "histogram", + "field": "bytes", + "settings": { "interval": 10, "min_doc_count": 2, "missing": 5 } + } + ], + "metrics": [{"type": "count", "id": "1" }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + firstLevel := sr.Aggs[0] + So(firstLevel.Key, ShouldEqual, "3") + So(firstLevel.Aggregation.Type, ShouldEqual, "histogram") + hAgg := firstLevel.Aggregation.Aggregation.(*es.HistogramAgg) + So(hAgg.Field, ShouldEqual, "bytes") + So(hAgg.Interval, ShouldEqual, 10) + So(hAgg.MinDocCount, ShouldEqual, 2) + So(*hAgg.Missing, ShouldEqual, 5) + }) + + Convey("With geo hash grid agg", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { + "id": "3", + "type": "geohash_grid", + "field": "@location", + "settings": { "precision": 3 } + } + ], + "metrics": [{"type": "count", "id": "1" }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + firstLevel := sr.Aggs[0] + So(firstLevel.Key, ShouldEqual, "3") + So(firstLevel.Aggregation.Type, ShouldEqual, "geohash_grid") + ghGridAgg := firstLevel.Aggregation.Aggregation.(*es.GeoHashGridAggregation) + So(ghGridAgg.Field, ShouldEqual, "@location") + So(ghGridAgg.Precision, ShouldEqual, 3) + }) + + Convey("With moving average", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { "type": "date_histogram", "field": "@timestamp", "id": "4" } + ], + "metrics": [ + { "id": "3", "type": "sum", "field": "@value" }, + { + "id": "2", + "type": "moving_avg", + "field": "3", + "pipelineAgg": "3" + } + ] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + firstLevel := sr.Aggs[0] + So(firstLevel.Key, ShouldEqual, "4") + So(firstLevel.Aggregation.Type, ShouldEqual, "date_histogram") + So(firstLevel.Aggregation.Aggs, ShouldHaveLength, 2) + + sumAgg := firstLevel.Aggregation.Aggs[0] + So(sumAgg.Key, ShouldEqual, "3") + So(sumAgg.Aggregation.Type, ShouldEqual, "sum") + mAgg := sumAgg.Aggregation.Aggregation.(*es.MetricAggregation) + So(mAgg.Field, ShouldEqual, "@value") + + movingAvgAgg := firstLevel.Aggregation.Aggs[1] + So(movingAvgAgg.Key, ShouldEqual, "2") + So(movingAvgAgg.Aggregation.Type, ShouldEqual, "moving_avg") + pl := movingAvgAgg.Aggregation.Aggregation.(*es.PipelineAggregation) + So(pl.BucketPath, ShouldEqual, "3") + }) + + Convey("With broken moving average", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { "type": "date_histogram", "field": "@timestamp", "id": "5" } + ], + "metrics": [ + { "id": "3", "type": "sum", "field": "@value" }, + { + "id": "2", + "type": "moving_avg", + "pipelineAgg": "3" + }, + { + "id": "4", + "type": "moving_avg", + "pipelineAgg": "Metric to apply moving average" + } + ] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + firstLevel := sr.Aggs[0] + So(firstLevel.Key, ShouldEqual, "5") + So(firstLevel.Aggregation.Type, ShouldEqual, "date_histogram") + + So(firstLevel.Aggregation.Aggs, ShouldHaveLength, 2) + + movingAvgAgg := firstLevel.Aggregation.Aggs[1] + So(movingAvgAgg.Key, ShouldEqual, "2") + plAgg := movingAvgAgg.Aggregation.Aggregation.(*es.PipelineAggregation) + So(plAgg.BucketPath, ShouldEqual, "3") + }) + + Convey("With derivative", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { "type": "date_histogram", "field": "@timestamp", "id": "4" } + ], + "metrics": [ + { "id": "3", "type": "sum", "field": "@value" }, + { + "id": "2", + "type": "derivative", + "pipelineAgg": "3" + } + ] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + firstLevel := sr.Aggs[0] + So(firstLevel.Key, ShouldEqual, "4") + So(firstLevel.Aggregation.Type, ShouldEqual, "date_histogram") + + derivativeAgg := firstLevel.Aggregation.Aggs[1] + So(derivativeAgg.Key, ShouldEqual, "2") + plAgg := derivativeAgg.Aggregation.Aggregation.(*es.PipelineAggregation) + So(plAgg.BucketPath, ShouldEqual, "3") + }) + + }) +} + +type fakeClient struct { + version int + timeField string + multiSearchResponse *es.MultiSearchResponse + multiSearchError error + builder *es.MultiSearchRequestBuilder + multisearchRequests []*es.MultiSearchRequest +} + +func newFakeClient(version int) *fakeClient { + return &fakeClient{ + version: version, + timeField: "@timestamp", + multisearchRequests: make([]*es.MultiSearchRequest, 0), + multiSearchResponse: &es.MultiSearchResponse{}, + } +} + +func (c *fakeClient) GetVersion() int { + return c.version +} + +func (c *fakeClient) GetTimeField() string { + return c.timeField +} + +func (c *fakeClient) GetMinInterval(queryInterval string) (time.Duration, error) { + return 15 * time.Second, nil +} + +func (c *fakeClient) ExecuteMultisearch(r *es.MultiSearchRequest) (*es.MultiSearchResponse, error) { + c.multisearchRequests = append(c.multisearchRequests, r) + return c.multiSearchResponse, c.multiSearchError +} + +func (c *fakeClient) MultiSearch() *es.MultiSearchRequestBuilder { + c.builder = es.NewMultiSearchRequestBuilder(c.version) + return c.builder +} + +func newTsdbQuery(body string) (*tsdb.TsdbQuery, error) { + json, err := simplejson.NewJson([]byte(body)) + if err != nil { + return nil, err + } + return &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: json, + }, + }, + }, nil +} + +func executeTsdbQuery(c es.Client, body string, from, to time.Time, minInterval time.Duration) (*tsdb.Response, error) { + json, err := simplejson.NewJson([]byte(body)) + if err != nil { + return nil, err + } + fromStr := fmt.Sprintf("%d", from.UnixNano()/int64(time.Millisecond)) + toStr := fmt.Sprintf("%d", to.UnixNano()/int64(time.Millisecond)) + tsdbQuery := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: json, + }, + }, + TimeRange: tsdb.NewTimeRange(fromStr, toStr), + } + query := newTimeSeriesQuery(c, tsdbQuery, tsdb.NewIntervalCalculator(&tsdb.IntervalOptions{MinInterval: minInterval})) + return query.execute() +} + func TestTimeSeriesQueryParser(t *testing.T) { Convey("Test time series query parser", t, func() { - ds := &models.DataSource{} - p := newTimeSeriesQueryParser(ds) + p := newTimeSeriesQueryParser() Convey("Should be able to parse query", func() { - json, err := simplejson.NewJson([]byte(`{ + body := `{ "timeField": "@timestamp", "query": "@metric:cpu", "alias": "{{@hostname}} {{metric}}", @@ -63,21 +556,14 @@ func TestTimeSeriesQueryParser(t *testing.T) { "type": "date_histogram" } ] - }`)) + }` + tsdbQuery, err := newTsdbQuery(body) So(err, ShouldBeNil) - tsdbQuery := &tsdb.TsdbQuery{ - Queries: []*tsdb.Query{ - { - DataSource: ds, - Model: json, - }, - }, - } - tsQuery, err := p.parse(tsdbQuery) + queries, err := p.parse(tsdbQuery) So(err, ShouldBeNil) - So(tsQuery.queries, ShouldHaveLength, 1) + So(queries, ShouldHaveLength, 1) - q := tsQuery.queries[0] + q := queries[0] So(q.TimeField, ShouldEqual, "@timestamp") So(q.RawQuery, ShouldEqual, "@metric:cpu") diff --git a/vendor/github.com/leibowitz/moment/diff.go b/vendor/github.com/leibowitz/moment/diff.go deleted file mode 100644 index 0d6b3935adf..00000000000 --- a/vendor/github.com/leibowitz/moment/diff.go +++ /dev/null @@ -1,75 +0,0 @@ -package moment - -import ( - "fmt" - "math" - "time" -) - -// @todo In months/years requires the old and new to calculate correctly, right? -// @todo decide how to handle rounding (i.e. always floor?) -type Diff struct { - duration time.Duration -} - -func (d *Diff) InSeconds() int { - return int(d.duration.Seconds()) -} - -func (d *Diff) InMinutes() int { - return int(d.duration.Minutes()) -} - -func (d *Diff) InHours() int { - return int(d.duration.Hours()) -} - -func (d *Diff) InDays() int { - return int(math.Floor(float64(d.InSeconds()) / 86400)) -} - -// This depends on where the weeks fall? -func (d *Diff) InWeeks() int { - return int(math.Floor(float64(d.InDays() / 7))) -} - -func (d *Diff) InMonths() int { - return 0 -} - -func (d *Diff) InYears() int { - return 0 -} - -// http://momentjs.com/docs/#/durations/humanize/ -func (d *Diff) Humanize() string { - diffInSeconds := d.InSeconds() - - if diffInSeconds <= 45 { - return fmt.Sprintf("%d seconds ago", diffInSeconds) - } else if diffInSeconds <= 90 { - return "a minute ago" - } - - diffInMinutes := d.InMinutes() - - if diffInMinutes <= 45 { - return fmt.Sprintf("%d minutes ago", diffInMinutes) - } else if diffInMinutes <= 90 { - return "an hour ago" - } - - diffInHours := d.InHours() - - if diffInHours <= 22 { - return fmt.Sprintf("%d hours ago", diffInHours) - } else if diffInHours <= 36 { - return "a day ago" - } - - return "diff is in days" -} - -// In Months - -// In years diff --git a/vendor/github.com/leibowitz/moment/moment.go b/vendor/github.com/leibowitz/moment/moment.go deleted file mode 100644 index 13c8ef7dbef..00000000000 --- a/vendor/github.com/leibowitz/moment/moment.go +++ /dev/null @@ -1,1185 +0,0 @@ -package moment - -import ( - "fmt" - "regexp" - "strconv" - "strings" - "time" -) - -// links -// http://en.wikipedia.org/wiki/ISO_week_date -// http://golang.org/src/pkg/time/format.go -// http://www.php.net/manual/en/class.datetime.php#datetime.constants.rfc822 -// http://php.net/manual/en/function.date.php -// http://www.php.net/manual/en/datetime.formats.relative.php - -// @todo are these constants needed if they are in the time package? -// There are a lot of extras here, and RFC822 doesn't match up. Why? -// Also, is timezone usage wrong? Double-check -const ( - ATOM = "2006-01-02T15:04:05Z07:00" - COOKIE = "Monday, 02-Jan-06 15:04:05 MST" - ISO8601 = "2006-01-02T15:04:05Z0700" - RFC822 = "Mon, 02 Jan 06 15:04:05 Z0700" - RFC850 = "Monday, 02-Jan-06 15:04:05 MST" - RFC1036 = "Mon, 02 Jan 06 15:04:05 Z0700" - RFC1123 = "Mon, 02 Jan 2006 15:04:05 Z0700" - RFC2822 = "Mon, 02 Jan 2006 15:04:05 Z0700" - RFC3339 = "2006-01-02T15:04:05Z07:00" - RSS = "Mon, 02 Jan 2006 15:04:05 Z0700" - W3C = "2006-01-02T15:04:05Z07:00" -) - -var ( - regex_days = "monday|mon|tuesday|tues|wednesday|wed|thursday|thurs|friday|fri|saturday|sat|sunday|sun" - regex_period = "second|minute|hour|day|week|month|year" - regex_numbers = "one|two|three|four|five|six|seven|eight|nine|ten" -) - -// regexp -var ( - compiled = regexp.MustCompile(`\s{2,}`) - relativeday = regexp.MustCompile(`(yesterday|today|tomorrow)`) - //relative1 = regexp.MustCompile(`(first|last) day of (this|next|last|previous) (week|month|year)`) - //relative2 = regexp.MustCompile(`(first|last) day of (` + "jan|january|feb|february|mar|march|apr|april|may|jun|june|jul|july|aug|august|sep|september|oct|october|nov|november|dec|december" + `)(?:\s(\d{4,4}))?`) - relative3 = regexp.MustCompile(`((?Pthis|next|last|previous) )?(` + regex_days + `)`) - //relativeval = regexp.MustCompile(`([0-9]+) (day|week|month|year)s? ago`) - ago = regexp.MustCompile(`([0-9]+) (` + regex_period + `)s? ago`) - ordinal = regexp.MustCompile("([0-9]+)(st|nd|rd|th)") - written = regexp.MustCompile(regex_numbers) - relativediff = regexp.MustCompile(`([\+\-])?([0-9]+),? ?(` + regex_period + `)s?`) - relativetime = regexp.MustCompile(`(?P\d\d?):(?P\d\d?)(:(?P\d\d?))?\s?(?Pam|pm)?\s?(?P[a-z]{3,3})?|(?Pnoon|midnight)`) - yearmonthday = regexp.MustCompile(`(?P\d{4})-(?P\d{1,2})-(?P\d{1,2})`) - relativeperiod = regexp.MustCompile(`(?Pthis|next|last) (week|month|year)`) - numberRegex = regexp.MustCompile("([0-9]+)(?:)") -) - -// http://golang.org/src/pkg/time/format.go?s=12686:12728#L404 - -// Timezone implementation -// https://groups.google.com/forum/#!topic/golang-nuts/XEVN4QwTvHw -// http://en.wikipedia.org/wiki/Zone.tab - -// Support ISO8601 Duration Parsing? -// http://en.wikipedia.org/wiki/ISO_8601 - -// Differences -// Months are NOT zero-index, MOmentJS they are -// Weeks are 0 indexed -// -- Sunday being the last day of the week ISO-8601 - is that diff from Moment? -// From/FromNow Return a Diff object rather than strings - -// Support for locale and languages with English as default - -// Support for strftime -// https://github.com/benjaminoakes/moment-strftime -// Format: https://php.net/strftime - -type Moment struct { - time time.Time - - Parser -} - -type Parser interface { - Convert(string) string -} - -func New() *Moment { - m := &Moment{time.Now(), new(MomentParser)} - - return m -} - -func NewMoment(t time.Time) *Moment { - m := &Moment{t, new(MomentParser)} - - return m -} - -func (m *Moment) GetTime() time.Time { - return m.time -} - -func (m *Moment) Now() *Moment { - m.time = time.Now().In(m.GetTime().Location()) - - return m -} - -func (m *Moment) Moment(layout string, datetime string) *Moment { - return m.MomentGo(m.Convert(layout), datetime) -} - -func (m *Moment) MomentGo(layout string, datetime string) *Moment { - time, _ := time.Parse(layout, datetime) - - m.time = time - - return m -} - -// This method is nowhere near done - requires lots of work. -func (m *Moment) Strtotime(str string) *Moment { - str = strings.ToLower(strings.TrimSpace(str)) - str = compiled.ReplaceAllString(str, " ") - - // Replace written numbers (i.e. nine, ten) with actual numbers (9, 10) - str = written.ReplaceAllStringFunc(str, func(n string) string { - switch n { - case "one": - return "1" - case "two": - return "2" - case "three": - return "3" - case "four": - return "4" - case "five": - return "5" - case "six": - return "6" - case "seven": - return "7" - case "eight": - return "8" - case "nine": - return "9" - case "ten": - return "10" - } - - return "" - }) - - // Remove ordinal suffixes st, nd, rd, th - str = ordinal.ReplaceAllString(str, "$1") - - // Replace n second|minute|hour... ago to -n second|minute|hour... to consolidate parsing - str = ago.ReplaceAllString(str, "-$1 $2") - - // Look for relative +1day, +3 days 5 hours 15 minutes - if match := relativediff.FindAllStringSubmatch(str, -1); match != nil { - for i := range match { - switch match[i][1] { - case "-": - number, _ := strconv.Atoi(match[i][2]) - m.Subtract(match[i][3], number) - default: - number, _ := strconv.Atoi(match[i][2]) - m.Add(match[i][3], number) - } - - str = strings.Replace(str, match[i][0], "", 1) - } - } - - // Remove any words that aren't needed for consistency - str = strings.Replace(str, " at ", " ", -1) - str = strings.Replace(str, " on ", " ", -1) - - // Support for interchangeable previous/last - str = strings.Replace(str, "previous", "last", -1) - - var dateDefaults = map[string]int{ - "year": 0, - "month": 0, - "day": 0, - } - - dateMatches := dateDefaults - if match := yearmonthday.FindStringSubmatch(str); match != nil { - for i, name := range yearmonthday.SubexpNames() { - if i == 0 { - str = strings.Replace(str, match[i], "", 1) - continue - } - - if match[i] == "" { - continue - } - - if name == "year" || name == "month" || name == "day" { - dateMatches[name], _ = strconv.Atoi(match[i]) - } - - } - - defer m.strtotimeSetDate(dateMatches) - if str == "" { - // Nothing left to parse - return m - } - - str = strings.TrimSpace(str) - } - - // Try to parse out time from the string - var timeDefaults = map[string]int{ - "hour": 0, - "minutes": 0, - "seconds": 0, - } - - timeMatches := timeDefaults - var zone string - if match := relativetime.FindStringSubmatch(str); match != nil { - for i, name := range relativetime.SubexpNames() { - if i == 0 { - str = strings.Replace(str, match[i], "", 1) - continue - } - - if match[i] == "" { - continue - } - - // Midnight is all zero's so nothing to do - if name == "relativetime" && match[i] == "noon" { - timeDefaults["hour"] = 12 - } - - if name == "zone" { - zone = match[i] - } - - if name == "meridiem" && match[i] == "pm" && timeMatches["hour"] < 12 { - timeMatches["hour"] += 12 - } - - if name == "hour" || name == "minutes" || name == "seconds" { - timeMatches[name], _ = strconv.Atoi(match[i]) - } - } - - // Processing time is always last - defer m.strtotimeSetTime(timeMatches, zone) - - if str == "" { - // Nothing left to parse - return m - } - - str = strings.TrimSpace(str) - } - - // m.StartOf("month", "January").GoTo(time.Sunday) - - if match := relativeperiod.FindStringSubmatch(str); match != nil { - period := match[1] - unit := match[2] - - str = strings.Replace(str, match[0], "", 1) - - switch period { - case "next": - if unit == "year" { - m.AddYears(1) - } - if unit == "month" { - m.AddMonths(1) - } - if unit == "week" { - m.AddWeeks(1) - } - case "last": - if unit == "year" { - m.SubYears(1) - } - if unit == "month" { - m.SubMonths(1) - } - if unit == "week" { - m.SubWeeks(1) - } - } - - str = strings.TrimSpace(str) - - // first := regexp.MustCompile("(?Pfirst|last)?") - } - - /* - - relativeday: first day of - relativeperiod: this, last, next - relativeperiodunit week, month, year - day: monday, tues, wednesday - month: january, feb - - - YYYY-MM-DD (HH:MM:SS MST)? - MM-DD-YYYY (HH:MM:SS MST) - 10 September 2015 (HH:MM:SS MST)? - September, 10 2015 (HH:MM:SS MST)? - September 10 2015 (HH:MM:SS M - - this year 2014 - next year 2015 - last year 2013 - - this month April - next month May - last month Mar - - first day of April - last day of April - - - DONE 3PM - DONE 3:00 PM - DONE 3:00:05 MST - 3PM on January 5th - January 5th at 3:00PM - first saturday _of_ next month - first saturday _of_ next month _at_ 3:00PM - saturday of next week - saturday of last week - saturday next week - monday next week - saturday of this week - saturday at 3:00pm - saturday at 4:00PM - saturday at midn - first of january - last of january - january of next year - first day of january - last day of january - first day of February - - DONE midnight - DONE noon - DONE 3 days ago - DONE ten days - DONE 9 weeks ago // Convert to -9 weeks - DONE -9 weeks - - */ - - if match := relativeday.FindStringSubmatch(str); match != nil && len(match) > 1 { - day := match[1] - - str = strings.Replace(str, match[0], "", 1) - - switch day { - case "today": - m.Today() - case "yesterday": - m.Yesterday() - case "tomorrow": - m.Tomorrow() - } - } - - if match := relative3.FindStringSubmatch(str); match != nil { - var when string - for i, name := range relative3.SubexpNames() { - if name == "relperiod" { - when = match[i] - } - } - weekDay := match[len(match)-1] - - str = strings.Replace(str, match[0], "", 1) - - wDay, err := ParseWeekDay(weekDay) - if err == nil { - switch when { - case "last", "previous": - m.GoBackTo(wDay, true) - - case "next": - m.GoTo(wDay, true) - - case "", "this": - m.GoTo(wDay, false) - default: - m.GoTo(wDay, false) - } - } - } - - /* - - - yesterday 11:00 - today 11:00 - tomorrow 11:00 - midnight - noon - DONE +n (second|day|week|month|year)s? - DONE -n (second|day|week|month|year)s? - next (monday|tuesday|wednesday|thursday|friday|saturday|sunday) 11:00 - last (monday|tuesday|wednesday|thursday|friday|saturday|sunday) 11:00 - next (month|year) - last (month|year) - first day of (january|february|march...|december) 2014 - last day of (january|february|march...|december) 2014 - first day of (this|next|last) (week|month|year) - last day of (this|next|last) (week|month|year) - first (monday|tuesday|wednesday) of July 2014 - last (monday|tuesday|wednesday) of July 2014 - n (day|week|month|year)s? ago - Monday|Tuesday|Wednesday|Thursday|Friday - Monday (last|this|next) week - - DONE +1 week 2 days 3 hours 4 minutes 5 seconds - */ - - return m -} - -// @todo deal with timezone -func (m *Moment) strtotimeSetTime(time map[string]int, zone string) { - m.SetHour(time["hour"]).SetMinute(time["minutes"]).SetSecond(time["seconds"]) -} - -func (m *Moment) strtotimeSetDate(date map[string]int) { - m.SetYear(date["year"]).SetMonth(time.Month(date["month"])).SetDay(date["day"]) -} - -func (m Moment) Clone() *Moment { - copy := New() - copy.time = m.GetTime() - - return copy -} - -/** - * Getters - * - */ -// https://groups.google.com/forum/#!topic/golang-nuts/pret7hjDc70 -func (m *Moment) Millisecond() { - -} - -func (m *Moment) Second() int { - return m.GetTime().Second() -} - -func (m *Moment) Minute() int { - return m.GetTime().Minute() -} - -func (m *Moment) Hour() int { - return m.GetTime().Hour() -} - -// Day of month -func (m *Moment) Date() int { - return m.DayOfMonth() -} - -// Carbon convenience method -func (m *Moment) DayOfMonth() int { - return m.GetTime().Day() -} - -// Day of week (int or string) -func (m *Moment) Day() time.Weekday { - return m.DayOfWeek() -} - -// Carbon convenience method -func (m *Moment) DayOfWeek() time.Weekday { - return m.GetTime().Weekday() -} - -func (m *Moment) DayOfWeekISO() int { - day := m.GetTime().Weekday() - - if day == time.Sunday { - return 7 - } - - return int(day) -} - -func (m *Moment) DayOfYear() int { - return m.GetTime().YearDay() -} - -// Day of Year with zero padding -func (m *Moment) dayOfYearZero() string { - day := m.GetTime().YearDay() - - if day < 10 { - return fmt.Sprintf("00%d", day) - } - - if day < 100 { - return fmt.Sprintf("0%d", day) - } - - return fmt.Sprintf("%d", day) -} - -// todo panic? -func (m *Moment) Weekday(index int) string { - if index > 6 { - panic("Weekday index must be between 0 and 6") - } - - return time.Weekday(index).String() -} - -func (m *Moment) Week() int { - return 0 -} - -// Is this the week number where as ISOWeekYear is the number of weeks in the year? -// @see http://stackoverflow.com/questions/18478741/get-weeks-in-year -func (m *Moment) ISOWeek() int { - _, week := m.GetTime().ISOWeek() - - return week -} - -// @todo Consider language support -func (m *Moment) Month() time.Month { - return m.GetTime().Month() -} - -func (m *Moment) Quarter() (quarter int) { - quarter = 4 - - switch m.Month() { - case time.January, time.February, time.March: - quarter = 1 - case time.April, time.May, time.June: - quarter = 2 - case time.July, time.August, time.September: - quarter = 3 - } - - return -} - -func (m *Moment) Year() int { - return m.GetTime().Year() -} - -// @see comments for ISOWeek -func (m *Moment) WeekYear() { - -} - -func (m *Moment) ISOWeekYear() { - -} - -/** - * Manipulate - * - */ -func (m *Moment) Add(key string, value int) *Moment { - switch key { - case "years", "year", "y": - m.AddYears(value) - case "months", "month", "M": - m.AddMonths(value) - case "weeks", "week", "w": - m.AddWeeks(value) - case "days", "day", "d": - m.AddDays(value) - case "hours", "hour", "h": - m.AddHours(value) - case "minutes", "minute", "m": - m.AddMinutes(value) - case "seconds", "second", "s": - m.AddSeconds(value) - case "milliseconds", "millisecond", "ms": - - } - - return m -} - -// Carbon -func (m *Moment) AddSeconds(seconds int) *Moment { - return m.addTime(time.Second * time.Duration(seconds)) -} - -// Carbon -func (m *Moment) AddMinutes(minutes int) *Moment { - return m.addTime(time.Minute * time.Duration(minutes)) -} - -// Carbon -func (m *Moment) AddHours(hours int) *Moment { - return m.addTime(time.Hour * time.Duration(hours)) -} - -// Carbon -func (m *Moment) AddDay() *Moment { - return m.AddDays(1) -} - -// Carbon -func (m *Moment) AddDays(days int) *Moment { - m.time = m.GetTime().AddDate(0, 0, days) - - return m -} - -// Carbon -func (m *Moment) AddWeeks(weeks int) *Moment { - return m.AddDays(weeks * 7) -} - -// Carbon -func (m *Moment) AddMonths(months int) *Moment { - m.time = m.GetTime().AddDate(0, months, 0) - - return m -} - -// Carbon -func (m *Moment) AddYears(years int) *Moment { - m.time = m.GetTime().AddDate(years, 0, 0) - - return m -} - -func (m *Moment) addTime(d time.Duration) *Moment { - m.time = m.GetTime().Add(d) - - return m -} - -func (m *Moment) Subtract(key string, value int) *Moment { - switch key { - case "years", "year", "y": - m.SubYears(value) - case "months", "month", "M": - m.SubMonths(value) - case "weeks", "week", "w": - m.SubWeeks(value) - case "days", "day", "d": - m.SubDays(value) - case "hours", "hour", "h": - m.SubHours(value) - case "minutes", "minute", "m": - m.SubMinutes(value) - case "seconds", "second", "s": - m.SubSeconds(value) - case "milliseconds", "millisecond", "ms": - - } - - return m -} - -// Carbon -func (m *Moment) SubSeconds(seconds int) *Moment { - return m.addTime(time.Second * time.Duration(seconds*-1)) -} - -// Carbon -func (m *Moment) SubMinutes(minutes int) *Moment { - return m.addTime(time.Minute * time.Duration(minutes*-1)) -} - -// Carbon -func (m *Moment) SubHours(hours int) *Moment { - return m.addTime(time.Hour * time.Duration(hours*-1)) -} - -// Carbon -func (m *Moment) SubDay() *Moment { - return m.SubDays(1) -} - -// Carbon -func (m *Moment) SubDays(days int) *Moment { - return m.AddDays(days * -1) -} - -func (m *Moment) SubWeeks(weeks int) *Moment { - return m.SubDays(weeks * 7) -} - -// Carbon -func (m *Moment) SubMonths(months int) *Moment { - return m.AddMonths(months * -1) -} - -// Carbon -func (m *Moment) SubYears(years int) *Moment { - return m.AddYears(years * -1) -} - -// Carbon -func (m *Moment) Today() *Moment { - return m.Now() -} - -// Carbon -func (m *Moment) Tomorrow() *Moment { - return m.Today().AddDay() -} - -// Carbon -func (m *Moment) Yesterday() *Moment { - return m.Today().SubDay() -} - -func (m *Moment) StartOf(key string) *Moment { - switch key { - case "year", "y": - m.StartOfYear() - case "month", "M": - m.StartOfMonth() - case "week", "w": - m.StartOfWeek() - case "day", "d": - m.StartOfDay() - case "hour", "h": - if m.Minute() > 0 { - m.SubMinutes(m.Minute()) - } - - if m.Second() > 0 { - m.SubSeconds(m.Second()) - } - case "minute", "m": - if m.Second() > 0 { - m.SubSeconds(m.Second()) - } - case "second", "s": - - } - - return m -} - -// Carbon -func (m *Moment) StartOfDay() *Moment { - if m.Hour() > 0 { - _, timeOffset := m.GetTime().Zone() - m.SubHours(m.Hour()) - - _, newTimeOffset := m.GetTime().Zone() - diffOffset := timeOffset - newTimeOffset - if diffOffset != 0 { - // we need to adjust for time zone difference - m.AddSeconds(diffOffset) - } - } - - return m.StartOf("hour") -} - -// @todo ISO8601 Starts on Monday -func (m *Moment) StartOfWeek() *Moment { - return m.GoBackTo(time.Monday, false).StartOfDay() -} - -// Carbon -func (m *Moment) StartOfMonth() *Moment { - return m.SetDay(1).StartOfDay() -} - -// Carbon -func (m *Moment) StartOfYear() *Moment { - return m.SetMonth(time.January).SetDay(1).StartOfDay() -} - -// Carbon -func (m *Moment) EndOf(key string) *Moment { - switch key { - case "year", "y": - m.EndOfYear() - case "month", "M": - m.EndOfMonth() - case "week", "w": - m.EndOfWeek() - case "day", "d": - m.EndOfDay() - case "hour", "h": - if m.Minute() < 59 { - m.AddMinutes(59 - m.Minute()) - } - case "minute", "m": - if m.Second() < 59 { - m.AddSeconds(59 - m.Second()) - } - case "second", "s": - - } - - return m -} - -// Carbon -func (m *Moment) EndOfDay() *Moment { - if m.Hour() < 23 { - _, timeOffset := m.GetTime().Zone() - m.AddHours(23 - m.Hour()) - - _, newTimeOffset := m.GetTime().Zone() - diffOffset := newTimeOffset - timeOffset - if diffOffset != 0 { - // we need to adjust for time zone difference - m.SubSeconds(diffOffset) - } - } - - return m.EndOf("hour") -} - -// @todo ISO8601 Ends on Sunday -func (m *Moment) EndOfWeek() *Moment { - return m.GoTo(time.Sunday, false).EndOfDay() -} - -// Carbon -func (m *Moment) EndOfMonth() *Moment { - return m.SetDay(m.DaysInMonth()).EndOfDay() -} - -// Carbon -func (m *Moment) EndOfYear() *Moment { - return m.GoToMonth(time.December, false).EndOfMonth() -} - -// Custom -func (m *Moment) GoTo(day time.Weekday, next bool) *Moment { - if m.Day() == day { - if !next { - return m - } else { - m.AddDay() - } - } - - var diff int - if diff = int(day) - int(m.Day()); diff > 0 { - return m.AddDays(diff) - } - - return m.AddDays(7 + diff) -} - -// Custom -func (m *Moment) GoBackTo(day time.Weekday, previous bool) *Moment { - if m.Day() == day { - if !previous { - return m - } else { - m.SubDay() - } - } - - var diff int - if diff = int(day) - int(m.Day()); diff > 0 { - return m.SubDays(7 - diff) - } - - return m.SubDays(diff * -1) -} - -// Custom -func (m *Moment) GoToMonth(month time.Month, next bool) *Moment { - if m.Month() == month { - if !next { - return m - } else { - m.AddMonths(1) - } - } - - var diff int - if diff = int(month - m.Month()); diff > 0 { - return m.AddMonths(diff) - } - - return m.AddMonths(12 + diff) -} - -// Custom -func (m *Moment) GoBackToMonth(month time.Month, previous bool) *Moment { - if m.Month() == month { - if !previous { - return m - } else { - m.SubMonths(1) - } - } - - var diff int - if diff = int(month) - int(m.Month()); diff > 0 { - return m.SubMonths(12 - diff) - } - - return m.SubMonths(diff * -1) -} - -func (m *Moment) SetSecond(seconds int) *Moment { - if seconds >= 0 && seconds <= 60 { - return m.AddSeconds(seconds - m.Second()) - } - - return m -} - -func (m *Moment) SetMinute(minute int) *Moment { - if minute >= 0 && minute <= 60 { - return m.AddMinutes(minute - m.Minute()) - } - - return m -} - -func (m *Moment) SetHour(hour int) *Moment { - if hour >= 0 && hour <= 23 { - return m.AddHours(hour - m.Hour()) - } - - return m -} - -// Custom -func (m *Moment) SetDay(day int) *Moment { - if m.DayOfMonth() == day { - return m - } - - return m.AddDays(day - m.DayOfMonth()) -} - -// Custom -func (m *Moment) SetMonth(month time.Month) *Moment { - if m.Month() > month { - return m.GoBackToMonth(month, false) - } - - return m.GoToMonth(month, false) -} - -// Custom -func (m *Moment) SetYear(year int) *Moment { - if m.Year() == year { - return m - } - - return m.AddYears(year - m.Year()) -} - -// UTC Mode. @see http://momentjs.com/docs/#/parsing/utc/ -func (m *Moment) UTC() *Moment { - return m -} - -// http://momentjs.com/docs/#/manipulating/timezone-offset/ -func (m *Moment) Zone() int { - _, offset := m.GetTime().Zone() - - return (offset / 60) * -1 -} - -/** - * Display - * - */ -func (m *Moment) Format(layout string) string { - format := m.Convert(layout) - hasCustom := false - - formatted := m.GetTime().Format(format) - - if strings.Contains(formatted, "", fmt.Sprintf("%d", m.Unix()), -1) - formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", m.ISOWeek()), -1) - formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", m.DayOfWeek()), -1) - formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", m.DayOfWeekISO()), -1) - formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", m.DayOfYear()), -1) - formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", m.Quarter()), -1) - formatted = strings.Replace(formatted, "", m.dayOfYearZero(), -1) - formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", m.Hour()), -1) - } - - // This has to happen after time.Format - if hasCustom && strings.Contains(formatted, "") { - formatted = numberRegex.ReplaceAllStringFunc(formatted, func(n string) string { - ordinal, _ := strconv.Atoi(strings.Replace(n, "", "", 1)) - return m.ordinal(ordinal) - }) - } - - return formatted -} - -func (m *Moment) FormatGo(layout string) string { - return m.GetTime().Format(layout) -} - -// From Dmytro Shteflyuk @https://groups.google.com/forum/#!topic/golang-nuts/l8NhI74jl-4 -func (m *Moment) ordinal(x int) string { - suffix := "th" - switch x % 10 { - case 1: - if x%100 != 11 { - suffix = "st" - } - case 2: - if x%100 != 12 { - suffix = "nd" - } - case 3: - if x%100 != 13 { - suffix = "rd" - } - } - - return strconv.Itoa(x) + suffix -} - -func (m *Moment) FromNow() Diff { - now := new(Moment) - now.Now() - - return m.From(now) -} - -// Carbon -func (m *Moment) From(f *Moment) Diff { - return m.GetDiff(f) -} - -/** - * Difference - * - */ -func (m *Moment) Diff(t *Moment, unit string) int { - diff := m.GetDiff(t) - - switch unit { - case "years": - return diff.InYears() - case "months": - return diff.InMonths() - case "weeks": - return diff.InWeeks() - case "days": - return diff.InDays() - case "hours": - return diff.InHours() - case "minutes": - return diff.InMinutes() - case "seconds": - return diff.InSeconds() - } - - return 0 -} - -// Custom -func (m *Moment) GetDiff(t *Moment) Diff { - duration := m.GetTime().Sub(t.GetTime()) - - return Diff{duration} -} - -/** - * Display - * - */ -func (m *Moment) ValueOf() int64 { - return m.Unix() * 1000 -} - -func (m *Moment) Unix() int64 { - return m.GetTime().Unix() -} - -func (m *Moment) DaysInMonth() int { - days := 31 - switch m.Month() { - case time.April, time.June, time.September, time.November: - days = 30 - break - case time.February: - days = 28 - if m.IsLeapYear() { - days = 29 - } - break - } - - return days -} - -// or ToSlice? -func (m *Moment) ToArray() []int { - return []int{ - m.Year(), - int(m.Month()), - m.DayOfMonth(), - m.Hour(), - m.Minute(), - m.Second(), - } -} - -/** - * Query - * - */ -func (m *Moment) IsBefore(t Moment) bool { - return m.GetTime().Before(t.GetTime()) -} - -func (m *Moment) IsSame(t *Moment, layout string) bool { - return m.Format(layout) == t.Format(layout) -} - -func (m *Moment) IsAfter(t Moment) bool { - return m.GetTime().After(t.GetTime()) -} - -// Carbon -func (m *Moment) IsToday() bool { - today := m.Clone().Today() - - return m.Year() == today.Year() && m.Month() == today.Month() && m.Day() == today.Day() -} - -// Carbon -func (m *Moment) IsTomorrow() bool { - tomorrow := m.Clone().Tomorrow() - - return m.Year() == tomorrow.Year() && m.Month() == tomorrow.Month() && m.Day() == tomorrow.Day() -} - -// Carbon -func (m *Moment) IsYesterday() bool { - yesterday := m.Clone().Yesterday() - - return m.Year() == yesterday.Year() && m.Month() == yesterday.Month() && m.Day() == yesterday.Day() -} - -// Carbon -func (m *Moment) IsWeekday() bool { - return !m.IsWeekend() -} - -// Carbon -func (m *Moment) IsWeekend() bool { - return m.DayOfWeek() == time.Sunday || m.DayOfWeek() == time.Saturday -} - -func (m *Moment) IsLeapYear() bool { - year := m.Year() - return year%4 == 0 && (year%100 != 0 || year%400 == 0) -} - -// Custom -func (m *Moment) Range(start Moment, end Moment) bool { - return m.IsAfter(start) && m.IsBefore(end) -} diff --git a/vendor/github.com/leibowitz/moment/moment_parser.go b/vendor/github.com/leibowitz/moment/moment_parser.go deleted file mode 100644 index 3361cfba113..00000000000 --- a/vendor/github.com/leibowitz/moment/moment_parser.go +++ /dev/null @@ -1,100 +0,0 @@ -package moment - -import ( - "regexp" - "strings" -) - -type MomentParser struct{} - -var ( - date_pattern = regexp.MustCompile("(LT|LL?L?L?|l{1,4}|Mo|MM?M?M?|Do|DDDo|DD?D?D?|ddd?d?|do?|w[o|w]?|W[o|W]?|YYYYY|YYYY|YY|gg(ggg?)?|GG(GGG?)?|e|E|a|A|hh?|HH?|mm?|ss?|SS?S?|X|zz?|ZZ?|Q)") -) - -/* - + S (makes any number before it ordinal) - + stdDayOfYear 1,2,365 - + stdDayOfYearZero 001, 002, 365 - + stdDayOfWeek w 0, 1, 2 numeric day of the week (0 = sunday) - + stdDayOfWeekISO N 1 = Monday - + stdWeekOfYear W Iso week number of year - + stdUnix U - + stdQuarter -*/ - -// Thanks to https://github.com/fightbulc/moment.php for replacement keys and regex -var moment_replacements = map[string]string{ - "M": "1", // stdNumMonth 1 2 ... 11 12 - "Mo": "1", // stdNumMonth 1st 2nd ... 11th 12th - "MM": "01", // stdZeroMonth 01 02 ... 11 12 - "MMM": "Jan", // stdMonth Jan Feb ... Nov Dec - "MMMM": "January", // stdLongMonth January February ... November December - "D": "2", // stdDay 1 2 ... 30 30 - "Do": "2", // stdDay 1st 2nd ... 30th 31st @todo support st nd th etch - "DD": "02", // stdZeroDay 01 02 ... 30 31 - "DDD": "", // Day of the year 1 2 ... 364 365 - "DDDo": "", // Day of the year 1st 2nd ... 364th 365th - "DDDD": "", // Day of the year 001 002 ... 364 365 @todo**** - "d": "", // Numeric representation of day of the week 0 1 ... 5 6 - "do": "", // 0th 1st ... 5th 6th - "dd": "Mon", // ***Su Mo ... Fr Sa @todo - "ddd": "Mon", // Sun Mon ... Fri Sat - "dddd": "Monday", // stdLongWeekDay Sunday Monday ... Friday Saturday - "e": "", // Numeric representation of day of the week 0 1 ... 5 6 @todo - "E": "", // ISO-8601 numeric representation of the day of the week (added in PHP 5.1.0) 1 2 ... 6 7 @todo - "w": "", // 1 2 ... 52 53 - "wo": "", // 1st 2nd ... 52nd 53rd - "ww": "", // ***01 02 ... 52 53 @todo - "W": "", // 1 2 ... 52 53 - "Wo": "", // 1st 2nd ... 52nd 53rd - "WW": "", // ***01 02 ... 52 53 @todo - "YY": "06", // stdYear 70 71 ... 29 30 - "YYYY": "2006", // stdLongYear 1970 1971 ... 2029 2030 - // "gg" : "o", // ISO-8601 year number 70 71 ... 29 30 @todo - // "gggg" : "o", // ***1970 1971 ... 2029 2030 @todo - // "GG" : "o", //70 71 ... 29 30 @todo - // "GGGG" : "o", // ***1970 1971 ... 2029 2030 @todo - "Q": "", - "A": "PM", // stdPM AM PM - "a": "pm", // stdpm am pm - "H": "", // stdHour 0 1 ... 22 23 - "HH": "15", // 00 01 ... 22 23 - "h": "3", // stdHour12 1 2 ... 11 12 - "hh": "03", // stdZeroHour12 01 02 ... 11 12 - "m": "4", // stdZeroMinute 0 1 ... 58 59 - "mm": "04", // stdZeroMinute 00 01 ... 58 59 - "s": "5", // stdSecond 0 1 ... 58 59 - "ss": "05", // stdZeroSecond ***00 01 ... 58 59 - // "S" : "", //0 1 ... 8 9 - // "SS" : "", //0 1 ... 98 99 - // "SSS" : "", //0 1 ... 998 999 - "z": "MST", //EST CST ... MST PST - "zz": "MST", //EST CST ... MST PST - "Z": "Z07:00", // stdNumColonTZ -07:00 -06:00 ... +06:00 +07:00 - "ZZ": "-0700", // stdNumTZ -0700 -0600 ... +0600 +0700 - "X": "", // Seconds since unix epoch 1360013296 - "LT": "3:04 PM", // 8:30 PM - "L": "01/02/2006", //09/04/1986 - "l": "1/2/2006", //9/4/1986 - "LL": "January 2 2006", //September 4th 1986 the php s flag isn't supported - "ll": "Jan 2 2006", //Sep 4 1986 - "LLL": "January 2 2006 3:04 PM", //September 4th 1986 8:30 PM @todo the php s flag isn't supported - "lll": "Jan 2 2006 3:04 PM", //Sep 4 1986 8:30 PM - "LLLL": "Monday, January 2 2006 3:04 PM", //Thursday, September 4th 1986 8:30 PM the php s flag isn't supported - "llll": "Mon, Jan 2 2006 3:04 PM", //Thu, Sep 4 1986 8:30 PM -} - -func (p *MomentParser) Convert(layout string) string { - var match [][]string - if match = date_pattern.FindAllStringSubmatch(layout, -1); match == nil { - return layout - } - - for i := range match { - if replace, ok := moment_replacements[match[i][0]]; ok { - layout = strings.Replace(layout, match[i][0], replace, 1) - } - } - - return layout -} diff --git a/vendor/github.com/leibowitz/moment/parse_day.go b/vendor/github.com/leibowitz/moment/parse_day.go deleted file mode 100644 index e8e890a462e..00000000000 --- a/vendor/github.com/leibowitz/moment/parse_day.go +++ /dev/null @@ -1,32 +0,0 @@ -package moment - -import ( - "fmt" - "strings" - "time" -) - -var ( - days = []time.Weekday{ - time.Sunday, - time.Monday, - time.Tuesday, - time.Wednesday, - time.Thursday, - time.Friday, - time.Saturday, - } -) - -func ParseWeekDay(day string) (time.Weekday, error) { - - day = strings.ToLower(day) - - for _, d := range days { - if day == strings.ToLower(d.String()) { - return d, nil - } - } - - return -1, fmt.Errorf("Unable to parse %s as week day", day) -} diff --git a/vendor/github.com/leibowitz/moment/strftime_parser.go b/vendor/github.com/leibowitz/moment/strftime_parser.go deleted file mode 100644 index 3c024376535..00000000000 --- a/vendor/github.com/leibowitz/moment/strftime_parser.go +++ /dev/null @@ -1,68 +0,0 @@ -package moment - -import ( - "regexp" - "strings" -) - -type StrftimeParser struct{} - -var ( - replacements_pattern = regexp.MustCompile("%[mbhBedjwuaAVgyGYpPkHlIMSZzsTrRTDFXx]") -) - -// Not implemented -// U -// C - -var strftime_replacements = map[string]string{ - "%m": "01", // stdZeroMonth 01 02 ... 11 12 - "%b": "Jan", // stdMonth Jan Feb ... Nov Dec - "%h": "Jan", - "%B": "January", // stdLongMonth January February ... November December - "%e": "2", // stdDay 1 2 ... 30 30 - "%d": "02", // stdZeroDay 01 02 ... 30 31 - "%j": "", // Day of the year ***001 002 ... 364 365 @todo**** - "%w": "", // Numeric representation of day of the week 0 1 ... 5 6 - "%u": "", // ISO-8601 numeric representation of the day of the week (added in PHP 5.1.0) 1 2 ... 6 7 @todo - "%a": "Mon", // Sun Mon ... Fri Sat - "%A": "Monday", // stdLongWeekDay Sunday Monday ... Friday Saturday - "%V": "", // ***01 02 ... 52 53 @todo begin with zeros - "%g": "06", // stdYear 70 71 ... 29 30 - "%y": "06", - "%G": "2006", // stdLongYear 1970 1971 ... 2029 2030 - "%Y": "2006", - "%p": "PM", // stdPM AM PM - "%P": "pm", // stdpm am pm - "%k": "15", // stdHour 0 1 ... 22 23 - "%H": "15", // 00 01 ... 22 23 - "%l": "3", // stdHour12 1 2 ... 11 12 - "%I": "03", // stdZeroHour12 01 02 ... 11 12 - "%M": "04", // stdZeroMinute 00 01 ... 58 59 - "%S": "05", // stdZeroSecond ***00 01 ... 58 59 - "%Z": "MST", //EST CST ... MST PST - "%z": "-0700", // stdNumTZ -0700 -0600 ... +0600 +0700 - "%s": "", // Seconds since unix epoch 1360013296 - "%r": "03:04:05 PM", - "%R": "15:04", - "%T": "15:04:05", - "%D": "01/02/06", - "%F": "2006-01-02", - "%X": "15:04:05", - "%x": "01/02/06", -} - -func (p *StrftimeParser) Convert(layout string) string { - var match [][]string - if match = replacements_pattern.FindAllStringSubmatch(layout, -1); match == nil { - return layout - } - - for i := range match { - if replace, ok := strftime_replacements[match[i][0]]; ok { - layout = strings.Replace(layout, match[i][0], replace, 1) - } - } - - return layout -} From dce59ccff2f13bbefafb645cbffdcf8b07f94f20 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 23 May 2018 15:28:36 +0200 Subject: [PATCH 14/87] fix: remove deadcode to make gometalinter happy --- pkg/tsdb/elasticsearch/client/client.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/tsdb/elasticsearch/client/client.go b/pkg/tsdb/elasticsearch/client/client.go index 3dae343bd37..3762a58317a 100644 --- a/pkg/tsdb/elasticsearch/client/client.go +++ b/pkg/tsdb/elasticsearch/client/client.go @@ -22,8 +22,7 @@ import ( const loggerName = "tsdb.elasticsearch.client" var ( - clientLog = log.New(loggerName) - intervalCalculator = tsdb.NewIntervalCalculator(&tsdb.IntervalOptions{MinInterval: 15 * time.Second}) + clientLog = log.New(loggerName) ) // Client represents a client which can interact with elasticsearch api From 242689abe242d30bfdd4ab9b8a6be6347eb37931 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 23 May 2018 16:46:26 +0200 Subject: [PATCH 15/87] elasticsearch: pipeline aggregation fix for json encoding --- pkg/tsdb/elasticsearch/client/models.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/tsdb/elasticsearch/client/models.go b/pkg/tsdb/elasticsearch/client/models.go index 3c86dcce825..2d9839dfd53 100644 --- a/pkg/tsdb/elasticsearch/client/models.go +++ b/pkg/tsdb/elasticsearch/client/models.go @@ -293,7 +293,7 @@ type PipelineAggregation struct { // MarshalJSON returns the JSON encoding of the pipeline aggregation func (a *PipelineAggregation) MarshalJSON() ([]byte, error) { root := map[string]interface{}{ - "bucket_path": a.BucketPath, + "buckets_path": a.BucketPath, } for k, v := range a.Settings { From 61b296afaddd439976d72a3c3ed4f431e7c43c51 Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Wed, 23 May 2018 17:08:54 +0200 Subject: [PATCH 16/87] Document table row merge for multiple queries * added section to table feature docs * marked as 5.0+ feature * concrete examples of what works and the limits --- docs/sources/features/panels/table_panel.md | 42 ++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/docs/sources/features/panels/table_panel.md b/docs/sources/features/panels/table_panel.md index 32f7764e415..ed2632f29d6 100644 --- a/docs/sources/features/panels/table_panel.md +++ b/docs/sources/features/panels/table_panel.md @@ -14,11 +14,51 @@ weight = 2 -The new table panel is very flexible, supporting both multiple modes for time series as well as for +The table panel is very flexible, supporting both multiple modes for time series as well as for table, annotation and raw JSON data. It also provides date formatting and value formatting and coloring options. To view table panels in action and test different configurations with sample data, check out the [Table Panel Showcase in the Grafana Playground](http://play.grafana.org/dashboard/db/table-panel-showcase). +## Querying Data + +The table panel displays the results of a query specified in the **Metrics** tab. +The result being displayed depends on the datasource and the query, but generally there is one row per datapoint, with extra columns for associated keys and values, as well as one column for the numeric value of the datapoint. +You can change the behavior in the section **Data to Table** below. + +### Multiple Queries per Table + +> Only available in Grafana v5.0+. + +Sometimes it is useful to display the results of multiple queries in the same table on corresponding rows, e.g., when comparing capacity and actual usage of resources. +In this example usage and capacity are metrics that will have corresponding datapoints, while their associated keys and values can be used to match them. +(This matching is only available with the **Table Transform** set to **Table**.) + +In its simplest case, both queries return time-series data with a numeric value and a timestamp. +If the timestamps are the same, datapoints will be matched and rendered on the same row. +Some datasources return keys and values (labels, tags) associated with the datapoint. +These are being matched as well iff they are present in both results and have the same value. +The following datapoints will end up on the same row with one time column, two label columns ("host" and "job") and two value columns: + +``` +Datapoint for query A: {time: 1, host: "node-2", job: "job-8", value: 3} +Datapoint for query B: {time: 1, host: "node-2", value: 4} +``` + +The following two results cannot be matched and will be rendered on separate rows: + +``` +Different time +Datapoint for query A: {time: 1, host: "node-2", job: "job-8", value: 3} +Datapoint for query B: {time: 2, host: "node-2", value: 4} + +Different label "host" +Datapoint for query A: {time: 1, host: "node-2", job: "job-8", value: 3} +Datapoint for query B: {time: 1, host: "node-9", value: 4} +``` + +You can still merge both of the above cases by changing the conflicting column's **Type** to **hidden** in the **Column Styles**. +Note that if each datapoint of your query results have multiple value fields like max, min, mean, etc., they will likely have different values and therefor will not match and render on separate rows. + ## Options overview The table panel has many ways to manipulate your data for optimal presentation. From 8870e3e85b84a5adf371faeb3b833e90385fd006 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 23 May 2018 21:44:09 +0200 Subject: [PATCH 17/87] elasticsearch: default interval fix 5s instead of 15s --- pkg/tsdb/elasticsearch/client/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/tsdb/elasticsearch/client/client.go b/pkg/tsdb/elasticsearch/client/client.go index 3762a58317a..e3583070d38 100644 --- a/pkg/tsdb/elasticsearch/client/client.go +++ b/pkg/tsdb/elasticsearch/client/client.go @@ -106,7 +106,7 @@ func (c *baseClientImpl) GetTimeField() string { func (c *baseClientImpl) GetMinInterval(queryInterval string) (time.Duration, error) { return tsdb.GetIntervalFrom(c.ds, simplejson.NewFromAny(map[string]string{ "interval": queryInterval, - }), 15*time.Second) + }), 5*time.Second) } func (c *baseClientImpl) getSettings() *simplejson.Json { From 4436b8da12c092df4f0cc583a3b8c9e36de6023d Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 23 May 2018 22:07:52 +0200 Subject: [PATCH 18/87] elasticsearch: query interval override fix --- pkg/tsdb/elasticsearch/client/client.go | 2 +- pkg/tsdb/elasticsearch/time_series_query.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/tsdb/elasticsearch/client/client.go b/pkg/tsdb/elasticsearch/client/client.go index e3583070d38..4fe5c40b127 100644 --- a/pkg/tsdb/elasticsearch/client/client.go +++ b/pkg/tsdb/elasticsearch/client/client.go @@ -104,7 +104,7 @@ func (c *baseClientImpl) GetTimeField() string { } func (c *baseClientImpl) GetMinInterval(queryInterval string) (time.Duration, error) { - return tsdb.GetIntervalFrom(c.ds, simplejson.NewFromAny(map[string]string{ + return tsdb.GetIntervalFrom(c.ds, simplejson.NewFromAny(map[string]interface{}{ "interval": queryInterval, }), 5*time.Second) } diff --git a/pkg/tsdb/elasticsearch/time_series_query.go b/pkg/tsdb/elasticsearch/time_series_query.go index ae4af7704fb..ef59c62c1dc 100644 --- a/pkg/tsdb/elasticsearch/time_series_query.go +++ b/pkg/tsdb/elasticsearch/time_series_query.go @@ -258,7 +258,7 @@ func (p *timeSeriesQueryParser) parse(tsdbQuery *tsdb.TsdbQuery) ([]*Query, erro return nil, err } alias := model.Get("alias").MustString("") - interval := model.Get("interval").MustString() + interval := strconv.FormatInt(q.IntervalMs, 10) + "ms" queries = append(queries, &Query{ TimeField: timeField, From 688f5b830ce728f99551b6032a40abd2958a2144 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 23 May 2018 22:21:41 +0200 Subject: [PATCH 19/87] elasticsearch: metric and pipeline agg setting json encoding fix --- pkg/tsdb/elasticsearch/client/models.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/tsdb/elasticsearch/client/models.go b/pkg/tsdb/elasticsearch/client/models.go index 2d9839dfd53..2f4f5dcd162 100644 --- a/pkg/tsdb/elasticsearch/client/models.go +++ b/pkg/tsdb/elasticsearch/client/models.go @@ -278,7 +278,9 @@ func (a *MetricAggregation) MarshalJSON() ([]byte, error) { } for k, v := range a.Settings { - root[k] = v + if k != "" && v != nil { + root[k] = v + } } return json.Marshal(root) @@ -297,7 +299,9 @@ func (a *PipelineAggregation) MarshalJSON() ([]byte, error) { } for k, v := range a.Settings { - root[k] = v + if k != "" && v != nil { + root[k] = v + } } return json.Marshal(root) From 16c3566a873ed7327604f09f88bff7d6a8d90bcb Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 23 May 2018 22:57:46 +0200 Subject: [PATCH 20/87] elasticsearch: handle NaN values --- pkg/tsdb/elasticsearch/response_parser.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/tsdb/elasticsearch/response_parser.go b/pkg/tsdb/elasticsearch/response_parser.go index 029b2e02142..4a45d6271b9 100644 --- a/pkg/tsdb/elasticsearch/response_parser.go +++ b/pkg/tsdb/elasticsearch/response_parser.go @@ -490,10 +490,14 @@ func castToNullFloat(j *simplejson.Json) null.Float { return null.FloatFrom(f) } - s, err := j.String() - if err == nil { - v, _ := strconv.ParseFloat(s, 64) - return null.FloatFromPtr(&v) + if s, err := j.String(); err == nil { + if strings.ToLower(s) == "nan" { + return null.NewFloat(0, false) + } + + if v, err := strconv.ParseFloat(s, 64); err == nil { + return null.FloatFromPtr(&v) + } } return null.NewFloat(0, false) From 0d3f24ce54782c06ce0ba534786eb0e2e90ac2e6 Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Mon, 30 Apr 2018 17:25:25 +0200 Subject: [PATCH 21/87] Explore: time selector * time selector for explore section * mostly ported the angular time selector, but left out the timepicker (3rd-party angular component) * can be initialised via url parameters (jump from panels to explore) * refreshing not implemented for now * moved the forward/backward nav buttons around the time selector --- public/app/containers/Explore/ElapsedTime.tsx | 2 +- public/app/containers/Explore/Explore.tsx | 73 ++++--- public/app/containers/Explore/Graph.tsx | 13 +- public/app/containers/Explore/TimePicker.tsx | 192 ++++++++++++++++++ public/app/containers/Explore/utils/query.ts | 9 +- public/app/core/services/keybindingSrv.ts | 9 +- .../app/features/panel/metrics_panel_ctrl.ts | 7 +- public/sass/pages/_explore.scss | 14 ++ 8 files changed, 273 insertions(+), 46 deletions(-) create mode 100644 public/app/containers/Explore/TimePicker.tsx diff --git a/public/app/containers/Explore/ElapsedTime.tsx b/public/app/containers/Explore/ElapsedTime.tsx index 9cd8f674186..a2d941515cd 100644 --- a/public/app/containers/Explore/ElapsedTime.tsx +++ b/public/app/containers/Explore/ElapsedTime.tsx @@ -41,6 +41,6 @@ export default class ElapsedTime extends PureComponent { const { elapsed } = this.state; const { className, time } = this.props; const value = (time || elapsed) / 1000; - return {value.toFixed(1)}s; + return {value.toFixed(1)}s; } } diff --git a/public/app/containers/Explore/Explore.tsx b/public/app/containers/Explore/Explore.tsx index 40261ee635a..66500353812 100644 --- a/public/app/containers/Explore/Explore.tsx +++ b/public/app/containers/Explore/Explore.tsx @@ -8,6 +8,7 @@ import Legend from './Legend'; import QueryRows from './QueryRows'; import Graph from './Graph'; import Table from './Table'; +import TimePicker, { DEFAULT_RANGE } from './TimePicker'; import { DatasourceSrv } from 'app/features/plugins/datasource_srv'; import { buildQueryOptions, ensureQueries, generateQueryKey, hasQuery } from './utils/query'; import { decodePathComponent } from 'app/core/utils/location_util'; @@ -15,40 +16,33 @@ import { decodePathComponent } from 'app/core/utils/location_util'; function makeTimeSeriesList(dataList, options) { return dataList.map((seriesData, index) => { const datapoints = seriesData.datapoints || []; - const alias = seriesData.target; - + const responseAlias = seriesData.target; + const query = options.targets[index].expr; + const alias = responseAlias && responseAlias !== '{}' ? responseAlias : query; const colorIndex = index % colors.length; const color = colors[colorIndex]; const series = new TimeSeries({ - datapoints: datapoints, - alias: alias, - color: color, + datapoints, + alias, + color, unit: seriesData.unit, }); - if (datapoints && datapoints.length > 0) { - const last = datapoints[datapoints.length - 1][1]; - const from = options.range.from; - if (last - from < -10000) { - series.isOutsideRange = true; - } - } - return series; }); } -function parseInitialQueries(initial) { - if (!initial) { - return []; - } +function parseInitialState(initial) { try { const parsed = JSON.parse(decodePathComponent(initial)); - return parsed.queries.map(q => q.query); + return { + queries: parsed.queries.map(q => q.query), + range: parsed.range, + }; } catch (e) { console.error(e); - return []; + return { queries: [], range: DEFAULT_RANGE }; } } @@ -60,6 +54,7 @@ interface IExploreState { latency: number; loading: any; queries: any; + range: any; requestOptions: any; showingGraph: boolean; showingTable: boolean; @@ -72,7 +67,7 @@ export class Explore extends React.Component { constructor(props) { super(props); - const initialQueries = parseInitialQueries(props.routeParams.initial); + const { range, queries } = parseInitialState(props.routeParams.initial); this.state = { datasource: null, datasourceError: null, @@ -80,7 +75,8 @@ export class Explore extends React.Component { graphResult: null, latency: 0, loading: false, - queries: ensureQueries(initialQueries), + queries: ensureQueries(queries), + range: range || { ...DEFAULT_RANGE }, requestOptions: null, showingGraph: true, showingTable: true, @@ -119,6 +115,14 @@ export class Explore extends React.Component { this.setState({ queries: nextQueries }); }; + handleChangeTime = nextRange => { + const range = { + from: nextRange.from, + to: nextRange.to, + }; + this.setState({ range }, () => this.handleSubmit()); + }; + handleClickGraphButton = () => { this.setState(state => ({ showingGraph: !state.showingGraph })); }; @@ -147,7 +151,7 @@ export class Explore extends React.Component { }; async runGraphQuery() { - const { datasource, queries } = this.state; + const { datasource, queries, range } = this.state; if (!hasQuery(queries)) { return; } @@ -157,7 +161,7 @@ export class Explore extends React.Component { format: 'time_series', interval: datasource.interval, instant: false, - now, + range, queries: queries.map(q => q.query), }); try { @@ -172,7 +176,7 @@ export class Explore extends React.Component { } async runTableQuery() { - const { datasource, queries } = this.state; + const { datasource, queries, range } = this.state; if (!hasQuery(queries)) { return; } @@ -182,7 +186,7 @@ export class Explore extends React.Component { format: 'table', interval: datasource.interval, instant: true, - now, + range, queries: queries.map(q => q.query), }); try { @@ -210,6 +214,7 @@ export class Explore extends React.Component { latency, loading, queries, + range, requestOptions, showingGraph, showingTable, @@ -229,14 +234,8 @@ export class Explore extends React.Component { {datasource ? (
-
-
- {loading || latency ? : null} - -
-
+
+
@@ -244,6 +243,14 @@ export class Explore extends React.Component { Table
+
+ +
+ +
+ {loading || latency ? : null}
{ const $el = $(`#${this.props.id}`); const ticks = $el.width() / 100; - const min = userOptions.range.from.valueOf(); - const max = userOptions.range.to.valueOf(); + let { from, to } = userOptions.range; + if (!moment.isMoment(from)) { + from = dateMath.parse(from, false); + } + if (!moment.isMoment(to)) { + to = dateMath.parse(to, true); + } + const min = from.valueOf(); + const max = to.valueOf(); const dynamicOptions = { xaxis: { mode: 'time', diff --git a/public/app/containers/Explore/TimePicker.tsx b/public/app/containers/Explore/TimePicker.tsx new file mode 100644 index 00000000000..b67cd532019 --- /dev/null +++ b/public/app/containers/Explore/TimePicker.tsx @@ -0,0 +1,192 @@ +import React, { PureComponent } from 'react'; +import moment from 'moment'; + +import * as dateMath from 'app/core/utils/datemath'; +import * as rangeUtil from 'app/core/utils/rangeutil'; + +export const DEFAULT_RANGE = { + from: 'now-6h', + to: 'now', +}; + +export default class TimePicker extends PureComponent { + dropdownEl: any; + constructor(props) { + super(props); + this.state = { + fromRaw: props.range ? props.range.from : DEFAULT_RANGE.from, + isOpen: false, + isUtc: false, + rangeString: rangeUtil.describeTimeRange(props.range || DEFAULT_RANGE), + refreshInterval: '', + toRaw: props.range ? props.range.to : DEFAULT_RANGE.to, + }; + } + + move(direction) { + const { onChangeTime } = this.props; + const { fromRaw, toRaw } = this.state; + const range = { + from: dateMath.parse(fromRaw, false), + to: dateMath.parse(toRaw, true), + }; + + const timespan = (range.to.valueOf() - range.from.valueOf()) / 2; + let to, from; + if (direction === -1) { + to = range.to.valueOf() - timespan; + from = range.from.valueOf() - timespan; + } else if (direction === 1) { + to = range.to.valueOf() + timespan; + from = range.from.valueOf() + timespan; + if (to > Date.now() && range.to < Date.now()) { + to = Date.now(); + from = range.from.valueOf(); + } + } else { + to = range.to.valueOf(); + from = range.from.valueOf(); + } + + const rangeString = rangeUtil.describeTimeRange(range); + to = moment.utc(to); + from = moment.utc(from); + + this.setState( + { + rangeString, + fromRaw: from, + toRaw: to, + }, + () => { + onChangeTime({ to, from }); + } + ); + } + + handleChangeFrom = e => { + this.setState({ + fromRaw: e.target.value, + }); + }; + + handleChangeTo = e => { + this.setState({ + toRaw: e.target.value, + }); + }; + + handleClickLeft = () => this.move(-1); + handleClickPicker = () => { + this.setState(state => ({ + isOpen: !state.isOpen, + })); + }; + handleClickRight = () => this.move(1); + handleClickRefresh = () => {}; + handleClickRelativeOption = range => { + const { onChangeTime } = this.props; + const rangeString = rangeUtil.describeTimeRange(range); + this.setState( + { + toRaw: range.to, + fromRaw: range.from, + isOpen: false, + rangeString, + }, + () => { + if (onChangeTime) { + onChangeTime(range); + } + } + ); + }; + + getTimeOptions() { + return rangeUtil.getRelativeTimesList({}, this.state.rangeString); + } + + dropdownRef = el => { + this.dropdownEl = el; + }; + + renderDropdown() { + const { fromRaw, isOpen, toRaw } = this.state; + if (!isOpen) { + return null; + } + const timeOptions = this.getTimeOptions(); + return ( +
+
+

Custom range

+ + +
+
+ +
+
+ + +
+
+ +
+
+ + {/* +
+
+ +
+
*/} +
+ +
+

Quick ranges

+ {Object.keys(timeOptions).map(section => { + const group = timeOptions[section]; + return ( + + ); + })} +
+
+ ); + } + + render() { + const { isUtc, rangeString, refreshInterval } = this.state; + return ( +
+
+ + + +
+ {this.renderDropdown()} +
+ ); + } +} diff --git a/public/app/containers/Explore/utils/query.ts b/public/app/containers/Explore/utils/query.ts index d51c7339944..3aa0cc5b357 100644 --- a/public/app/containers/Explore/utils/query.ts +++ b/public/app/containers/Explore/utils/query.ts @@ -1,12 +1,7 @@ -export function buildQueryOptions({ format, interval, instant, now, queries }) { - const to = now; - const from = to - 1000 * 60 * 60 * 3; +export function buildQueryOptions({ format, interval, instant, range, queries }) { return { interval, - range: { - from, - to, - }, + range, targets: queries.map(expr => ({ expr, format, diff --git a/public/app/core/services/keybindingSrv.ts b/public/app/core/services/keybindingSrv.ts index 94bf9efb31b..25d00ab37f1 100644 --- a/public/app/core/services/keybindingSrv.ts +++ b/public/app/core/services/keybindingSrv.ts @@ -14,7 +14,7 @@ export class KeybindingSrv { timepickerOpen = false; /** @ngInject */ - constructor(private $rootScope, private $location, private datasourceSrv) { + constructor(private $rootScope, private $location, private datasourceSrv, private timeSrv) { // clear out all shortcuts on route change $rootScope.$on('$routeChangeSuccess', () => { Mousetrap.reset(); @@ -182,7 +182,12 @@ export class KeybindingSrv { const panel = dashboard.getPanelById(dashboard.meta.focusPanelId); const datasource = await this.datasourceSrv.get(panel.datasource); if (datasource && datasource.supportsExplore) { - const exploreState = encodePathComponent(JSON.stringify(datasource.getExploreState(panel))); + const range = this.timeSrv.timeRangeForUrl(); + const state = { + ...datasource.getExploreState(panel), + range, + }; + const exploreState = encodePathComponent(JSON.stringify(state)); this.$location.url(`/explore/${exploreState}`); } } diff --git a/public/app/features/panel/metrics_panel_ctrl.ts b/public/app/features/panel/metrics_panel_ctrl.ts index d460b27a679..3c48119ba3a 100644 --- a/public/app/features/panel/metrics_panel_ctrl.ts +++ b/public/app/features/panel/metrics_panel_ctrl.ts @@ -324,7 +324,12 @@ class MetricsPanelCtrl extends PanelCtrl { } explore() { - const exploreState = encodePathComponent(JSON.stringify(this.datasource.getExploreState(this.panel))); + const range = this.timeSrv.timeRangeForUrl(); + const state = { + ...this.datasource.getExploreState(this.panel), + range, + }; + const exploreState = encodePathComponent(JSON.stringify(state)); this.$location.url(`/explore/${exploreState}`); } diff --git a/public/sass/pages/_explore.scss b/public/sass/pages/_explore.scss index 855d11cb859..200af40341e 100644 --- a/public/sass/pages/_explore.scss +++ b/public/sass/pages/_explore.scss @@ -1,7 +1,21 @@ .explore { + .navbar { + padding-left: 0; + padding-right: 0; + } + + .elapsed-time { + position: absolute; + right: -2.4rem; + top: 1.2rem; + } .graph-legend { flex-wrap: wrap; } + + .timepicker { + display: flex; + } } .query-row { From eadaff619157b12ba697b9f7cf34aff7b83823b9 Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Tue, 1 May 2018 13:27:25 +0200 Subject: [PATCH 22/87] Explore: Design integration * style header like other grafana components * use panel container for graph and same styles for query field * fix typeahead CSS selector (was created outside of .explore) * use navbar buttons for +/- of rows * moved elapsed time under run query button * fix JS error on multiple timeseries being returned * fix color for graph lines * show prometheus query errors --- public/app/containers/Explore/Explore.tsx | 130 ++--- public/app/containers/Explore/Graph.tsx | 19 +- public/app/containers/Explore/QueryField.tsx | 2 +- public/app/containers/Explore/QueryRows.tsx | 5 +- .../datasource/prometheus/datasource.ts | 1 + .../prometheus/result_transformer.ts | 11 +- public/sass/pages/_explore.scss | 471 +++++++++--------- 7 files changed, 345 insertions(+), 294 deletions(-) diff --git a/public/app/containers/Explore/Explore.tsx b/public/app/containers/Explore/Explore.tsx index 66500353812..cf8cb41a593 100644 --- a/public/app/containers/Explore/Explore.tsx +++ b/public/app/containers/Explore/Explore.tsx @@ -4,7 +4,6 @@ import colors from 'app/core/utils/colors'; import TimeSeries from 'app/core/time_series2'; import ElapsedTime from './ElapsedTime'; -import Legend from './Legend'; import QueryRows from './QueryRows'; import Graph from './Graph'; import Table from './Table'; @@ -16,9 +15,7 @@ import { decodePathComponent } from 'app/core/utils/location_util'; function makeTimeSeriesList(dataList, options) { return dataList.map((seriesData, index) => { const datapoints = seriesData.datapoints || []; - const responseAlias = seriesData.target; - const query = options.targets[index].expr; - const alias = responseAlias && responseAlias !== '{}' ? responseAlias : query; + const alias = seriesData.target; const colorIndex = index % colors.length; const color = colors[colorIndex]; @@ -54,6 +51,7 @@ interface IExploreState { latency: number; loading: any; queries: any; + queryError: any; range: any; requestOptions: any; showingGraph: boolean; @@ -76,6 +74,7 @@ export class Explore extends React.Component { latency: 0, loading: false, queries: ensureQueries(queries), + queryError: null, range: range || { ...DEFAULT_RANGE }, requestOptions: null, showingGraph: true, @@ -94,6 +93,10 @@ export class Explore extends React.Component { } } + componentDidCatch(error) { + console.error(error); + } + handleAddQueryRow = index => { const { queries } = this.state; const nextQueries = [ @@ -155,7 +158,7 @@ export class Explore extends React.Component { if (!hasQuery(queries)) { return; } - this.setState({ latency: 0, loading: true, graphResult: null }); + this.setState({ latency: 0, loading: true, graphResult: null, queryError: null }); const now = Date.now(); const options = buildQueryOptions({ format: 'time_series', @@ -169,9 +172,10 @@ export class Explore extends React.Component { const result = makeTimeSeriesList(res.data, options); const latency = Date.now() - now; this.setState({ latency, loading: false, graphResult: result, requestOptions: options }); - } catch (error) { - console.error(error); - this.setState({ loading: false, graphResult: error }); + } catch (response) { + console.error(response); + const queryError = response.data ? response.data.error : response; + this.setState({ loading: false, queryError }); } } @@ -180,7 +184,7 @@ export class Explore extends React.Component { if (!hasQuery(queries)) { return; } - this.setState({ latency: 0, loading: true, tableResult: null }); + this.setState({ latency: 0, loading: true, queryError: null, tableResult: null }); const now = Date.now(); const options = buildQueryOptions({ format: 'table', @@ -194,9 +198,10 @@ export class Explore extends React.Component { const tableModel = res.data[0]; const latency = Date.now() - now; this.setState({ latency, loading: false, tableResult: tableModel, requestOptions: options }); - } catch (error) { - console.error(error); - this.setState({ loading: false, tableResult: null }); + } catch (response) { + console.error(response); + const queryError = response.data ? response.data.error : response; + this.setState({ loading: false, queryError }); } } @@ -214,6 +219,7 @@ export class Explore extends React.Component { latency, loading, queries, + queryError, range, requestOptions, showingGraph, @@ -221,55 +227,63 @@ export class Explore extends React.Component { tableResult, } = this.state; const showingBoth = showingGraph && showingTable; - const graphHeight = showingBoth ? '200px' : null; - const graphButtonClassName = showingBoth || showingGraph ? 'btn m-r-1' : 'btn btn-inverse m-r-1'; - const tableButtonClassName = showingBoth || showingTable ? 'btn m-r-1' : 'btn btn-inverse m-r-1'; + const graphHeight = showingBoth ? '200px' : '400px'; + const graphButtonActive = showingBoth || showingGraph ? 'active' : ''; + const tableButtonActive = showingBoth || showingTable ? 'active' : ''; return (
-
-

Explore

- {datasourceLoading ?
Loading datasource...
: null} - - {datasourceError ?
Error connecting to datasource.
: null} - - {datasource ? ( -
-
-
- - -
-
- -
- -
- {loading || latency ? : null} -
- -
- {showingGraph ? ( - - ) : null} - {showingGraph ? : null} - {showingTable ? : null} - - - ) : null} +
+ +
+
+ + +
+ +
+ + {loading || latency ? : null} +
+ + {datasourceLoading ?
Loading datasource...
: null} + + {datasourceError ? ( +
+ Error connecting to datasource. +
+ ) : null} + + {datasource ? ( +
+ + {queryError ?
{queryError}
: null} +
+ {showingGraph ? ( + + ) : null} + {showingTable ?
: null} + + + ) : null} ); } diff --git a/public/app/containers/Explore/Graph.tsx b/public/app/containers/Explore/Graph.tsx index b8bda8696bc..d797a579512 100644 --- a/public/app/containers/Explore/Graph.tsx +++ b/public/app/containers/Explore/Graph.tsx @@ -2,11 +2,12 @@ import $ from 'jquery'; import React, { Component } from 'react'; import moment from 'moment'; +import 'vendor/flot/jquery.flot'; +import 'vendor/flot/jquery.flot.time'; import * as dateMath from 'app/core/utils/datemath'; import TimeSeries from 'app/core/time_series2'; -import 'vendor/flot/jquery.flot'; -import 'vendor/flot/jquery.flot.time'; +import Legend from './Legend'; // Copied from graph.ts function time_format(ticks, min, max) { @@ -86,6 +87,7 @@ class Graph extends Component { return; } const series = data.map((ts: TimeSeries) => ({ + color: ts.color, label: ts.label, data: ts.getFlotPairs('null'), })); @@ -120,12 +122,13 @@ class Graph extends Component { } render() { - const style = { - height: this.props.height || '400px', - width: this.props.width || '100%', - }; - - return
; + const { data, height } = this.props; + return ( +
+
+ +
+ ); } } diff --git a/public/app/containers/Explore/QueryField.tsx b/public/app/containers/Explore/QueryField.tsx index 816473619fd..53354584fea 100644 --- a/public/app/containers/Explore/QueryField.tsx +++ b/public/app/containers/Explore/QueryField.tsx @@ -50,7 +50,7 @@ class Portal extends React.Component { constructor(props) { super(props); this.node = document.createElement('div'); - this.node.classList.add(`query-field-portal-${props.index}`); + this.node.classList.add('explore-typeahead', `explore-typeahead-${props.index}`); document.body.appendChild(this.node); } diff --git a/public/app/containers/Explore/QueryRows.tsx b/public/app/containers/Explore/QueryRows.tsx index 3940d16b2f6..74f6c28d41b 100644 --- a/public/app/containers/Explore/QueryRows.tsx +++ b/public/app/containers/Explore/QueryRows.tsx @@ -48,10 +48,10 @@ class QueryRow extends PureComponent { return (
- -
@@ -60,6 +60,7 @@ class QueryRow extends PureComponent { initialQuery={edited ? null : query} onPressEnter={this.handlePressEnter} onQueryChange={this.handleChangeQuery} + placeholder="Enter a PromQL query" request={request} />
diff --git a/public/app/plugins/datasource/prometheus/datasource.ts b/public/app/plugins/datasource/prometheus/datasource.ts index a52f3aefa2e..7470885177b 100644 --- a/public/app/plugins/datasource/prometheus/datasource.ts +++ b/public/app/plugins/datasource/prometheus/datasource.ts @@ -164,6 +164,7 @@ export class PrometheusDatasource { legendFormat: activeTargets[index].legendFormat, start: start, end: end, + query: queries[index].expr, responseListLength: responseList.length, responseIndex: index, refId: activeTargets[index].refId, diff --git a/public/app/plugins/datasource/prometheus/result_transformer.ts b/public/app/plugins/datasource/prometheus/result_transformer.ts index d5feda7d28c..7f5430bf7d6 100644 --- a/public/app/plugins/datasource/prometheus/result_transformer.ts +++ b/public/app/plugins/datasource/prometheus/result_transformer.ts @@ -123,11 +123,16 @@ export class ResultTransformer { } createMetricLabel(labelData, options) { + let label = ''; if (_.isUndefined(options) || _.isEmpty(options.legendFormat)) { - return this.getOriginalMetricName(labelData); + label = this.getOriginalMetricName(labelData); + } else { + label = this.renderTemplate(this.templateSrv.replace(options.legendFormat), labelData); } - - return this.renderTemplate(this.templateSrv.replace(options.legendFormat), labelData) || '{}'; + if (!label || label === '{}') { + label = options.query; + } + return label; } renderTemplate(aliasPattern, aliasData) { diff --git a/public/sass/pages/_explore.scss b/public/sass/pages/_explore.scss index 200af40341e..541477877bc 100644 --- a/public/sass/pages/_explore.scss +++ b/public/sass/pages/_explore.scss @@ -1,14 +1,35 @@ .explore { - .navbar { - padding-left: 0; - padding-right: 0; + .explore-container { + padding: 2rem; + } + + .explore-graph { + width: 100%; + height: 100%; + } + + .panel-container { + padding: 10px 10px 5px 10px; + } + + .navbar-page-btn .fa { + position: relative; + top: -1px; + font-size: 19px; + line-height: 8px; + opacity: 0.75; + margin-right: 8px; } .elapsed-time { position: absolute; - right: -2.4rem; - top: 1.2rem; + left: 0; + right: 0; + top: 3.5rem; + text-align: center; + font-size: 0.8rem; } + .graph-legend { flex-wrap: wrap; } @@ -16,10 +37,19 @@ .timepicker { display: flex; } + + .run-icon { + margin-left: 0.5em; + transform: rotate(90deg); + } + + .relative { + position: relative; + } } .query-row { - position: relative; + display: flex; & + & { margin-top: 0.5rem; @@ -27,12 +57,7 @@ } .query-row-tools { - position: absolute; - left: -4rem; - top: 0.33rem; - > * { - margin-right: 0.25rem; - } + width: 4rem; } .query-field { @@ -49,14 +74,14 @@ cursor: text; line-height: 1.5; color: rgba(0, 0, 0, 0.65); - background-color: #fff; + background-color: $panel-bg; background-image: none; - border: 1px solid lightgray; + border: $panel-border; border-radius: 3px; transition: all 0.3s; } -.explore { +.explore-typeahead { .typeahead { position: absolute; z-index: auto; @@ -117,221 +142,223 @@ * @author Tim Shedor */ -code[class*='language-'], -pre[class*='language-'] { - color: black; - background: none; - font-family: Consolas, Monaco, 'Andale Mono', 'Ubuntu Mono', monospace; - text-align: left; - white-space: pre; - word-spacing: normal; - word-break: normal; - word-wrap: normal; - line-height: 1.5; +.explore { + code[class*='language-'], + pre[class*='language-'] { + color: black; + background: none; + font-family: Consolas, Monaco, 'Andale Mono', 'Ubuntu Mono', monospace; + text-align: left; + white-space: pre; + word-spacing: normal; + word-break: normal; + word-wrap: normal; + line-height: 1.5; - -moz-tab-size: 4; - -o-tab-size: 4; - tab-size: 4; + -moz-tab-size: 4; + -o-tab-size: 4; + tab-size: 4; - -webkit-hyphens: none; - -moz-hyphens: none; - -ms-hyphens: none; - hyphens: none; -} + -webkit-hyphens: none; + -moz-hyphens: none; + -ms-hyphens: none; + hyphens: none; + } -/* Code blocks */ -pre[class*='language-'] { - position: relative; - margin: 0.5em 0; - overflow: visible; - padding: 0; -} -pre[class*='language-'] > code { - position: relative; - border-left: 10px solid #358ccb; - box-shadow: -1px 0px 0px 0px #358ccb, 0px 0px 0px 1px #dfdfdf; - background-color: #fdfdfd; - background-image: linear-gradient(transparent 50%, rgba(69, 142, 209, 0.04) 50%); - background-size: 3em 3em; - background-origin: content-box; - background-attachment: local; -} + /* Code blocks */ + pre[class*='language-'] { + position: relative; + margin: 0.5em 0; + overflow: visible; + padding: 0; + } + pre[class*='language-'] > code { + position: relative; + border-left: 10px solid #358ccb; + box-shadow: -1px 0px 0px 0px #358ccb, 0px 0px 0px 1px #dfdfdf; + background-color: #fdfdfd; + background-image: linear-gradient(transparent 50%, rgba(69, 142, 209, 0.04) 50%); + background-size: 3em 3em; + background-origin: content-box; + background-attachment: local; + } -code[class*='language'] { - max-height: inherit; - height: inherit; - padding: 0 1em; - display: block; - overflow: auto; -} + code[class*='language'] { + max-height: inherit; + height: inherit; + padding: 0 1em; + display: block; + overflow: auto; + } -/* Margin bottom to accomodate shadow */ -:not(pre) > code[class*='language-'], -pre[class*='language-'] { - background-color: #fdfdfd; - -webkit-box-sizing: border-box; - -moz-box-sizing: border-box; - box-sizing: border-box; - margin-bottom: 1em; -} + /* Margin bottom to accomodate shadow */ + :not(pre) > code[class*='language-'], + pre[class*='language-'] { + background-color: #fdfdfd; + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + box-sizing: border-box; + margin-bottom: 1em; + } -/* Inline code */ -:not(pre) > code[class*='language-'] { - position: relative; - padding: 0.2em; - border-radius: 0.3em; - color: #c92c2c; - border: 1px solid rgba(0, 0, 0, 0.1); - display: inline; - white-space: normal; -} + /* Inline code */ + :not(pre) > code[class*='language-'] { + position: relative; + padding: 0.2em; + border-radius: 0.3em; + color: #c92c2c; + border: 1px solid rgba(0, 0, 0, 0.1); + display: inline; + white-space: normal; + } -pre[class*='language-']:before, -pre[class*='language-']:after { - content: ''; - z-index: -2; - display: block; - position: absolute; - bottom: 0.75em; - left: 0.18em; - width: 40%; - height: 20%; - max-height: 13em; - box-shadow: 0px 13px 8px #979797; - -webkit-transform: rotate(-2deg); - -moz-transform: rotate(-2deg); - -ms-transform: rotate(-2deg); - -o-transform: rotate(-2deg); - transform: rotate(-2deg); -} - -:not(pre) > code[class*='language-']:after, -pre[class*='language-']:after { - right: 0.75em; - left: auto; - -webkit-transform: rotate(2deg); - -moz-transform: rotate(2deg); - -ms-transform: rotate(2deg); - -o-transform: rotate(2deg); - transform: rotate(2deg); -} - -.token.comment, -.token.block-comment, -.token.prolog, -.token.doctype, -.token.cdata { - color: #7d8b99; -} - -.token.punctuation { - color: #5f6364; -} - -.token.property, -.token.tag, -.token.boolean, -.token.number, -.token.function-name, -.token.constant, -.token.symbol, -.token.deleted { - color: #c92c2c; -} - -.token.selector, -.token.attr-name, -.token.string, -.token.char, -.token.function, -.token.builtin, -.token.inserted { - color: #2f9c0a; -} - -.token.operator, -.token.entity, -.token.url, -.token.variable { - color: #a67f59; - background: rgba(255, 255, 255, 0.5); -} - -.token.atrule, -.token.attr-value, -.token.keyword, -.token.class-name { - color: #1990b8; -} - -.token.regex, -.token.important { - color: #e90; -} - -.language-css .token.string, -.style .token.string { - color: #a67f59; - background: rgba(255, 255, 255, 0.5); -} - -.token.important { - font-weight: normal; -} - -.token.bold { - font-weight: bold; -} -.token.italic { - font-style: italic; -} - -.token.entity { - cursor: help; -} - -.namespace { - opacity: 0.7; -} - -@media screen and (max-width: 767px) { pre[class*='language-']:before, pre[class*='language-']:after { - bottom: 14px; - box-shadow: none; + content: ''; + z-index: -2; + display: block; + position: absolute; + bottom: 0.75em; + left: 0.18em; + width: 40%; + height: 20%; + max-height: 13em; + box-shadow: 0px 13px 8px #979797; + -webkit-transform: rotate(-2deg); + -moz-transform: rotate(-2deg); + -ms-transform: rotate(-2deg); + -o-transform: rotate(-2deg); + transform: rotate(-2deg); + } + + :not(pre) > code[class*='language-']:after, + pre[class*='language-']:after { + right: 0.75em; + left: auto; + -webkit-transform: rotate(2deg); + -moz-transform: rotate(2deg); + -ms-transform: rotate(2deg); + -o-transform: rotate(2deg); + transform: rotate(2deg); + } + + .token.comment, + .token.block-comment, + .token.prolog, + .token.doctype, + .token.cdata { + color: #7d8b99; + } + + .token.punctuation { + color: #5f6364; + } + + .token.property, + .token.tag, + .token.boolean, + .token.number, + .token.function-name, + .token.constant, + .token.symbol, + .token.deleted { + color: #c92c2c; + } + + .token.selector, + .token.attr-name, + .token.string, + .token.char, + .token.function, + .token.builtin, + .token.inserted { + color: #2f9c0a; + } + + .token.operator, + .token.entity, + .token.url, + .token.variable { + color: #a67f59; + background: rgba(255, 255, 255, 0.5); + } + + .token.atrule, + .token.attr-value, + .token.keyword, + .token.class-name { + color: #1990b8; + } + + .token.regex, + .token.important { + color: #e90; + } + + .language-css .token.string, + .style .token.string { + color: #a67f59; + background: rgba(255, 255, 255, 0.5); + } + + .token.important { + font-weight: normal; + } + + .token.bold { + font-weight: bold; + } + .token.italic { + font-style: italic; + } + + .token.entity { + cursor: help; + } + + .namespace { + opacity: 0.7; + } + + @media screen and (max-width: 767px) { + pre[class*='language-']:before, + pre[class*='language-']:after { + bottom: 14px; + box-shadow: none; + } + } + + /* Plugin styles */ + .token.tab:not(:empty):before, + .token.cr:before, + .token.lf:before { + color: #e0d7d1; + } + + /* Plugin styles: Line Numbers */ + pre[class*='language-'].line-numbers { + padding-left: 0; + } + + pre[class*='language-'].line-numbers code { + padding-left: 3.8em; + } + + pre[class*='language-'].line-numbers .line-numbers-rows { + left: 0; + } + + /* Plugin styles: Line Highlight */ + pre[class*='language-'][data-line] { + padding-top: 0; + padding-bottom: 0; + padding-left: 0; + } + pre[data-line] code { + position: relative; + padding-left: 4em; + } + pre .line-highlight { + margin-top: 0; } } - -/* Plugin styles */ -.token.tab:not(:empty):before, -.token.cr:before, -.token.lf:before { - color: #e0d7d1; -} - -/* Plugin styles: Line Numbers */ -pre[class*='language-'].line-numbers { - padding-left: 0; -} - -pre[class*='language-'].line-numbers code { - padding-left: 3.8em; -} - -pre[class*='language-'].line-numbers .line-numbers-rows { - left: 0; -} - -/* Plugin styles: Line Highlight */ -pre[class*='language-'][data-line] { - padding-top: 0; - padding-bottom: 0; - padding-left: 0; -} -pre[data-line] code { - position: relative; - padding-left: 4em; -} -pre .line-highlight { - margin-top: 0; -} From 23c9da6162f268e1134023f2aba3d1b52c5310de Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Tue, 15 May 2018 15:37:44 +0200 Subject: [PATCH 23/87] Fixed custom dates for react timepicker * added jest tests for timepicker component --- .../containers/Explore/TimePicker.jest.tsx | 74 ++++++++++++++++ public/app/containers/Explore/TimePicker.tsx | 85 +++++++++++++++---- public/sass/pages/_explore.scss | 4 + 3 files changed, 147 insertions(+), 16 deletions(-) create mode 100644 public/app/containers/Explore/TimePicker.jest.tsx diff --git a/public/app/containers/Explore/TimePicker.jest.tsx b/public/app/containers/Explore/TimePicker.jest.tsx new file mode 100644 index 00000000000..afe6b092901 --- /dev/null +++ b/public/app/containers/Explore/TimePicker.jest.tsx @@ -0,0 +1,74 @@ +import React from 'react'; +import { shallow } from 'enzyme'; +import sinon from 'sinon'; + +import * as rangeUtil from 'app/core/utils/rangeutil'; +import TimePicker, { DEFAULT_RANGE, parseTime } from './TimePicker'; + +describe('', () => { + it('renders closed with default values', () => { + const rangeString = rangeUtil.describeTimeRange(DEFAULT_RANGE); + const wrapper = shallow(); + expect(wrapper.find('.timepicker-rangestring').text()).toBe(rangeString); + expect(wrapper.find('.gf-timepicker-dropdown').exists()).toBe(false); + }); + + it('renders with relative range', () => { + const range = { + from: 'now-7h', + to: 'now', + }; + const rangeString = rangeUtil.describeTimeRange(range); + const wrapper = shallow(); + expect(wrapper.find('.timepicker-rangestring').text()).toBe(rangeString); + expect(wrapper.state('fromRaw')).toBe(range.from); + expect(wrapper.state('toRaw')).toBe(range.to); + expect(wrapper.find('.timepicker-from').props().value).toBe(range.from); + expect(wrapper.find('.timepicker-to').props().value).toBe(range.to); + }); + + it('renders with epoch (millies) range converted to ISO-ish', () => { + const range = { + from: '1', + to: '1000', + }; + const rangeString = rangeUtil.describeTimeRange({ + from: parseTime(range.from), + to: parseTime(range.to), + }); + const wrapper = shallow(); + expect(wrapper.state('fromRaw')).toBe('1970-01-01 00:00:00'); + expect(wrapper.state('toRaw')).toBe('1970-01-01 00:00:01'); + expect(wrapper.find('.timepicker-rangestring').text()).toBe(rangeString); + expect(wrapper.find('.timepicker-from').props().value).toBe('1970-01-01 00:00:00'); + expect(wrapper.find('.timepicker-to').props().value).toBe('1970-01-01 00:00:01'); + }); + + it('moves ranges forward and backward by half the range on arrow click', () => { + const range = { + from: '2000', + to: '4000', + }; + const rangeString = rangeUtil.describeTimeRange({ + from: parseTime(range.from), + to: parseTime(range.to), + }); + + const onChangeTime = sinon.spy(); + const wrapper = shallow(); + expect(wrapper.state('fromRaw')).toBe('1970-01-01 00:00:02'); + expect(wrapper.state('toRaw')).toBe('1970-01-01 00:00:04'); + expect(wrapper.find('.timepicker-rangestring').text()).toBe(rangeString); + expect(wrapper.find('.timepicker-from').props().value).toBe('1970-01-01 00:00:02'); + expect(wrapper.find('.timepicker-to').props().value).toBe('1970-01-01 00:00:04'); + + wrapper.find('.timepicker-left').simulate('click'); + expect(onChangeTime.calledOnce).toBe(true); + expect(wrapper.state('fromRaw')).toBe('1970-01-01 00:00:01'); + expect(wrapper.state('toRaw')).toBe('1970-01-01 00:00:03'); + + wrapper.find('.timepicker-right').simulate('click'); + expect(wrapper.state('fromRaw')).toBe('1970-01-01 00:00:02'); + expect(wrapper.state('toRaw')).toBe('1970-01-01 00:00:04'); + }); +}); diff --git a/public/app/containers/Explore/TimePicker.tsx b/public/app/containers/Explore/TimePicker.tsx index b67cd532019..3ae4ea4a83c 100644 --- a/public/app/containers/Explore/TimePicker.tsx +++ b/public/app/containers/Explore/TimePicker.tsx @@ -4,22 +4,43 @@ import moment from 'moment'; import * as dateMath from 'app/core/utils/datemath'; import * as rangeUtil from 'app/core/utils/rangeutil'; +const DATE_FORMAT = 'YYYY-MM-DD HH:mm:ss'; + export const DEFAULT_RANGE = { from: 'now-6h', to: 'now', }; +export function parseTime(value, isUtc = false, asString = false) { + if (value.indexOf('now') !== -1) { + return value; + } + if (!isNaN(value)) { + const epoch = parseInt(value); + const m = isUtc ? moment.utc(epoch) : moment(epoch); + return asString ? m.format(DATE_FORMAT) : m; + } + return undefined; +} + export default class TimePicker extends PureComponent { dropdownEl: any; constructor(props) { super(props); + + const fromRaw = props.range ? props.range.from : DEFAULT_RANGE.from; + const toRaw = props.range ? props.range.to : DEFAULT_RANGE.to; + const range = { + from: parseTime(fromRaw), + to: parseTime(toRaw), + }; this.state = { - fromRaw: props.range ? props.range.from : DEFAULT_RANGE.from, - isOpen: false, - isUtc: false, - rangeString: rangeUtil.describeTimeRange(props.range || DEFAULT_RANGE), + fromRaw: parseTime(fromRaw, props.isUtc, true), + isOpen: props.isOpen, + isUtc: props.isUtc, + rangeString: rangeUtil.describeTimeRange(range), refreshInterval: '', - toRaw: props.range ? props.range.to : DEFAULT_RANGE.to, + toRaw: parseTime(toRaw, props.isUtc, true), }; } @@ -49,14 +70,15 @@ export default class TimePicker extends PureComponent { } const rangeString = rangeUtil.describeTimeRange(range); - to = moment.utc(to); - from = moment.utc(from); + // No need to convert to UTC again + to = moment(to); + from = moment(from); this.setState( { rangeString, - fromRaw: from, - toRaw: to, + fromRaw: from.format(DATE_FORMAT), + toRaw: to.format(DATE_FORMAT), }, () => { onChangeTime({ to, from }); @@ -76,6 +98,27 @@ export default class TimePicker extends PureComponent { }); }; + handleClickApply = () => { + const { onChangeTime } = this.props; + const { toRaw, fromRaw } = this.state; + const range = { + from: dateMath.parse(fromRaw, false), + to: dateMath.parse(toRaw, true), + }; + const rangeString = rangeUtil.describeTimeRange(range); + this.setState( + { + isOpen: false, + rangeString, + }, + () => { + if (onChangeTime) { + onChangeTime(range); + } + } + ); + }; + handleClickLeft = () => this.move(-1); handleClickPicker = () => { this.setState(state => ({ @@ -118,7 +161,7 @@ export default class TimePicker extends PureComponent { const timeOptions = this.getTimeOptions(); return (
-
+

Custom range

@@ -126,7 +169,7 @@ export default class TimePicker extends PureComponent {
@@ -136,7 +179,12 @@ export default class TimePicker extends PureComponent {
- +
@@ -146,7 +194,12 @@ export default class TimePicker extends PureComponent {
*/} - +
+ +
+

Quick ranges

@@ -172,16 +225,16 @@ export default class TimePicker extends PureComponent { return (
- -
diff --git a/public/sass/pages/_explore.scss b/public/sass/pages/_explore.scss index 541477877bc..47fb3857225 100644 --- a/public/sass/pages/_explore.scss +++ b/public/sass/pages/_explore.scss @@ -36,6 +36,10 @@ .timepicker { display: flex; + + &-rangestring { + margin-left: 0.5em; + } } .run-icon { From f5e351af8b7ebd72800502b65aa7e6f741c08106 Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Tue, 15 May 2018 17:07:38 +0200 Subject: [PATCH 24/87] Explore split view * button to bring a up a duplicate explore area to compare * side by side rendering of two explore components * right component has close button * left component has page title --- public/app/containers/Explore/Explore.tsx | 54 +++++++++++++++++++---- public/app/containers/Explore/Graph.tsx | 1 + public/app/containers/Explore/Wrapper.tsx | 33 ++++++++++++++ public/app/routes/routes.ts | 3 +- public/sass/pages/_explore.scss | 48 ++++++++++++++++---- 5 files changed, 121 insertions(+), 18 deletions(-) create mode 100644 public/app/containers/Explore/Wrapper.tsx diff --git a/public/app/containers/Explore/Explore.tsx b/public/app/containers/Explore/Explore.tsx index cf8cb41a593..deebe84f2c8 100644 --- a/public/app/containers/Explore/Explore.tsx +++ b/public/app/containers/Explore/Explore.tsx @@ -80,6 +80,7 @@ export class Explore extends React.Component { showingGraph: true, showingTable: true, tableResult: null, + ...props.initialState, }; } @@ -126,10 +127,24 @@ export class Explore extends React.Component { this.setState({ range }, () => this.handleSubmit()); }; + handleClickCloseSplit = () => { + const { onChangeSplit } = this.props; + if (onChangeSplit) { + onChangeSplit(false); + } + }; + handleClickGraphButton = () => { this.setState(state => ({ showingGraph: !state.showingGraph })); }; + handleClickSplit = () => { + const { onChangeSplit } = this.props; + if (onChangeSplit) { + onChangeSplit(true, this.state); + } + }; + handleClickTableButton = () => { this.setState(state => ({ showingTable: !state.showingTable })); }; @@ -211,6 +226,7 @@ export class Explore extends React.Component { }; render() { + const { position, split } = this.props; const { datasource, datasourceError, @@ -230,16 +246,32 @@ export class Explore extends React.Component { const graphHeight = showingBoth ? '200px' : '400px'; const graphButtonActive = showingBoth || showingGraph ? 'active' : ''; const tableButtonActive = showingBoth || showingTable ? 'active' : ''; + const exploreClass = split ? 'explore explore-split' : 'explore'; return ( -
+
- + {position === 'left' ? ( + + ) : ( +
+ +
+ )}
+ {position === 'left' && !split ? ( +
+ +
+ ) : null}
: null} diff --git a/public/app/containers/Explore/Graph.tsx b/public/app/containers/Explore/Graph.tsx index d797a579512..a43ddfb2aa5 100644 --- a/public/app/containers/Explore/Graph.tsx +++ b/public/app/containers/Explore/Graph.tsx @@ -75,6 +75,7 @@ class Graph extends Component { if ( prevProps.data !== this.props.data || prevProps.options !== this.props.options || + prevProps.split !== this.props.split || prevProps.height !== this.props.height ) { this.draw(); diff --git a/public/app/containers/Explore/Wrapper.tsx b/public/app/containers/Explore/Wrapper.tsx new file mode 100644 index 00000000000..6bdbd7cc42f --- /dev/null +++ b/public/app/containers/Explore/Wrapper.tsx @@ -0,0 +1,33 @@ +import React, { PureComponent } from 'react'; + +import Explore from './Explore'; + +export default class Wrapper extends PureComponent { + state = { + initialState: null, + split: false, + }; + + handleChangeSplit = (split, initialState) => { + this.setState({ split, initialState }); + }; + + render() { + // State overrides for props from first Explore + const { initialState, split } = this.state; + return ( +
+ + {split ? ( + + ) : null} +
+ ); + } +} diff --git a/public/app/routes/routes.ts b/public/app/routes/routes.ts index 6a61315f956..b10084d1941 100644 --- a/public/app/routes/routes.ts +++ b/public/app/routes/routes.ts @@ -3,7 +3,6 @@ import './ReactContainer'; import ServerStats from 'app/containers/ServerStats/ServerStats'; import AlertRuleList from 'app/containers/AlertRuleList/AlertRuleList'; -// import Explore from 'app/containers/Explore/Explore'; import FolderSettings from 'app/containers/ManageDashboards/FolderSettings'; import FolderPermissions from 'app/containers/ManageDashboards/FolderPermissions'; @@ -114,7 +113,7 @@ export function setupAngularRoutes($routeProvider, $locationProvider) { .when('/explore/:initial?', { template: '', resolve: { - component: () => import(/* webpackChunkName: "explore" */ 'app/containers/Explore/Explore'), + component: () => import(/* webpackChunkName: "explore" */ 'app/containers/Explore/Wrapper'), }, }) .when('/org', { diff --git a/public/sass/pages/_explore.scss b/public/sass/pages/_explore.scss index 47fb3857225..b618b8eb5b7 100644 --- a/public/sass/pages/_explore.scss +++ b/public/sass/pages/_explore.scss @@ -1,8 +1,22 @@ .explore { - .explore-container { + width: 100%; + + &-container { padding: 2rem; } + &-wrapper { + display: flex; + + > .explore-split { + width: 50%; + } + } + + .explore-first-button { + margin-left: 15px; + } + .explore-graph { width: 100%; height: 100%; @@ -12,13 +26,27 @@ padding: 10px 10px 5px 10px; } - .navbar-page-btn .fa { - position: relative; - top: -1px; - font-size: 19px; - line-height: 8px; - opacity: 0.75; - margin-right: 8px; + .navbar { + flex-wrap: wrap; + height: auto; + } + + .navbar-page-btn { + margin-right: 1rem; + + .fa { + position: relative; + top: -1px; + font-size: 19px; + line-height: 8px; + opacity: 0.75; + margin-right: 8px; + } + } + + .navbar-button.active { + color: #0083b3; + background-color: white; } .elapsed-time { @@ -52,6 +80,10 @@ } } +.explore + .explore { + border-left: 1px dotted #aaa; +} + .query-row { display: flex; From fac0333f472920237acc9ced610b7a870a2dfaca Mon Sep 17 00:00:00 2001 From: balyn Date: Thu, 24 May 2018 19:35:04 +0300 Subject: [PATCH 25/87] The old code for centering removed Old code removed --- public/sass/components/_panel_singlestat.scss | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/public/sass/components/_panel_singlestat.scss b/public/sass/components/_panel_singlestat.scss index c84234bde9f..faaa6fc2447 100644 --- a/public/sass/components/_panel_singlestat.scss +++ b/public/sass/components/_panel_singlestat.scss @@ -8,13 +8,11 @@ .singlestat-panel-value-container { line-height: 1; display: table-cell; - vertical-align: middle; - text-align: center; + position: absolute; z-index: 1; font-size: 3em; font-weight: bold; margin: 0; - position: absolute; top: 50%; left: 50%; transform: translate(-50%, -50%); From 4752d7884ab44323fcb425955c135b6182b673be Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Fri, 25 May 2018 10:31:56 +0200 Subject: [PATCH 26/87] elasticsearch: adds some more/better debug logging to client --- pkg/tsdb/elasticsearch/client/client.go | 30 +++++++++++++++++++------ 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/pkg/tsdb/elasticsearch/client/client.go b/pkg/tsdb/elasticsearch/client/client.go index 4fe5c40b127..3a2d31b42c6 100644 --- a/pkg/tsdb/elasticsearch/client/client.go +++ b/pkg/tsdb/elasticsearch/client/client.go @@ -119,6 +119,9 @@ type multiRequest struct { } func (c *baseClientImpl) executeBatchRequest(uriPath string, requests []*multiRequest) (*http.Response, error) { + clientLog.Debug("Encoding batch requests to json", "batch requests", len(requests)) + start := time.Now() + payload := bytes.Buffer{} for _, r := range requests { reqHeader, err := json.Marshal(r.header) @@ -134,6 +137,9 @@ func (c *baseClientImpl) executeBatchRequest(uriPath string, requests []*multiRe payload.WriteString(string(reqBody) + "\n") } + elapsed := time.Now().Sub(start) + clientLog.Debug("Encoded batch requests to json", "took", elapsed) + return c.executeRequest(http.MethodPost, uriPath, payload.Bytes()) } @@ -151,6 +157,9 @@ func (c *baseClientImpl) executeRequest(method, uriPath string, body []byte) (*h if err != nil { return nil, err } + + clientLog.Debug("Executing request", "url", req.URL.String(), "method", method) + req.Header.Set("User-Agent", "Grafana") req.Header.Set("Content-Type", "application/json") @@ -169,22 +178,28 @@ func (c *baseClientImpl) executeRequest(method, uriPath string, body []byte) (*h return nil, err } - if method == http.MethodPost { - clientLog.Debug("Executing request", "url", req.URL.String(), "method", method) - } else { - clientLog.Debug("Executing request", "url", req.URL.String(), "method", method) - } - + start := time.Now() + defer func() { + elapsed := time.Now().Sub(start) + clientLog.Debug("Executed request", "took", elapsed) + }() return ctxhttp.Do(c.ctx, httpClient, req) } func (c *baseClientImpl) ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearchResponse, error) { + clientLog.Debug("Executing multisearch", "search requests", len(r.Requests)) + multiRequests := c.createMultiSearchRequests(r.Requests) res, err := c.executeBatchRequest("_msearch", multiRequests) if err != nil { return nil, err } + clientLog.Debug("Received multisearch response", "code", res.StatusCode, "status", res.Status, "content-length", res.ContentLength) + + start := time.Now() + clientLog.Debug("Decoding multisearch json response") + var msr MultiSearchResponse defer res.Body.Close() dec := json.NewDecoder(res.Body) @@ -193,7 +208,8 @@ func (c *baseClientImpl) ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearch return nil, err } - clientLog.Debug("Received multisearch response", "code", res.StatusCode, "status", res.Status, "content-length", res.ContentLength) + elapsed := time.Now().Sub(start) + clientLog.Debug("Decoded multisearch json response", "took", elapsed) msr.status = res.StatusCode From 448b1cbc155f611083e1b99e8ac7fe2816d4893e Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Fri, 25 May 2018 12:51:27 +0200 Subject: [PATCH 27/87] Integrated dark theme for explore UI --- public/sass/_variables.dark.scss | 12 ++ public/sass/_variables.light.scss | 12 ++ public/sass/pages/_explore.scss | 236 +++++------------------------- 3 files changed, 63 insertions(+), 197 deletions(-) diff --git a/public/sass/_variables.dark.scss b/public/sass/_variables.dark.scss index 6e86aa1872e..4907540815d 100644 --- a/public/sass/_variables.dark.scss +++ b/public/sass/_variables.dark.scss @@ -45,6 +45,10 @@ $brand-warning: $brand-primary; $brand-danger: $red; $query-blue: $blue; +$query-red: $red; +$query-green: $green; +$query-purple: $purple; +$query-orange: $orange; // Status colors // ------------------------- @@ -176,6 +180,9 @@ $btn-inverse-bg-hl: lighten($dark-3, 4%); $btn-inverse-text-color: $link-color; $btn-inverse-text-shadow: 0px 1px 0 rgba(0, 0, 0, 0.1); +$btn-active-bg: $gray-4; +$btn-active-text-color: $blue-dark; + $btn-link-color: $gray-3; $iconContainerBackground: $black; @@ -204,6 +211,11 @@ $input-invalid-border-color: lighten($red, 5%); $search-shadow: 0 0 30px 0 $black; $search-filter-box-bg: $gray-blue; +// Typeahead +$typeahead-shadow: 0 5px 10px 0 $black; +$typeahead-selected-bg: $dark-4; +$typeahead-selected-color: $blue; + // Dropdowns // ------------------------- $dropdownBackground: $dark-3; diff --git a/public/sass/_variables.light.scss b/public/sass/_variables.light.scss index bb8f93dbe69..14716f6dfef 100644 --- a/public/sass/_variables.light.scss +++ b/public/sass/_variables.light.scss @@ -46,6 +46,10 @@ $brand-warning: $orange; $brand-danger: $red; $query-blue: $blue-dark; +$query-red: $red; +$query-green: $green; +$query-purple: $purple; +$query-orange: $orange; // Status colors // ------------------------- @@ -173,6 +177,9 @@ $btn-inverse-bg-hl: darken($gray-6, 5%); $btn-inverse-text-color: $gray-1; $btn-inverse-text-shadow: 0 1px 0 rgba(255, 255, 255, 0.4); +$btn-active-bg: $white; +$btn-active-text-color: $blue-dark; + $btn-link-color: $gray-1; $btn-divider-left: $gray-4; @@ -226,6 +233,11 @@ $tab-border-color: $gray-5; $search-shadow: 0 5px 30px 0 $gray-4; $search-filter-box-bg: $gray-7; +// Typeahead +$typeahead-shadow: 0 5px 10px 0 $gray-5; +$typeahead-selected-bg: lighten($blue, 25%); +$typeahead-selected-color: $blue-dark; + // Dropdowns // ------------------------- $dropdownBackground: $white; diff --git a/public/sass/pages/_explore.scss b/public/sass/pages/_explore.scss index b618b8eb5b7..7dacccf6a87 100644 --- a/public/sass/pages/_explore.scss +++ b/public/sass/pages/_explore.scss @@ -13,19 +13,18 @@ } } + // Push split button a bit .explore-first-button { margin-left: 15px; } - .explore-graph { - width: 100%; - height: 100%; - } - + // Graph panel needs a bit extra padding at top .panel-container { - padding: 10px 10px 5px 10px; + padding: $panel-padding; + padding-top: 10px; } + // Make sure wrap buttons around on small screens .navbar { flex-wrap: wrap; height: auto; @@ -34,19 +33,18 @@ .navbar-page-btn { margin-right: 1rem; + // Explore icon in header .fa { - position: relative; - top: -1px; - font-size: 19px; - line-height: 8px; + font-size: 100%; opacity: 0.75; - margin-right: 8px; + margin-right: 0.5em; } } + // Toggle mode .navbar-button.active { - color: #0083b3; - background-color: white; + color: $btn-active-text-color; + background-color: $btn-active-bg; } .elapsed-time { @@ -81,7 +79,7 @@ } .explore + .explore { - border-left: 1px dotted #aaa; + border-left: 1px dotted $table-border; } .query-row { @@ -97,8 +95,8 @@ } .query-field { - font-size: 14px; - font-family: Consolas, Menlo, Courier, monospace; + font-size: $font-size-root; + font-family: $font-family-monospace; height: auto; } @@ -108,12 +106,12 @@ padding: 6px 7px 4px; width: 100%; cursor: text; - line-height: 1.5; - color: rgba(0, 0, 0, 0.65); + line-height: $line-height-base; + color: $text-color-weak; background-color: $panel-bg; background-image: none; border: $panel-border; - border-radius: 3px; + border-radius: $border-radius; transition: all 0.3s; } @@ -124,38 +122,36 @@ top: -10000px; left: -10000px; opacity: 0; - border-radius: 4px; + border-radius: $border-radius; transition: opacity 0.75s; - border: 1px solid #e4e4e4; + border: $panel-border; max-height: calc(66vh); overflow-y: scroll; max-width: calc(66%); overflow-x: hidden; outline: none; list-style: none; - background: #fff; - color: rgba(0, 0, 0, 0.65); + background: $panel-bg; + color: $text-color; transition: opacity 0.4s ease-out; + box-shadow: $typeahead-shadow; } .typeahead-group__title { - color: rgba(0, 0, 0, 0.43); - font-size: 12px; - line-height: 1.5; - padding: 8px 16px; + color: $text-color-weak; + font-size: $font-size-sm; + line-height: $line-height-base; + padding: $input-padding-y $input-padding-x; } .typeahead-item { - line-height: 200%; height: auto; - font-family: Consolas, Menlo, Courier, monospace; - padding: 0 16px 0 28px; - font-size: 12px; + font-family: $font-family-monospace; + padding: $input-padding-y $input-padding-x; + padding-left: $input-padding-x-lg; + font-size: $font-size-sm; text-overflow: ellipsis; overflow: hidden; - margin-left: -1px; - left: 1px; - position: relative; z-index: 1; display: block; white-space: nowrap; @@ -165,129 +161,24 @@ } .typeahead-item__selected { - background-color: #ecf6fd; - color: #108ee9; + background-color: $typeahead-selected-bg; + color: $typeahead-selected-color; } } /* SYNTAX */ -/** - * prism.js Coy theme for JavaScript, CoffeeScript, CSS and HTML - * Based on https://github.com/tshedor/workshop-wp-theme (Example: http://workshop.kansan.com/category/sessions/basics or http://workshop.timshedor.com/category/sessions/basics); - * @author Tim Shedor - */ - .explore { - code[class*='language-'], - pre[class*='language-'] { - color: black; - background: none; - font-family: Consolas, Monaco, 'Andale Mono', 'Ubuntu Mono', monospace; - text-align: left; - white-space: pre; - word-spacing: normal; - word-break: normal; - word-wrap: normal; - line-height: 1.5; - - -moz-tab-size: 4; - -o-tab-size: 4; - tab-size: 4; - - -webkit-hyphens: none; - -moz-hyphens: none; - -ms-hyphens: none; - hyphens: none; - } - - /* Code blocks */ - pre[class*='language-'] { - position: relative; - margin: 0.5em 0; - overflow: visible; - padding: 0; - } - pre[class*='language-'] > code { - position: relative; - border-left: 10px solid #358ccb; - box-shadow: -1px 0px 0px 0px #358ccb, 0px 0px 0px 1px #dfdfdf; - background-color: #fdfdfd; - background-image: linear-gradient(transparent 50%, rgba(69, 142, 209, 0.04) 50%); - background-size: 3em 3em; - background-origin: content-box; - background-attachment: local; - } - - code[class*='language'] { - max-height: inherit; - height: inherit; - padding: 0 1em; - display: block; - overflow: auto; - } - - /* Margin bottom to accomodate shadow */ - :not(pre) > code[class*='language-'], - pre[class*='language-'] { - background-color: #fdfdfd; - -webkit-box-sizing: border-box; - -moz-box-sizing: border-box; - box-sizing: border-box; - margin-bottom: 1em; - } - - /* Inline code */ - :not(pre) > code[class*='language-'] { - position: relative; - padding: 0.2em; - border-radius: 0.3em; - color: #c92c2c; - border: 1px solid rgba(0, 0, 0, 0.1); - display: inline; - white-space: normal; - } - - pre[class*='language-']:before, - pre[class*='language-']:after { - content: ''; - z-index: -2; - display: block; - position: absolute; - bottom: 0.75em; - left: 0.18em; - width: 40%; - height: 20%; - max-height: 13em; - box-shadow: 0px 13px 8px #979797; - -webkit-transform: rotate(-2deg); - -moz-transform: rotate(-2deg); - -ms-transform: rotate(-2deg); - -o-transform: rotate(-2deg); - transform: rotate(-2deg); - } - - :not(pre) > code[class*='language-']:after, - pre[class*='language-']:after { - right: 0.75em; - left: auto; - -webkit-transform: rotate(2deg); - -moz-transform: rotate(2deg); - -ms-transform: rotate(2deg); - -o-transform: rotate(2deg); - transform: rotate(2deg); - } - .token.comment, .token.block-comment, .token.prolog, .token.doctype, .token.cdata { - color: #7d8b99; + color: $text-color-weak; } .token.punctuation { - color: #5f6364; + color: $text-color-weak; } .token.property, @@ -298,7 +189,7 @@ .token.constant, .token.symbol, .token.deleted { - color: #c92c2c; + color: $query-red; } .token.selector, @@ -308,33 +199,26 @@ .token.function, .token.builtin, .token.inserted { - color: #2f9c0a; + color: $query-green; } .token.operator, .token.entity, .token.url, .token.variable { - color: #a67f59; - background: rgba(255, 255, 255, 0.5); + color: $query-purple; } .token.atrule, .token.attr-value, .token.keyword, .token.class-name { - color: #1990b8; + color: $query-blue; } .token.regex, .token.important { - color: #e90; - } - - .language-css .token.string, - .style .token.string { - color: #a67f59; - background: rgba(255, 255, 255, 0.5); + color: $query-orange; } .token.important { @@ -355,46 +239,4 @@ .namespace { opacity: 0.7; } - - @media screen and (max-width: 767px) { - pre[class*='language-']:before, - pre[class*='language-']:after { - bottom: 14px; - box-shadow: none; - } - } - - /* Plugin styles */ - .token.tab:not(:empty):before, - .token.cr:before, - .token.lf:before { - color: #e0d7d1; - } - - /* Plugin styles: Line Numbers */ - pre[class*='language-'].line-numbers { - padding-left: 0; - } - - pre[class*='language-'].line-numbers code { - padding-left: 3.8em; - } - - pre[class*='language-'].line-numbers .line-numbers-rows { - left: 0; - } - - /* Plugin styles: Line Highlight */ - pre[class*='language-'][data-line] { - padding-top: 0; - padding-bottom: 0; - padding-left: 0; - } - pre[data-line] code { - position: relative; - padding-left: 4em; - } - pre .line-highlight { - margin-top: 0; - } } From a6a12d36d740e282309bee96070ed38692399020 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Fri, 25 May 2018 13:32:55 +0200 Subject: [PATCH 28/87] add tests for sending usage stats --- pkg/metrics/metrics.go | 4 +- pkg/metrics/metrics_test.go | 154 ++++++++++++++++++++++++++++++++++++ 2 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 pkg/metrics/metrics_test.go diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 83505826910..fbab8af51dd 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -332,6 +332,8 @@ func updateTotalStats() { M_StatTotal_Orgs.Set(float64(statsQuery.Result.Orgs)) } +var usageStatsURL = "https://stats.grafana.org/grafana-usage-report" + func sendUsageStats() { if !setting.ReportingEnabled { return @@ -390,5 +392,5 @@ func sendUsageStats() { data := bytes.NewBuffer(out) client := http.Client{Timeout: 5 * time.Second} - go client.Post("https://stats.grafana.org/grafana-usage-report", "application/json", data) + go client.Post(usageStatsURL, "application/json", data) } diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go new file mode 100644 index 00000000000..a27e44d7105 --- /dev/null +++ b/pkg/metrics/metrics_test.go @@ -0,0 +1,154 @@ +package metrics + +import ( + "bytes" + "io/ioutil" + "runtime" + "sync" + "testing" + "time" + + "net/http" + "net/http/httptest" + + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/setting" + . "github.com/smartystreets/goconvey/convey" +) + +func TestMetrics(t *testing.T) { + Convey("Test send usage stats", t, func() { + var getSystemStatsQuery *models.GetSystemStatsQuery + bus.AddHandler("test", func(query *models.GetSystemStatsQuery) error { + query.Result = &models.SystemStats{ + Dashboards: 1, + Datasources: 2, + Users: 3, + ActiveUsers: 4, + Orgs: 5, + Playlists: 6, + Alerts: 7, + Stars: 8, + } + getSystemStatsQuery = query + return nil + }) + + var getDataSourceStatsQuery *models.GetDataSourceStatsQuery + bus.AddHandler("test", func(query *models.GetDataSourceStatsQuery) error { + query.Result = []*models.DataSourceStats{ + { + Type: models.DS_ES, + Count: 9, + }, + { + Type: models.DS_PROMETHEUS, + Count: 10, + }, + { + Type: "unknown_ds", + Count: 11, + }, + { + Type: "unknown_ds2", + Count: 12, + }, + } + getDataSourceStatsQuery = query + return nil + }) + + var wg sync.WaitGroup + var responseBuffer *bytes.Buffer + var req *http.Request + ts := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + req = r + buf, err := ioutil.ReadAll(r.Body) + if err != nil { + t.Fatalf("Failed to read response body, err=%v", err) + } + responseBuffer = bytes.NewBuffer(buf) + wg.Done() + })) + usageStatsURL = ts.URL + + sendUsageStats() + + Convey("Given reporting not enabled and sending usage stats", func() { + setting.ReportingEnabled = false + sendUsageStats() + + Convey("Should not gather stats or call http endpoint", func() { + So(getSystemStatsQuery, ShouldBeNil) + So(getDataSourceStatsQuery, ShouldBeNil) + So(req, ShouldBeNil) + }) + }) + + Convey("Given reporting enabled and sending usage stats", func() { + setting.ReportingEnabled = true + setting.BuildVersion = "5.0.0" + wg.Add(1) + sendUsageStats() + + Convey("Should gather stats and call http endpoint", func() { + if waitTimeout(&wg, 2*time.Second) { + t.Fatalf("Timed out waiting for http request") + } + + So(getSystemStatsQuery, ShouldNotBeNil) + So(getDataSourceStatsQuery, ShouldNotBeNil) + So(req, ShouldNotBeNil) + So(req.Method, ShouldEqual, http.MethodPost) + So(req.Header.Get("Content-Type"), ShouldEqual, "application/json") + + So(responseBuffer, ShouldNotBeNil) + + j, err := simplejson.NewFromReader(responseBuffer) + So(err, ShouldBeNil) + + So(j.Get("version").MustString(), ShouldEqual, "5_0_0") + So(j.Get("os").MustString(), ShouldEqual, runtime.GOOS) + So(j.Get("arch").MustString(), ShouldEqual, runtime.GOARCH) + + metrics := j.Get("metrics") + So(metrics.Get("stats.dashboards.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Dashboards) + So(metrics.Get("stats.users.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Users) + So(metrics.Get("stats.orgs.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Orgs) + So(metrics.Get("stats.playlist.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Playlists) + So(metrics.Get("stats.plugins.apps.count").MustInt(), ShouldEqual, len(plugins.Apps)) + So(metrics.Get("stats.plugins.panels.count").MustInt(), ShouldEqual, len(plugins.Panels)) + So(metrics.Get("stats.plugins.datasources.count").MustInt(), ShouldEqual, len(plugins.DataSources)) + So(metrics.Get("stats.alerts.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Alerts) + So(metrics.Get("stats.active_users.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.ActiveUsers) + So(metrics.Get("stats.datasources.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Datasources) + So(metrics.Get("stats.stars.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Stars) + + So(metrics.Get("stats.ds."+models.DS_ES+".count").MustInt(), ShouldEqual, 9) + So(metrics.Get("stats.ds."+models.DS_PROMETHEUS+".count").MustInt(), ShouldEqual, 10) + So(metrics.Get("stats.ds.other.count").MustInt(), ShouldEqual, 11+12) + }) + }) + + Reset(func() { + ts.Close() + }) + }) +} + +func waitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool { + c := make(chan struct{}) + go func() { + defer close(c) + wg.Wait() + }() + select { + case <-c: + return false // completed normally + case <-time.After(timeout): + return true // timed out + } +} From 2ea5b6fe3371d3f956b77130fcebcc5a88b915c3 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Fri, 25 May 2018 14:33:37 +0200 Subject: [PATCH 29/87] add additional usage stats metrics nr of folders nr of folder permissions nr of dashboard permissions nr of snapshots nr of teams nr of provisioned dashboards --- pkg/metrics/metrics.go | 6 +++++ pkg/metrics/metrics_test.go | 28 ++++++++++++++------ pkg/models/stats.go | 30 ++++++++++++++++------ pkg/services/sqlstore/sqlstore.go | 6 ++--- pkg/services/sqlstore/stats.go | 40 ++++++++++++++++++++++++++--- pkg/services/sqlstore/stats_test.go | 27 +++++++++++++++++++ 6 files changed, 114 insertions(+), 23 deletions(-) create mode 100644 pkg/services/sqlstore/stats_test.go diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index fbab8af51dd..03836abe2ad 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -368,6 +368,12 @@ func sendUsageStats() { metrics["stats.active_users.count"] = statsQuery.Result.ActiveUsers metrics["stats.datasources.count"] = statsQuery.Result.Datasources metrics["stats.stars.count"] = statsQuery.Result.Stars + metrics["stats.folders.count"] = statsQuery.Result.Folders + metrics["stats.dashboard_permissions.count"] = statsQuery.Result.DashboardPermissions + metrics["stats.folder_permissions.count"] = statsQuery.Result.FolderPermissions + metrics["stats.provisioned_dashboards.count"] = statsQuery.Result.ProvisionedDashboards + metrics["stats.snapshots.count"] = statsQuery.Result.Snapshots + metrics["stats.teams.count"] = statsQuery.Result.Teams dsStats := models.GetDataSourceStatsQuery{} if err := bus.Dispatch(&dsStats); err != nil { diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go index a27e44d7105..77a0aef3f24 100644 --- a/pkg/metrics/metrics_test.go +++ b/pkg/metrics/metrics_test.go @@ -24,14 +24,20 @@ func TestMetrics(t *testing.T) { var getSystemStatsQuery *models.GetSystemStatsQuery bus.AddHandler("test", func(query *models.GetSystemStatsQuery) error { query.Result = &models.SystemStats{ - Dashboards: 1, - Datasources: 2, - Users: 3, - ActiveUsers: 4, - Orgs: 5, - Playlists: 6, - Alerts: 7, - Stars: 8, + Dashboards: 1, + Datasources: 2, + Users: 3, + ActiveUsers: 4, + Orgs: 5, + Playlists: 6, + Alerts: 7, + Stars: 8, + Folders: 9, + DashboardPermissions: 10, + FolderPermissions: 11, + ProvisionedDashboards: 12, + Snapshots: 13, + Teams: 14, } getSystemStatsQuery = query return nil @@ -126,6 +132,12 @@ func TestMetrics(t *testing.T) { So(metrics.Get("stats.active_users.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.ActiveUsers) So(metrics.Get("stats.datasources.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Datasources) So(metrics.Get("stats.stars.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Stars) + So(metrics.Get("stats.folders.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Folders) + So(metrics.Get("stats.dashboard_permissions.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.DashboardPermissions) + So(metrics.Get("stats.folder_permissions.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.FolderPermissions) + So(metrics.Get("stats.provisioned_dashboards.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.ProvisionedDashboards) + So(metrics.Get("stats.snapshots.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Snapshots) + So(metrics.Get("stats.teams.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Teams) So(metrics.Get("stats.ds."+models.DS_ES+".count").MustInt(), ShouldEqual, 9) So(metrics.Get("stats.ds."+models.DS_PROMETHEUS+".count").MustInt(), ShouldEqual, 10) diff --git a/pkg/models/stats.go b/pkg/models/stats.go index e132d88c030..0e497621e14 100644 --- a/pkg/models/stats.go +++ b/pkg/models/stats.go @@ -1,14 +1,20 @@ package models type SystemStats struct { - Dashboards int64 - Datasources int64 - Users int64 - ActiveUsers int64 - Orgs int64 - Playlists int64 - Alerts int64 - Stars int64 + Dashboards int64 + Datasources int64 + Users int64 + ActiveUsers int64 + Orgs int64 + Playlists int64 + Alerts int64 + Stars int64 + Snapshots int64 + Teams int64 + DashboardPermissions int64 + FolderPermissions int64 + Folders int64 + ProvisionedDashboards int64 } type DataSourceStats struct { @@ -40,3 +46,11 @@ type AdminStats struct { type GetAdminStatsQuery struct { Result *AdminStats } + +type SystemUserCountStats struct { + Count int64 +} + +type GetSystemUserCountStatsQuery struct { + Result *SystemUserCountStats +} diff --git a/pkg/services/sqlstore/sqlstore.go b/pkg/services/sqlstore/sqlstore.go index 39aa2cb7ead..6af56f2f169 100644 --- a/pkg/services/sqlstore/sqlstore.go +++ b/pkg/services/sqlstore/sqlstore.go @@ -86,13 +86,13 @@ func (ss *SqlStore) Init() error { } func (ss *SqlStore) ensureAdminUser() error { - statsQuery := m.GetSystemStatsQuery{} + systemUserCountQuery := m.GetSystemUserCountStatsQuery{} - if err := bus.Dispatch(&statsQuery); err != nil { + if err := bus.Dispatch(&systemUserCountQuery); err != nil { fmt.Errorf("Could not determine if admin user exists: %v", err) } - if statsQuery.Result.Users > 0 { + if systemUserCountQuery.Result.Count > 0 { return nil } diff --git a/pkg/services/sqlstore/stats.go b/pkg/services/sqlstore/stats.go index 173a1e56634..e0bc0e2091e 100644 --- a/pkg/services/sqlstore/stats.go +++ b/pkg/services/sqlstore/stats.go @@ -11,6 +11,7 @@ func init() { bus.AddHandler("sql", GetSystemStats) bus.AddHandler("sql", GetDataSourceStats) bus.AddHandler("sql", GetAdminStats) + bus.AddHandler("sql", GetSystemUserCountStats) } var activeUserTimeLimit = time.Hour * 24 * 30 @@ -51,14 +52,32 @@ func GetSystemStats(query *m.GetSystemStatsQuery) error { SELECT COUNT(*) FROM ` + dialect.Quote("alert") + ` ) AS alerts, - ( - SELECT COUNT(*) FROM ` + dialect.Quote("user") + ` where last_seen_at > ? - ) as active_users + ( + SELECT COUNT(*) FROM ` + dialect.Quote("user") + ` where last_seen_at > ? + ) as active_users, + ( + SELECT COUNT(id) FROM ` + dialect.Quote("dashboard") + ` where is_folder = ? + ) as folders, + ( + SELECT COUNT(acl.id) FROM ` + dialect.Quote("dashboard_acl") + ` as acl inner join ` + dialect.Quote("dashboard") + ` as d on d.id = acl.dashboard_id where d.is_folder = ? + ) as dashboard_permissions, + ( + SELECT COUNT(acl.id) FROM ` + dialect.Quote("dashboard_acl") + ` as acl inner join ` + dialect.Quote("dashboard") + ` as d on d.id = acl.dashboard_id where d.is_folder = ? + ) as folder_permissions, + ( + SELECT COUNT(id) FROM ` + dialect.Quote("dashboard_provisioning") + ` + ) as provisioned_dashboards, + ( + SELECT COUNT(id) FROM ` + dialect.Quote("dashboard_snapshot") + ` + ) as snapshots, + ( + SELECT COUNT(id) FROM ` + dialect.Quote("team") + ` + ) as teams ` activeUserDeadlineDate := time.Now().Add(-activeUserTimeLimit) var stats m.SystemStats - _, err := x.SQL(rawSql, activeUserDeadlineDate).Get(&stats) + _, err := x.SQL(rawSql, activeUserDeadlineDate, dialect.BooleanStr(true), dialect.BooleanStr(false), dialect.BooleanStr(true)).Get(&stats) if err != nil { return err } @@ -122,3 +141,16 @@ func GetAdminStats(query *m.GetAdminStatsQuery) error { query.Result = &stats return err } + +func GetSystemUserCountStats(query *m.GetSystemUserCountStatsQuery) error { + var rawSql = `SELECT COUNT(id) AS Count FROM ` + dialect.Quote("user") + var stats m.SystemUserCountStats + _, err := x.SQL(rawSql).Get(&stats) + if err != nil { + return err + } + + query.Result = &stats + + return err +} diff --git a/pkg/services/sqlstore/stats_test.go b/pkg/services/sqlstore/stats_test.go new file mode 100644 index 00000000000..c98556a68d3 --- /dev/null +++ b/pkg/services/sqlstore/stats_test.go @@ -0,0 +1,27 @@ +package sqlstore + +import ( + "testing" + + m "github.com/grafana/grafana/pkg/models" + . "github.com/smartystreets/goconvey/convey" +) + +func TestStatsDataAccess(t *testing.T) { + + Convey("Testing Stats Data Access", t, func() { + InitTestDB(t) + + Convey("Get system stats should not results in error", func() { + query := m.GetSystemStatsQuery{} + err := GetSystemStats(&query) + So(err, ShouldBeNil) + }) + + Convey("Get system user count stats should not results in error", func() { + query := m.GetSystemUserCountStatsQuery{} + err := GetSystemUserCountStats(&query) + So(err, ShouldBeNil) + }) + }) +} From 27e7a28b37032ce7872a58a83b265775feebeaa4 Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Wed, 23 May 2018 17:16:08 +0200 Subject: [PATCH 30/87] Review feedback (heading, typos) * iff and therefor * mention merge in heading * add note about checking query inspector --- docs/sources/features/panels/table_panel.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/sources/features/panels/table_panel.md b/docs/sources/features/panels/table_panel.md index ed2632f29d6..2cbb601820e 100644 --- a/docs/sources/features/panels/table_panel.md +++ b/docs/sources/features/panels/table_panel.md @@ -25,7 +25,7 @@ The table panel displays the results of a query specified in the **Metrics** tab The result being displayed depends on the datasource and the query, but generally there is one row per datapoint, with extra columns for associated keys and values, as well as one column for the numeric value of the datapoint. You can change the behavior in the section **Data to Table** below. -### Multiple Queries per Table +### Merge Multiple Queries per Table > Only available in Grafana v5.0+. @@ -36,7 +36,7 @@ In this example usage and capacity are metrics that will have corresponding data In its simplest case, both queries return time-series data with a numeric value and a timestamp. If the timestamps are the same, datapoints will be matched and rendered on the same row. Some datasources return keys and values (labels, tags) associated with the datapoint. -These are being matched as well iff they are present in both results and have the same value. +These are being matched as well if they are present in both results and have the same value. The following datapoints will end up on the same row with one time column, two label columns ("host" and "job") and two value columns: ``` @@ -57,7 +57,9 @@ Datapoint for query B: {time: 1, host: "node-9", value: 4} ``` You can still merge both of the above cases by changing the conflicting column's **Type** to **hidden** in the **Column Styles**. -Note that if each datapoint of your query results have multiple value fields like max, min, mean, etc., they will likely have different values and therefor will not match and render on separate rows. + +Note that if each datapoint of your query results have multiple value fields like max, min, mean, etc., they will likely have different values and therefore will not match and render on separate rows. +If you intend for rows to be merged but see them rendered on separate rows, check the query results in the **Query Inspector** for field values being identical across datapoints that should be merged into a row. ## Options overview From fbc44025dc2e8579a82edcde513280e872c38132 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Fri, 25 May 2018 16:00:15 +0200 Subject: [PATCH 31/87] add usage stats for datasource access mode --- pkg/metrics/metrics.go | 29 +++++++++++++++ pkg/metrics/metrics_test.go | 56 +++++++++++++++++++++++++++++ pkg/models/stats.go | 10 ++++++ pkg/services/sqlstore/stats.go | 8 +++++ pkg/services/sqlstore/stats_test.go | 12 +++++++ 5 files changed, 115 insertions(+) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 03836abe2ad..3d3cfc2e1b6 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -394,6 +394,35 @@ func sendUsageStats() { } metrics["stats.ds.other.count"] = dsOtherCount + dsAccessStats := models.GetDataSourceAccessStatsQuery{} + if err := bus.Dispatch(&dsAccessStats); err != nil { + metricsLogger.Error("Failed to get datasource access stats", "error", err) + return + } + + // send access counters for each data source + // but ignore any custom data sources + // as sending that name could be sensitive information + dsAccessOtherCount := make(map[string]int64) + for _, dsAccessStat := range dsAccessStats.Result { + if dsAccessStat.Access == "" { + continue + } + + access := strings.ToLower(dsAccessStat.Access) + + if models.IsKnownDataSourcePlugin(dsAccessStat.Type) { + metrics["stats.ds_access."+dsAccessStat.Type+"."+access+".count"] = dsAccessStat.Count + } else { + old := dsAccessOtherCount[access] + dsAccessOtherCount[access] = old + dsAccessStat.Count + } + } + + for access, count := range dsAccessOtherCount { + metrics["stats.ds_access.other."+access+".count"] = count + } + out, _ := json.MarshalIndent(report, "", " ") data := bytes.NewBuffer(out) diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go index 77a0aef3f24..8d88e03d106 100644 --- a/pkg/metrics/metrics_test.go +++ b/pkg/metrics/metrics_test.go @@ -67,6 +67,54 @@ func TestMetrics(t *testing.T) { return nil }) + var getDataSourceAccessStatsQuery *models.GetDataSourceAccessStatsQuery + bus.AddHandler("test", func(query *models.GetDataSourceAccessStatsQuery) error { + query.Result = []*models.DataSourceAccessStats{ + { + Type: models.DS_ES, + Access: "direct", + Count: 1, + }, + { + Type: models.DS_ES, + Access: "proxy", + Count: 2, + }, + { + Type: models.DS_PROMETHEUS, + Access: "proxy", + Count: 3, + }, + { + Type: "unknown_ds", + Access: "proxy", + Count: 4, + }, + { + Type: "unknown_ds2", + Access: "", + Count: 5, + }, + { + Type: "unknown_ds3", + Access: "direct", + Count: 6, + }, + { + Type: "unknown_ds4", + Access: "direct", + Count: 7, + }, + { + Type: "unknown_ds5", + Access: "proxy", + Count: 8, + }, + } + getDataSourceAccessStatsQuery = query + return nil + }) + var wg sync.WaitGroup var responseBuffer *bytes.Buffer var req *http.Request @@ -90,6 +138,7 @@ func TestMetrics(t *testing.T) { Convey("Should not gather stats or call http endpoint", func() { So(getSystemStatsQuery, ShouldBeNil) So(getDataSourceStatsQuery, ShouldBeNil) + So(getDataSourceAccessStatsQuery, ShouldBeNil) So(req, ShouldBeNil) }) }) @@ -107,6 +156,7 @@ func TestMetrics(t *testing.T) { So(getSystemStatsQuery, ShouldNotBeNil) So(getDataSourceStatsQuery, ShouldNotBeNil) + So(getDataSourceAccessStatsQuery, ShouldNotBeNil) So(req, ShouldNotBeNil) So(req.Method, ShouldEqual, http.MethodPost) So(req.Header.Get("Content-Type"), ShouldEqual, "application/json") @@ -142,6 +192,12 @@ func TestMetrics(t *testing.T) { So(metrics.Get("stats.ds."+models.DS_ES+".count").MustInt(), ShouldEqual, 9) So(metrics.Get("stats.ds."+models.DS_PROMETHEUS+".count").MustInt(), ShouldEqual, 10) So(metrics.Get("stats.ds.other.count").MustInt(), ShouldEqual, 11+12) + + So(metrics.Get("stats.ds_access."+models.DS_ES+".direct.count").MustInt(), ShouldEqual, 1) + So(metrics.Get("stats.ds_access."+models.DS_ES+".proxy.count").MustInt(), ShouldEqual, 2) + So(metrics.Get("stats.ds_access."+models.DS_PROMETHEUS+".proxy.count").MustInt(), ShouldEqual, 3) + So(metrics.Get("stats.ds_access.other.direct.count").MustInt(), ShouldEqual, 6+7) + So(metrics.Get("stats.ds_access.other.proxy.count").MustInt(), ShouldEqual, 4+8) }) }) diff --git a/pkg/models/stats.go b/pkg/models/stats.go index 0e497621e14..4cd50d37463 100644 --- a/pkg/models/stats.go +++ b/pkg/models/stats.go @@ -30,6 +30,16 @@ type GetDataSourceStatsQuery struct { Result []*DataSourceStats } +type DataSourceAccessStats struct { + Type string + Access string + Count int64 +} + +type GetDataSourceAccessStatsQuery struct { + Result []*DataSourceAccessStats +} + type AdminStats struct { Users int `json:"users"` Orgs int `json:"orgs"` diff --git a/pkg/services/sqlstore/stats.go b/pkg/services/sqlstore/stats.go index e0bc0e2091e..5634e8feb52 100644 --- a/pkg/services/sqlstore/stats.go +++ b/pkg/services/sqlstore/stats.go @@ -10,6 +10,7 @@ import ( func init() { bus.AddHandler("sql", GetSystemStats) bus.AddHandler("sql", GetDataSourceStats) + bus.AddHandler("sql", GetDataSourceAccessStats) bus.AddHandler("sql", GetAdminStats) bus.AddHandler("sql", GetSystemUserCountStats) } @@ -23,6 +24,13 @@ func GetDataSourceStats(query *m.GetDataSourceStatsQuery) error { return err } +func GetDataSourceAccessStats(query *m.GetDataSourceAccessStatsQuery) error { + var rawSql = `SELECT COUNT(*) as count, type, access FROM data_source GROUP BY type, access` + query.Result = make([]*m.DataSourceAccessStats, 0) + err := x.SQL(rawSql).Find(&query.Result) + return err +} + func GetSystemStats(query *m.GetSystemStatsQuery) error { var rawSql = `SELECT ( diff --git a/pkg/services/sqlstore/stats_test.go b/pkg/services/sqlstore/stats_test.go index c98556a68d3..97f0ca0c43e 100644 --- a/pkg/services/sqlstore/stats_test.go +++ b/pkg/services/sqlstore/stats_test.go @@ -23,5 +23,17 @@ func TestStatsDataAccess(t *testing.T) { err := GetSystemUserCountStats(&query) So(err, ShouldBeNil) }) + + Convey("Get datasource stats should not results in error", func() { + query := m.GetDataSourceStatsQuery{} + err := GetDataSourceStats(&query) + So(err, ShouldBeNil) + }) + + Convey("Get datasource access stats should not results in error", func() { + query := m.GetDataSourceAccessStatsQuery{} + err := GetDataSourceAccessStats(&query) + So(err, ShouldBeNil) + }) }) } From 750ea9bbdd002ab959d255a7690853298ef26e49 Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Fri, 25 May 2018 16:46:38 +0200 Subject: [PATCH 32/87] Changed Prometheus interval-alignment to cover whole panel range * the existing query date alignment shifts the range forward to match a multiple epoch of the interval, but keeps the range length the same, the result is that the start date is shifted forward as well, leaving a gap in the graph (or a zero-line when null-as-zero was set, issue #12024) * this pr extends the aligned range to cover the original start date as well --- .../datasource/prometheus/datasource.ts | 20 +++++----- .../prometheus/specs/datasource.jest.ts | 25 +++++++++++- .../prometheus/specs/datasource_specs.ts | 39 ++++--------------- 3 files changed, 42 insertions(+), 42 deletions(-) diff --git a/public/app/plugins/datasource/prometheus/datasource.ts b/public/app/plugins/datasource/prometheus/datasource.ts index a52f3aefa2e..ff7dc1b150c 100644 --- a/public/app/plugins/datasource/prometheus/datasource.ts +++ b/public/app/plugins/datasource/prometheus/datasource.ts @@ -7,6 +7,15 @@ import PrometheusMetricFindQuery from './metric_find_query'; import { ResultTransformer } from './result_transformer'; import { BackendSrv } from 'app/core/services/backend_srv'; +export function alignRange(start, end, step) { + const alignedEnd = Math.ceil(end / step) * step; + const alignedStart = Math.floor(start / step) * step; + return { + end: alignedEnd, + start: alignedStart, + }; +} + export function prometheusRegularEscape(value) { return value.replace(/'/g, "\\\\'"); } @@ -109,15 +118,6 @@ export class PrometheusDatasource { return this.templateSrv.variableExists(target.expr); } - clampRange(start, end, step) { - const clampedEnd = Math.ceil(end / step) * step; - const clampedRange = Math.floor((end - start) / step) * step; - return { - end: clampedEnd, - start: clampedEnd - clampedRange, - }; - } - query(options) { var start = this.getPrometheusTime(options.range.from, false); var end = this.getPrometheusTime(options.range.to, true); @@ -205,7 +205,7 @@ export class PrometheusDatasource { query.requestId = options.panelId + target.refId; // Align query interval with step - const adjusted = this.clampRange(start, end, query.step); + const adjusted = alignRange(start, end, query.step); query.start = adjusted.start; query.end = adjusted.end; diff --git a/public/app/plugins/datasource/prometheus/specs/datasource.jest.ts b/public/app/plugins/datasource/prometheus/specs/datasource.jest.ts index 2ab2895d731..0157322da58 100644 --- a/public/app/plugins/datasource/prometheus/specs/datasource.jest.ts +++ b/public/app/plugins/datasource/prometheus/specs/datasource.jest.ts @@ -1,7 +1,7 @@ import _ from 'lodash'; import moment from 'moment'; import q from 'q'; -import { PrometheusDatasource, prometheusSpecialRegexEscape, prometheusRegularEscape } from '../datasource'; +import { alignRange, PrometheusDatasource, prometheusSpecialRegexEscape, prometheusRegularEscape } from '../datasource'; describe('PrometheusDatasource', () => { let ctx: any = {}; @@ -142,6 +142,29 @@ describe('PrometheusDatasource', () => { }); }); + describe('alignRange', function() { + it('does not modify already aligned intervals with perfect step', function() { + const range = alignRange(0, 3, 3); + expect(range.start).toEqual(0); + expect(range.end).toEqual(3); + }); + it('does modify end-aligned intervals to reflect number of steps possible', function() { + const range = alignRange(1, 6, 3); + expect(range.start).toEqual(0); + expect(range.end).toEqual(6); + }); + it('does align intervals that are a multiple of steps', function() { + const range = alignRange(1, 4, 3); + expect(range.start).toEqual(0); + expect(range.end).toEqual(6); + }); + it('does align intervals that are not a multiple of steps', function() { + const range = alignRange(1, 5, 3); + expect(range.start).toEqual(0); + expect(range.end).toEqual(6); + }); + }); + describe('Prometheus regular escaping', function() { it('should not escape simple string', function() { expect(prometheusRegularEscape('cryptodepression')).toEqual('cryptodepression'); diff --git a/public/app/plugins/datasource/prometheus/specs/datasource_specs.ts b/public/app/plugins/datasource/prometheus/specs/datasource_specs.ts index ef51ff69206..c5da671b757 100644 --- a/public/app/plugins/datasource/prometheus/specs/datasource_specs.ts +++ b/public/app/plugins/datasource/prometheus/specs/datasource_specs.ts @@ -44,7 +44,7 @@ describe('PrometheusDatasource', function() { }; // Interval alignment with step var urlExpected = - 'proxied/api/v1/query_range?query=' + encodeURIComponent('test{job="testjob"}') + '&start=120&end=240&step=60'; + 'proxied/api/v1/query_range?query=' + encodeURIComponent('test{job="testjob"}') + '&start=60&end=240&step=60'; var response = { status: 'success', data: { @@ -181,7 +181,7 @@ describe('PrometheusDatasource', function() { var urlExpected = 'proxied/api/v1/query_range?query=' + encodeURIComponent('ALERTS{alertstate="firing"}') + - '&start=120&end=180&step=60'; + '&start=60&end=180&step=60'; var options = { annotation: { expr: 'ALERTS{alertstate="firing"}', @@ -348,7 +348,7 @@ describe('PrometheusDatasource', function() { interval: '5s', }; // times get rounded up to interval - var urlExpected = 'proxied/api/v1/query_range?query=test&start=100&end=450&step=50'; + var urlExpected = 'proxied/api/v1/query_range?query=test&start=50&end=450&step=50'; ctx.$httpBackend.expect('GET', urlExpected).respond(response); ctx.ds.query(query); ctx.$httpBackend.verifyNoOutstandingExpectation(); @@ -384,8 +384,8 @@ describe('PrometheusDatasource', function() { ], interval: '10s', }; - // times get rounded up to interval - var urlExpected = 'proxied/api/v1/query_range?query=test' + '&start=200&end=500&step=100'; + // times get aligned to interval + var urlExpected = 'proxied/api/v1/query_range?query=test' + '&start=0&end=500&step=100'; ctx.$httpBackend.expect('GET', urlExpected).respond(response); ctx.ds.query(query); ctx.$httpBackend.verifyNoOutstandingExpectation(); @@ -511,7 +511,7 @@ describe('PrometheusDatasource', function() { }, }; var urlExpected = - 'proxied/api/v1/query_range?query=' + encodeURIComponent('rate(test[100s])') + '&start=200&end=500&step=100'; + 'proxied/api/v1/query_range?query=' + encodeURIComponent('rate(test[100s])') + '&start=0&end=500&step=100'; ctx.$httpBackend.expect('GET', urlExpected).respond(response); ctx.ds.query(query); ctx.$httpBackend.verifyNoOutstandingExpectation(); @@ -539,7 +539,7 @@ describe('PrometheusDatasource', function() { }, }; var urlExpected = - 'proxied/api/v1/query_range?query=' + encodeURIComponent('rate(test[50s])') + '&start=100&end=450&step=50'; + 'proxied/api/v1/query_range?query=' + encodeURIComponent('rate(test[50s])') + '&start=50&end=450&step=50'; ctx.$httpBackend.expect('GET', urlExpected).respond(response); ctx.ds.query(query); ctx.$httpBackend.verifyNoOutstandingExpectation(); @@ -613,29 +613,6 @@ describe('PrometheusDatasource', function() { expect(query.scopedVars.__interval_ms.value).to.be(5 * 1000); }); }); - - describe('Step alignment of intervals', function() { - it('does not modify already aligned intervals with perfect step', function() { - const range = ctx.ds.clampRange(0, 3, 3); - expect(range.start).to.be(0); - expect(range.end).to.be(3); - }); - it('does modify end-aligned intervals to reflect number of steps possible', function() { - const range = ctx.ds.clampRange(1, 6, 3); - expect(range.start).to.be(3); - expect(range.end).to.be(6); - }); - it('does align intervals that are a multiple of steps', function() { - const range = ctx.ds.clampRange(1, 4, 3); - expect(range.start).to.be(3); - expect(range.end).to.be(6); - }); - it('does align intervals that are not a multiple of steps', function() { - const range = ctx.ds.clampRange(1, 5, 3); - expect(range.start).to.be(3); - expect(range.end).to.be(6); - }); - }); }); describe('PrometheusDatasource for POST', function() { @@ -667,7 +644,7 @@ describe('PrometheusDatasource for POST', function() { var urlExpected = 'proxied/api/v1/query_range'; var dataExpected = $.param({ query: 'test{job="testjob"}', - start: 2 * 60, + start: 1 * 60, end: 3 * 60, step: 60, }); From a5e6cb9a02bf9910968d0aa918161651d3990065 Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Sun, 27 May 2018 14:52:50 +0200 Subject: [PATCH 33/87] Fix #9847 Add a generic signout_redirect_url to enable oauth logout Signed-off-by: Julien Pivotto --- conf/defaults.ini | 3 +++ conf/sample.ini | 3 +++ pkg/api/login.go | 6 +++++- pkg/setting/setting.go | 2 ++ 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/conf/defaults.ini b/conf/defaults.ini index d45e270d65d..4ca993038f9 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -237,6 +237,9 @@ disable_login_form = false # Set to true to disable the signout link in the side menu. useful if you use auth.proxy disable_signout_menu = false +# URL to redirect the user to after sign out +signout_redirect_url = + #################################### Anonymous Auth ###################### [auth.anonymous] # enable anonymous access diff --git a/conf/sample.ini b/conf/sample.ini index f12d917039d..45888cbadd8 100644 --- a/conf/sample.ini +++ b/conf/sample.ini @@ -217,6 +217,9 @@ log_queries = # Set to true to disable the signout link in the side menu. useful if you use auth.proxy, defaults to false ;disable_signout_menu = false +# URL to redirect the user to after sign out +;signout_redirect_url = + #################################### Anonymous Auth ########################## [auth.anonymous] # enable anonymous access diff --git a/pkg/api/login.go b/pkg/api/login.go index 9d0fa31946f..01fa71a6e44 100644 --- a/pkg/api/login.go +++ b/pkg/api/login.go @@ -155,5 +155,9 @@ func Logout(c *m.ReqContext) { c.SetCookie(setting.CookieUserName, "", -1, setting.AppSubUrl+"/") c.SetCookie(setting.CookieRememberName, "", -1, setting.AppSubUrl+"/") c.Session.Destory(c.Context) - c.Redirect(setting.AppSubUrl + "/login") + if setting.SignoutRedirectUrl != "" { + c.Redirect(setting.SignoutRedirectUrl) + } else { + c.Redirect(setting.AppSubUrl + "/login") + } } diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index 99ce39cc18e..8d086637e94 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -104,6 +104,7 @@ var ( DefaultTheme string DisableLoginForm bool DisableSignoutMenu bool + SignoutRedirectUrl string ExternalUserMngLinkUrl string ExternalUserMngLinkName string ExternalUserMngInfo string @@ -601,6 +602,7 @@ func (cfg *Cfg) Load(args *CommandLineArgs) error { auth := iniFile.Section("auth") DisableLoginForm = auth.Key("disable_login_form").MustBool(false) DisableSignoutMenu = auth.Key("disable_signout_menu").MustBool(false) + SignoutRedirectUrl = auth.Key("signout_redirect_url").String() // anonymous access AnonymousEnabled = iniFile.Section("auth.anonymous").Key("enabled").MustBool(false) From c9e9f25699dab81ff09e89c0236adb2e291f12da Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Mon, 28 May 2018 10:37:17 +0200 Subject: [PATCH 34/87] use sql builder for the get system stats sql query --- pkg/services/sqlstore/stats.go | 85 ++++++++++++++-------------------- 1 file changed, 34 insertions(+), 51 deletions(-) diff --git a/pkg/services/sqlstore/stats.go b/pkg/services/sqlstore/stats.go index 5634e8feb52..3e3e83c4014 100644 --- a/pkg/services/sqlstore/stats.go +++ b/pkg/services/sqlstore/stats.go @@ -32,60 +32,43 @@ func GetDataSourceAccessStats(query *m.GetDataSourceAccessStatsQuery) error { } func GetSystemStats(query *m.GetSystemStatsQuery) error { - var rawSql = `SELECT - ( - SELECT COUNT(*) - FROM ` + dialect.Quote("user") + ` - ) AS users, - ( - SELECT COUNT(*) - FROM ` + dialect.Quote("org") + ` - ) AS orgs, - ( - SELECT COUNT(*) - FROM ` + dialect.Quote("dashboard") + ` - ) AS dashboards, - ( - SELECT COUNT(*) - FROM ` + dialect.Quote("data_source") + ` - ) AS datasources, - ( - SELECT COUNT(*) FROM ` + dialect.Quote("star") + ` - ) AS stars, - ( - SELECT COUNT(*) - FROM ` + dialect.Quote("playlist") + ` - ) AS playlists, - ( - SELECT COUNT(*) - FROM ` + dialect.Quote("alert") + ` - ) AS alerts, - ( - SELECT COUNT(*) FROM ` + dialect.Quote("user") + ` where last_seen_at > ? - ) as active_users, - ( - SELECT COUNT(id) FROM ` + dialect.Quote("dashboard") + ` where is_folder = ? - ) as folders, - ( - SELECT COUNT(acl.id) FROM ` + dialect.Quote("dashboard_acl") + ` as acl inner join ` + dialect.Quote("dashboard") + ` as d on d.id = acl.dashboard_id where d.is_folder = ? - ) as dashboard_permissions, - ( - SELECT COUNT(acl.id) FROM ` + dialect.Quote("dashboard_acl") + ` as acl inner join ` + dialect.Quote("dashboard") + ` as d on d.id = acl.dashboard_id where d.is_folder = ? - ) as folder_permissions, - ( - SELECT COUNT(id) FROM ` + dialect.Quote("dashboard_provisioning") + ` - ) as provisioned_dashboards, - ( - SELECT COUNT(id) FROM ` + dialect.Quote("dashboard_snapshot") + ` - ) as snapshots, - ( - SELECT COUNT(id) FROM ` + dialect.Quote("team") + ` - ) as teams - ` + sb := &SqlBuilder{} + sb.Write("SELECT ") + sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("user") + `) AS users,`) + sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("org") + `) AS orgs,`) + sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("dashboard") + `) AS dashboards,`) + sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("data_source") + `) AS datasources,`) + sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("star") + `) AS stars,`) + sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("playlist") + `) AS playlists,`) + sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("alert") + `) AS alerts,`) activeUserDeadlineDate := time.Now().Add(-activeUserTimeLimit) + sb.Write(`(SELECT COUNT(*) FROM `+dialect.Quote("user")+` where last_seen_at > ?) AS active_users,`, activeUserDeadlineDate) + + sb.Write(`(SELECT COUNT(id) FROM `+dialect.Quote("dashboard")+` where is_folder = ?) AS folders,`, dialect.BooleanStr(true)) + + sb.Write(`( + SELECT COUNT(acl.id) + FROM `+dialect.Quote("dashboard_acl")+` as acl + inner join `+dialect.Quote("dashboard")+` as d + on d.id = acl.dashboard_id + WHERE d.is_folder = ? + ) AS dashboard_permissions,`, dialect.BooleanStr(false)) + + sb.Write(`( + SELECT COUNT(acl.id) + FROM `+dialect.Quote("dashboard_acl")+` as acl + inner join `+dialect.Quote("dashboard")+` as d + on d.id = acl.dashboard_id + WHERE d.is_folder = ? + ) AS folder_permissions,`, dialect.BooleanStr(true)) + + sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("dashboard_provisioning") + `) AS provisioned_dashboards,`) + sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("dashboard_snapshot") + `) AS snapshots,`) + sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("team") + `) AS teams`) + var stats m.SystemStats - _, err := x.SQL(rawSql, activeUserDeadlineDate, dialect.BooleanStr(true), dialect.BooleanStr(false), dialect.BooleanStr(true)).Get(&stats) + _, err := x.SQL(sb.GetSqlString(), sb.params...).Get(&stats) if err != nil { return err } From a2ed0b15da6622fcc51a52466552ca3d9b574c67 Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 28 May 2018 10:39:42 +0200 Subject: [PATCH 35/87] build: fixes broken path for bra run removes os and arch from binary path when building in dev mode --- build.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/build.go b/build.go index 35531faf2dd..3f92f8833a2 100644 --- a/build.go +++ b/build.go @@ -156,8 +156,8 @@ func makeLatestDistCopies() { } latestMapping := map[string]string{ - "_amd64.deb": "dist/grafana_latest_amd64.deb", - ".x86_64.rpm": "dist/grafana-latest-1.x86_64.rpm", + "_amd64.deb": "dist/grafana_latest_amd64.deb", + ".x86_64.rpm": "dist/grafana-latest-1.x86_64.rpm", ".linux-amd64.tar.gz": "dist/grafana-latest.linux-x64.tar.gz", } @@ -232,7 +232,7 @@ func createDebPackages() { previousPkgArch := pkgArch if pkgArch == "armv7" { pkgArch = "armhf" - } + } createPackage(linuxPackageOptions{ packageType: "deb", homeDir: "/usr/share/grafana", @@ -256,8 +256,10 @@ func createDebPackages() { func createRpmPackages() { previousPkgArch := pkgArch switch { - case pkgArch == "armv7" : pkgArch = "armhfp" - case pkgArch == "arm64" : pkgArch = "aarch64" + case pkgArch == "armv7": + pkgArch = "armhfp" + case pkgArch == "arm64": + pkgArch = "aarch64" } createPackage(linuxPackageOptions{ packageType: "rpm", @@ -416,6 +418,10 @@ func test(pkg string) { func build(binaryName, pkg string, tags []string) { binary := fmt.Sprintf("./bin/%s-%s/%s", goos, goarch, binaryName) + if isDev { + //dont include os and arch in output path in dev environment + binary = fmt.Sprintf("./bin/%s", binaryName) + } if goos == "windows" { binary += ".exe" From 5a96863eedb195d5792e43f9199c2c6d529af500 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Mon, 28 May 2018 13:06:27 +0200 Subject: [PATCH 36/87] pin versions of xorm to resolve sql tests Resolves issue with postgres tests. Also sets timezone of test instance and database to utc to resolve mysql tests. Closes #12065 --- Gopkg.lock | 10 +- Gopkg.toml | 4 +- pkg/services/sqlstore/sqlstore.go | 4 +- vendor/github.com/go-xorm/core/cache.go | 8 +- vendor/github.com/go-xorm/core/column.go | 16 +- vendor/github.com/go-xorm/core/db.go | 57 +- vendor/github.com/go-xorm/core/dialect.go | 7 +- vendor/github.com/go-xorm/core/filter.go | 6 +- vendor/github.com/go-xorm/core/index.go | 2 - vendor/github.com/go-xorm/core/rows.go | 64 +- vendor/github.com/go-xorm/core/scan.go | 3 - vendor/github.com/go-xorm/core/type.go | 36 +- .../github.com/go-xorm/xorm/dialect_mysql.go | 74 -- .../go-xorm/xorm/dialect_postgres.go | 96 +-- vendor/github.com/go-xorm/xorm/engine.go | 170 ++-- vendor/github.com/go-xorm/xorm/engine_cond.go | 5 +- .../github.com/go-xorm/xorm/engine_table.go | 113 --- vendor/github.com/go-xorm/xorm/error.go | 13 - vendor/github.com/go-xorm/xorm/helpers.go | 162 ++++ vendor/github.com/go-xorm/xorm/interface.go | 6 - vendor/github.com/go-xorm/xorm/rows.go | 6 +- vendor/github.com/go-xorm/xorm/session.go | 727 +++++++++--------- .../github.com/go-xorm/xorm/session_cols.go | 107 --- .../github.com/go-xorm/xorm/session_delete.go | 6 +- .../github.com/go-xorm/xorm/session_exist.go | 15 +- .../github.com/go-xorm/xorm/session_find.go | 49 +- vendor/github.com/go-xorm/xorm/session_get.go | 15 +- .../github.com/go-xorm/xorm/session_insert.go | 158 ++-- .../github.com/go-xorm/xorm/session_query.go | 18 +- .../github.com/go-xorm/xorm/session_schema.go | 87 ++- .../github.com/go-xorm/xorm/session_update.go | 115 +-- vendor/github.com/go-xorm/xorm/statement.go | 221 +++--- vendor/github.com/go-xorm/xorm/xorm.go | 10 +- 33 files changed, 1026 insertions(+), 1364 deletions(-) delete mode 100644 vendor/github.com/go-xorm/xorm/engine_table.go diff --git a/Gopkg.lock b/Gopkg.lock index 24d713bbdb7..0753ee66b51 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -186,14 +186,14 @@ [[projects]] name = "github.com/go-xorm/core" packages = ["."] - revision = "f43c33d9a48db006417a7ac4c16b08897e3e1458" - version = "v0.5.8" + revision = "da1adaf7a28ca792961721a34e6e04945200c890" + version = "v0.5.7" [[projects]] name = "github.com/go-xorm/xorm" packages = ["."] - revision = "fc1b13e0d8e240788213230aa5747eb557f80f41" - version = "v0.6.6" + revision = "1933dd69e294c0a26c0266637067f24dbb25770c" + version = "v0.6.4" [[projects]] branch = "master" @@ -670,6 +670,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "cdeb99713eda72e1ea84b5e6b110819785823cec9bc38b147efa0b86949ecff0" + inputs-digest = "6c7ae4bcbe7fa4430d3bdbf204df1b7c59cba88151fbcefa167ce15e6351b6d3" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 101be04efaa..0f51e8a6fa3 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -85,11 +85,11 @@ ignored = [ [[constraint]] name = "github.com/go-xorm/core" - version = "0.5.7" + version = "=0.5.7" [[constraint]] name = "github.com/go-xorm/xorm" - version = "0.6.4" + version = "=0.6.4" [[constraint]] name = "github.com/gorilla/websocket" diff --git a/pkg/services/sqlstore/sqlstore.go b/pkg/services/sqlstore/sqlstore.go index 6af56f2f169..28ba0eef374 100644 --- a/pkg/services/sqlstore/sqlstore.go +++ b/pkg/services/sqlstore/sqlstore.go @@ -277,8 +277,8 @@ func InitTestDB(t *testing.T) *SqlStore { t.Fatalf("Failed to init test database: %v", err) } - //// sqlstore.engine.DatabaseTZ = time.UTC - //// sqlstore.engine.TZLocation = time.UTC + sqlstore.engine.DatabaseTZ = time.UTC + sqlstore.engine.TZLocation = time.UTC return sqlstore } diff --git a/vendor/github.com/go-xorm/core/cache.go b/vendor/github.com/go-xorm/core/cache.go index 8f9531da940..bf81bd52ba4 100644 --- a/vendor/github.com/go-xorm/core/cache.go +++ b/vendor/github.com/go-xorm/core/cache.go @@ -1,12 +1,11 @@ package core import ( - "bytes" - "encoding/gob" "errors" "fmt" - "strings" "time" + "bytes" + "encoding/gob" ) const ( @@ -56,10 +55,11 @@ func encodeIds(ids []PK) (string, error) { return buf.String(), err } + func decodeIds(s string) ([]PK, error) { pks := make([]PK, 0) - dec := gob.NewDecoder(strings.NewReader(s)) + dec := gob.NewDecoder(bytes.NewBufferString(s)) err := dec.Decode(&pks) return pks, err diff --git a/vendor/github.com/go-xorm/core/column.go b/vendor/github.com/go-xorm/core/column.go index 65370bb5ba1..d9362e98578 100644 --- a/vendor/github.com/go-xorm/core/column.go +++ b/vendor/github.com/go-xorm/core/column.go @@ -79,10 +79,6 @@ func (col *Column) String(d Dialect) string { } } - if col.Default != "" { - sql += "DEFAULT " + col.Default + " " - } - if d.ShowCreateNull() { if col.Nullable { sql += "NULL " @@ -91,6 +87,10 @@ func (col *Column) String(d Dialect) string { } } + if col.Default != "" { + sql += "DEFAULT " + col.Default + " " + } + return sql } @@ -99,10 +99,6 @@ func (col *Column) StringNoPk(d Dialect) string { sql += d.SqlType(col) + " " - if col.Default != "" { - sql += "DEFAULT " + col.Default + " " - } - if d.ShowCreateNull() { if col.Nullable { sql += "NULL " @@ -111,6 +107,10 @@ func (col *Column) StringNoPk(d Dialect) string { } } + if col.Default != "" { + sql += "DEFAULT " + col.Default + " " + } + return sql } diff --git a/vendor/github.com/go-xorm/core/db.go b/vendor/github.com/go-xorm/core/db.go index 9969fa43134..6111c4b332f 100644 --- a/vendor/github.com/go-xorm/core/db.go +++ b/vendor/github.com/go-xorm/core/db.go @@ -7,11 +7,6 @@ import ( "fmt" "reflect" "regexp" - "sync" -) - -var ( - DefaultCacheSize = 200 ) func MapToSlice(query string, mp interface{}) (string, []interface{}, error) { @@ -63,16 +58,9 @@ func StructToSlice(query string, st interface{}) (string, []interface{}, error) return query, args, nil } -type cacheStruct struct { - value reflect.Value - idx int -} - type DB struct { *sql.DB - Mapper IMapper - reflectCache map[reflect.Type]*cacheStruct - reflectCacheMutex sync.RWMutex + Mapper IMapper } func Open(driverName, dataSourceName string) (*DB, error) { @@ -80,32 +68,11 @@ func Open(driverName, dataSourceName string) (*DB, error) { if err != nil { return nil, err } - return &DB{ - DB: db, - Mapper: NewCacheMapper(&SnakeMapper{}), - reflectCache: make(map[reflect.Type]*cacheStruct), - }, nil + return &DB{db, NewCacheMapper(&SnakeMapper{})}, nil } func FromDB(db *sql.DB) *DB { - return &DB{ - DB: db, - Mapper: NewCacheMapper(&SnakeMapper{}), - reflectCache: make(map[reflect.Type]*cacheStruct), - } -} - -func (db *DB) reflectNew(typ reflect.Type) reflect.Value { - db.reflectCacheMutex.Lock() - defer db.reflectCacheMutex.Unlock() - cs, ok := db.reflectCache[typ] - if !ok || cs.idx+1 > DefaultCacheSize-1 { - cs = &cacheStruct{reflect.MakeSlice(reflect.SliceOf(typ), DefaultCacheSize, DefaultCacheSize), 0} - db.reflectCache[typ] = cs - } else { - cs.idx = cs.idx + 1 - } - return cs.value.Index(cs.idx).Addr() + return &DB{db, NewCacheMapper(&SnakeMapper{})} } func (db *DB) Query(query string, args ...interface{}) (*Rows, error) { @@ -116,7 +83,7 @@ func (db *DB) Query(query string, args ...interface{}) (*Rows, error) { } return nil, err } - return &Rows{rows, db}, nil + return &Rows{rows, db.Mapper}, nil } func (db *DB) QueryMap(query string, mp interface{}) (*Rows, error) { @@ -161,8 +128,8 @@ func (db *DB) QueryRowStruct(query string, st interface{}) *Row { type Stmt struct { *sql.Stmt - db *DB - names map[string]int + Mapper IMapper + names map[string]int } func (db *DB) Prepare(query string) (*Stmt, error) { @@ -178,7 +145,7 @@ func (db *DB) Prepare(query string) (*Stmt, error) { if err != nil { return nil, err } - return &Stmt{stmt, db, names}, nil + return &Stmt{stmt, db.Mapper, names}, nil } func (s *Stmt) ExecMap(mp interface{}) (sql.Result, error) { @@ -212,7 +179,7 @@ func (s *Stmt) Query(args ...interface{}) (*Rows, error) { if err != nil { return nil, err } - return &Rows{rows, s.db}, nil + return &Rows{rows, s.Mapper}, nil } func (s *Stmt) QueryMap(mp interface{}) (*Rows, error) { @@ -307,7 +274,7 @@ func (EmptyScanner) Scan(src interface{}) error { type Tx struct { *sql.Tx - db *DB + Mapper IMapper } func (db *DB) Begin() (*Tx, error) { @@ -315,7 +282,7 @@ func (db *DB) Begin() (*Tx, error) { if err != nil { return nil, err } - return &Tx{tx, db}, nil + return &Tx{tx, db.Mapper}, nil } func (tx *Tx) Prepare(query string) (*Stmt, error) { @@ -331,7 +298,7 @@ func (tx *Tx) Prepare(query string) (*Stmt, error) { if err != nil { return nil, err } - return &Stmt{stmt, tx.db, names}, nil + return &Stmt{stmt, tx.Mapper, names}, nil } func (tx *Tx) Stmt(stmt *Stmt) *Stmt { @@ -360,7 +327,7 @@ func (tx *Tx) Query(query string, args ...interface{}) (*Rows, error) { if err != nil { return nil, err } - return &Rows{rows, tx.db}, nil + return &Rows{rows, tx.Mapper}, nil } func (tx *Tx) QueryMap(query string, mp interface{}) (*Rows, error) { diff --git a/vendor/github.com/go-xorm/core/dialect.go b/vendor/github.com/go-xorm/core/dialect.go index c288a084783..6f2e81d017b 100644 --- a/vendor/github.com/go-xorm/core/dialect.go +++ b/vendor/github.com/go-xorm/core/dialect.go @@ -74,7 +74,6 @@ type Dialect interface { GetIndexes(tableName string) (map[string]*Index, error) Filters() []Filter - SetParams(params map[string]string) } func OpenDialect(dialect Dialect) (*DB, error) { @@ -149,8 +148,7 @@ func (db *Base) SupportDropIfExists() bool { } func (db *Base) DropTableSql(tableName string) string { - quote := db.dialect.Quote - return fmt.Sprintf("DROP TABLE IF EXISTS %s", quote(tableName)) + return fmt.Sprintf("DROP TABLE IF EXISTS `%s`", tableName) } func (db *Base) HasRecords(query string, args ...interface{}) (bool, error) { @@ -291,9 +289,6 @@ func (b *Base) LogSQL(sql string, args []interface{}) { } } -func (b *Base) SetParams(params map[string]string) { -} - var ( dialects = map[string]func() Dialect{} ) diff --git a/vendor/github.com/go-xorm/core/filter.go b/vendor/github.com/go-xorm/core/filter.go index 35b0ece6764..60caaf29026 100644 --- a/vendor/github.com/go-xorm/core/filter.go +++ b/vendor/github.com/go-xorm/core/filter.go @@ -37,9 +37,9 @@ func (q *Quoter) Quote(content string) string { func (i *IdFilter) Do(sql string, dialect Dialect, table *Table) string { quoter := NewQuoter(dialect) if table != nil && len(table.PrimaryKeys) == 1 { - sql = strings.Replace(sql, " `(id)` ", " "+quoter.Quote(table.PrimaryKeys[0])+" ", -1) - sql = strings.Replace(sql, " "+quoter.Quote("(id)")+" ", " "+quoter.Quote(table.PrimaryKeys[0])+" ", -1) - return strings.Replace(sql, " (id) ", " "+quoter.Quote(table.PrimaryKeys[0])+" ", -1) + sql = strings.Replace(sql, "`(id)`", quoter.Quote(table.PrimaryKeys[0]), -1) + sql = strings.Replace(sql, quoter.Quote("(id)"), quoter.Quote(table.PrimaryKeys[0]), -1) + return strings.Replace(sql, "(id)", quoter.Quote(table.PrimaryKeys[0]), -1) } return sql } diff --git a/vendor/github.com/go-xorm/core/index.go b/vendor/github.com/go-xorm/core/index.go index 9aa1b7ac99b..73b95175adc 100644 --- a/vendor/github.com/go-xorm/core/index.go +++ b/vendor/github.com/go-xorm/core/index.go @@ -22,8 +22,6 @@ type Index struct { func (index *Index) XName(tableName string) string { if !strings.HasPrefix(index.Name, "UQE_") && !strings.HasPrefix(index.Name, "IDX_") { - tableName = strings.Replace(tableName, `"`, "", -1) - tableName = strings.Replace(tableName, `.`, "_", -1) if index.Type == UniqueType { return fmt.Sprintf("UQE_%v_%v", tableName, index.Name) } diff --git a/vendor/github.com/go-xorm/core/rows.go b/vendor/github.com/go-xorm/core/rows.go index 580de4f9c66..4a4acaa4c26 100644 --- a/vendor/github.com/go-xorm/core/rows.go +++ b/vendor/github.com/go-xorm/core/rows.go @@ -9,7 +9,7 @@ import ( type Rows struct { *sql.Rows - db *DB + Mapper IMapper } func (rs *Rows) ToMapString() ([]map[string]string, error) { @@ -105,7 +105,7 @@ func (rs *Rows) ScanStructByName(dest interface{}) error { newDest := make([]interface{}, len(cols)) var v EmptyScanner for j, name := range cols { - f := fieldByName(vv.Elem(), rs.db.Mapper.Table2Obj(name)) + f := fieldByName(vv.Elem(), rs.Mapper.Table2Obj(name)) if f.IsValid() { newDest[j] = f.Addr().Interface() } else { @@ -116,6 +116,36 @@ func (rs *Rows) ScanStructByName(dest interface{}) error { return rs.Rows.Scan(newDest...) } +type cacheStruct struct { + value reflect.Value + idx int +} + +var ( + reflectCache = make(map[reflect.Type]*cacheStruct) + reflectCacheMutex sync.RWMutex +) + +func ReflectNew(typ reflect.Type) reflect.Value { + reflectCacheMutex.RLock() + cs, ok := reflectCache[typ] + reflectCacheMutex.RUnlock() + + const newSize = 200 + + if !ok || cs.idx+1 > newSize-1 { + cs = &cacheStruct{reflect.MakeSlice(reflect.SliceOf(typ), newSize, newSize), 0} + reflectCacheMutex.Lock() + reflectCache[typ] = cs + reflectCacheMutex.Unlock() + } else { + reflectCacheMutex.Lock() + cs.idx = cs.idx + 1 + reflectCacheMutex.Unlock() + } + return cs.value.Index(cs.idx).Addr() +} + // scan data to a slice's pointer, slice's length should equal to columns' number func (rs *Rows) ScanSlice(dest interface{}) error { vv := reflect.ValueOf(dest) @@ -167,7 +197,9 @@ func (rs *Rows) ScanMap(dest interface{}) error { vvv := vv.Elem() for i, _ := range cols { - newDest[i] = rs.db.reflectNew(vvv.Type().Elem()).Interface() + newDest[i] = ReflectNew(vvv.Type().Elem()).Interface() + //v := reflect.New(vvv.Type().Elem()) + //newDest[i] = v.Interface() } err = rs.Rows.Scan(newDest...) @@ -183,6 +215,32 @@ func (rs *Rows) ScanMap(dest interface{}) error { return nil } +/*func (rs *Rows) ScanMap(dest interface{}) error { + vv := reflect.ValueOf(dest) + if vv.Kind() != reflect.Ptr || vv.Elem().Kind() != reflect.Map { + return errors.New("dest should be a map's pointer") + } + + cols, err := rs.Columns() + if err != nil { + return err + } + + newDest := make([]interface{}, len(cols)) + err = rs.ScanSlice(newDest) + if err != nil { + return err + } + + vvv := vv.Elem() + + for i, name := range cols { + vname := reflect.ValueOf(name) + vvv.SetMapIndex(vname, reflect.ValueOf(newDest[i]).Elem()) + } + + return nil +}*/ type Row struct { rows *Rows // One of these two will be non-nil: diff --git a/vendor/github.com/go-xorm/core/scan.go b/vendor/github.com/go-xorm/core/scan.go index b7c159b2740..7da338d8645 100644 --- a/vendor/github.com/go-xorm/core/scan.go +++ b/vendor/github.com/go-xorm/core/scan.go @@ -44,9 +44,6 @@ func convertTime(dest *NullTime, src interface{}) error { } *dest = NullTime(t) return nil - case time.Time: - *dest = NullTime(s) - return nil case nil: default: return fmt.Errorf("unsupported driver -> Scan pair: %T -> %T", src, dest) diff --git a/vendor/github.com/go-xorm/core/type.go b/vendor/github.com/go-xorm/core/type.go index 9171ce2d711..8010a2220fc 100644 --- a/vendor/github.com/go-xorm/core/type.go +++ b/vendor/github.com/go-xorm/core/type.go @@ -69,17 +69,15 @@ var ( Enum = "ENUM" Set = "SET" - Char = "CHAR" - Varchar = "VARCHAR" - NVarchar = "NVARCHAR" - TinyText = "TINYTEXT" - Text = "TEXT" - Clob = "CLOB" - MediumText = "MEDIUMTEXT" - LongText = "LONGTEXT" - Uuid = "UUID" - UniqueIdentifier = "UNIQUEIDENTIFIER" - SysName = "SYSNAME" + Char = "CHAR" + Varchar = "VARCHAR" + NVarchar = "NVARCHAR" + TinyText = "TINYTEXT" + Text = "TEXT" + Clob = "CLOB" + MediumText = "MEDIUMTEXT" + LongText = "LONGTEXT" + Uuid = "UUID" Date = "DATE" DateTime = "DATETIME" @@ -134,7 +132,6 @@ var ( LongText: TEXT_TYPE, Uuid: TEXT_TYPE, Clob: TEXT_TYPE, - SysName: TEXT_TYPE, Date: TIME_TYPE, DateTime: TIME_TYPE, @@ -151,12 +148,11 @@ var ( Binary: BLOB_TYPE, VarBinary: BLOB_TYPE, - TinyBlob: BLOB_TYPE, - Blob: BLOB_TYPE, - MediumBlob: BLOB_TYPE, - LongBlob: BLOB_TYPE, - Bytea: BLOB_TYPE, - UniqueIdentifier: BLOB_TYPE, + TinyBlob: BLOB_TYPE, + Blob: BLOB_TYPE, + MediumBlob: BLOB_TYPE, + LongBlob: BLOB_TYPE, + Bytea: BLOB_TYPE, Bool: NUMERIC_TYPE, @@ -293,9 +289,9 @@ func SQLType2Type(st SQLType) reflect.Type { return reflect.TypeOf(float32(1)) case Double: return reflect.TypeOf(float64(1)) - case Char, Varchar, NVarchar, TinyText, Text, MediumText, LongText, Enum, Set, Uuid, Clob, SysName: + case Char, Varchar, NVarchar, TinyText, Text, MediumText, LongText, Enum, Set, Uuid, Clob: return reflect.TypeOf("") - case TinyBlob, Blob, LongBlob, Bytea, Binary, MediumBlob, VarBinary, UniqueIdentifier: + case TinyBlob, Blob, LongBlob, Bytea, Binary, MediumBlob, VarBinary: return reflect.TypeOf([]byte{}) case Bool: return reflect.TypeOf(true) diff --git a/vendor/github.com/go-xorm/xorm/dialect_mysql.go b/vendor/github.com/go-xorm/xorm/dialect_mysql.go index f2b4ff7a786..99100b23251 100644 --- a/vendor/github.com/go-xorm/xorm/dialect_mysql.go +++ b/vendor/github.com/go-xorm/xorm/dialect_mysql.go @@ -172,33 +172,12 @@ type mysql struct { allowAllFiles bool allowOldPasswords bool clientFoundRows bool - rowFormat string } func (db *mysql) Init(d *core.DB, uri *core.Uri, drivername, dataSourceName string) error { return db.Base.Init(d, db, uri, drivername, dataSourceName) } -func (db *mysql) SetParams(params map[string]string) { - rowFormat, ok := params["rowFormat"] - if ok { - var t = strings.ToUpper(rowFormat) - switch t { - case "COMPACT": - fallthrough - case "REDUNDANT": - fallthrough - case "DYNAMIC": - fallthrough - case "COMPRESSED": - db.rowFormat = t - break - default: - break - } - } -} - func (db *mysql) SqlType(c *core.Column) string { var res string switch t := c.SQLType.Name; t { @@ -508,59 +487,6 @@ func (db *mysql) GetIndexes(tableName string) (map[string]*core.Index, error) { return indexes, nil } -func (db *mysql) CreateTableSql(table *core.Table, tableName, storeEngine, charset string) string { - var sql string - sql = "CREATE TABLE IF NOT EXISTS " - if tableName == "" { - tableName = table.Name - } - - sql += db.Quote(tableName) - sql += " (" - - if len(table.ColumnsSeq()) > 0 { - pkList := table.PrimaryKeys - - for _, colName := range table.ColumnsSeq() { - col := table.GetColumn(colName) - if col.IsPrimaryKey && len(pkList) == 1 { - sql += col.String(db) - } else { - sql += col.StringNoPk(db) - } - sql = strings.TrimSpace(sql) - if len(col.Comment) > 0 { - sql += " COMMENT '" + col.Comment + "'" - } - sql += ", " - } - - if len(pkList) > 1 { - sql += "PRIMARY KEY ( " - sql += db.Quote(strings.Join(pkList, db.Quote(","))) - sql += " ), " - } - - sql = sql[:len(sql)-2] - } - sql += ")" - - if storeEngine != "" { - sql += " ENGINE=" + storeEngine - } - - if len(charset) == 0 { - charset = db.URI().Charset - } else if len(charset) > 0 { - sql += " DEFAULT CHARSET " + charset - } - - if db.rowFormat != "" { - sql += " ROW_FORMAT=" + db.rowFormat - } - return sql -} - func (db *mysql) Filters() []core.Filter { return []core.Filter{&core.IdFilter{}} } diff --git a/vendor/github.com/go-xorm/xorm/dialect_postgres.go b/vendor/github.com/go-xorm/xorm/dialect_postgres.go index d907c68c05d..83e9a1015c4 100644 --- a/vendor/github.com/go-xorm/xorm/dialect_postgres.go +++ b/vendor/github.com/go-xorm/xorm/dialect_postgres.go @@ -764,26 +764,14 @@ var ( "YES": true, "ZONE": true, } - - // DefaultPostgresSchema default postgres schema - DefaultPostgresSchema = "public" ) -const postgresPublicSchema = "public" - type postgres struct { core.Base } func (db *postgres) Init(d *core.DB, uri *core.Uri, drivername, dataSourceName string) error { - err := db.Base.Init(d, db, uri, drivername, dataSourceName) - if err != nil { - return err - } - if db.Schema == "" { - db.Schema = DefaultPostgresSchema - } - return nil + return db.Base.Init(d, db, uri, drivername, dataSourceName) } func (db *postgres) SqlType(c *core.Column) string { @@ -880,42 +868,32 @@ func (db *postgres) IndexOnTable() bool { } func (db *postgres) IndexCheckSql(tableName, idxName string) (string, []interface{}) { - if len(db.Schema) == 0 { - args := []interface{}{tableName, idxName} - return `SELECT indexname FROM pg_indexes WHERE tablename = ? AND indexname = ?`, args - } - - args := []interface{}{db.Schema, tableName, idxName} + args := []interface{}{tableName, idxName} return `SELECT indexname FROM pg_indexes ` + - `WHERE schemaname = ? AND tablename = ? AND indexname = ?`, args + `WHERE tablename = ? AND indexname = ?`, args } func (db *postgres) TableCheckSql(tableName string) (string, []interface{}) { - if len(db.Schema) == 0 { - args := []interface{}{tableName} - return `SELECT tablename FROM pg_tables WHERE tablename = ?`, args - } - - args := []interface{}{db.Schema, tableName} - return `SELECT tablename FROM pg_tables WHERE schemaname = ? AND tablename = ?`, args + args := []interface{}{tableName} + return `SELECT tablename FROM pg_tables WHERE tablename = ?`, args } +/*func (db *postgres) ColumnCheckSql(tableName, colName string) (string, []interface{}) { + args := []interface{}{tableName, colName} + return "SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name = ?" + + " AND column_name = ?", args +}*/ + func (db *postgres) ModifyColumnSql(tableName string, col *core.Column) string { - if len(db.Schema) == 0 { - return fmt.Sprintf("alter table %s ALTER COLUMN %s TYPE %s", - tableName, col.Name, db.SqlType(col)) - } - return fmt.Sprintf("alter table %s.%s ALTER COLUMN %s TYPE %s", - db.Schema, tableName, col.Name, db.SqlType(col)) + return fmt.Sprintf("alter table %s ALTER COLUMN %s TYPE %s", + tableName, col.Name, db.SqlType(col)) } func (db *postgres) DropIndexSql(tableName string, index *core.Index) string { + //var unique string quote := db.Quote idxName := index.Name - tableName = strings.Replace(tableName, `"`, "", -1) - tableName = strings.Replace(tableName, `.`, "_", -1) - if !strings.HasPrefix(idxName, "UQE_") && !strings.HasPrefix(idxName, "IDX_") { if index.Type == core.UniqueType { @@ -924,21 +902,13 @@ func (db *postgres) DropIndexSql(tableName string, index *core.Index) string { idxName = fmt.Sprintf("IDX_%v_%v", tableName, index.Name) } } - if db.Uri.Schema != "" { - idxName = db.Uri.Schema + "." + idxName - } return fmt.Sprintf("DROP INDEX %v", quote(idxName)) } func (db *postgres) IsColumnExist(tableName, colName string) (bool, error) { - args := []interface{}{db.Schema, tableName, colName} - query := "SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS WHERE table_schema = $1 AND table_name = $2" + - " AND column_name = $3" - if len(db.Schema) == 0 { - args = []interface{}{tableName, colName} - query = "SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name = $1" + - " AND column_name = $2" - } + args := []interface{}{tableName, colName} + query := "SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name = $1" + + " AND column_name = $2" db.LogSQL(query, args) rows, err := db.DB().Query(query, args...) @@ -951,7 +921,8 @@ func (db *postgres) IsColumnExist(tableName, colName string) (bool, error) { } func (db *postgres) GetColumns(tableName string) ([]string, map[string]*core.Column, error) { - args := []interface{}{tableName} + // FIXME: the schema should be replaced by user custom's + args := []interface{}{tableName, "public"} s := `SELECT column_name, column_default, is_nullable, data_type, character_maximum_length, numeric_precision, numeric_precision_radix , CASE WHEN p.contype = 'p' THEN true ELSE false END AS primarykey, CASE WHEN p.contype = 'u' THEN true ELSE false END AS uniquekey @@ -962,15 +933,7 @@ FROM pg_attribute f LEFT JOIN pg_constraint p ON p.conrelid = c.oid AND f.attnum = ANY (p.conkey) LEFT JOIN pg_class AS g ON p.confrelid = g.oid LEFT JOIN INFORMATION_SCHEMA.COLUMNS s ON s.column_name=f.attname AND c.relname=s.table_name -WHERE c.relkind = 'r'::char AND c.relname = $1%s AND f.attnum > 0 ORDER BY f.attnum;` - - var f string - if len(db.Schema) != 0 { - args = append(args, db.Schema) - f = " AND s.table_schema = $2" - } - s = fmt.Sprintf(s, f) - +WHERE c.relkind = 'r'::char AND c.relname = $1 AND s.table_schema = $2 AND f.attnum > 0 ORDER BY f.attnum;` db.LogSQL(s, args) rows, err := db.DB().Query(s, args...) @@ -1060,13 +1023,9 @@ WHERE c.relkind = 'r'::char AND c.relname = $1%s AND f.attnum > 0 ORDER BY f.att } func (db *postgres) GetTables() ([]*core.Table, error) { - args := []interface{}{} - s := "SELECT tablename FROM pg_tables" - if len(db.Schema) != 0 { - args = append(args, db.Schema) - s = s + " WHERE schemaname = $1" - } - + // FIXME: replace public to user customrize schema + args := []interface{}{"public"} + s := fmt.Sprintf("SELECT tablename FROM pg_tables WHERE schemaname = $1") db.LogSQL(s, args) rows, err := db.DB().Query(s, args...) @@ -1090,12 +1049,9 @@ func (db *postgres) GetTables() ([]*core.Table, error) { } func (db *postgres) GetIndexes(tableName string) (map[string]*core.Index, error) { - args := []interface{}{tableName} - s := fmt.Sprintf("SELECT indexname, indexdef FROM pg_indexes WHERE tablename=$1") - if len(db.Schema) != 0 { - args = append(args, db.Schema) - s = s + " AND schemaname=$2" - } + // FIXME: replace the public schema to user specify schema + args := []interface{}{"public", tableName} + s := fmt.Sprintf("SELECT indexname, indexdef FROM pg_indexes WHERE schemaname=$1 AND tablename=$2") db.LogSQL(s, args) rows, err := db.DB().Query(s, args...) diff --git a/vendor/github.com/go-xorm/xorm/engine.go b/vendor/github.com/go-xorm/xorm/engine.go index 4984d37463b..444611afb16 100644 --- a/vendor/github.com/go-xorm/xorm/engine.go +++ b/vendor/github.com/go-xorm/xorm/engine.go @@ -49,35 +49,6 @@ type Engine struct { tagHandlers map[string]tagHandler engineGroup *EngineGroup - - cachers map[string]core.Cacher - cacherLock sync.RWMutex -} - -func (engine *Engine) setCacher(tableName string, cacher core.Cacher) { - engine.cacherLock.Lock() - engine.cachers[tableName] = cacher - engine.cacherLock.Unlock() -} - -func (engine *Engine) SetCacher(tableName string, cacher core.Cacher) { - engine.setCacher(tableName, cacher) -} - -func (engine *Engine) getCacher(tableName string) core.Cacher { - var cacher core.Cacher - var ok bool - engine.cacherLock.RLock() - cacher, ok = engine.cachers[tableName] - engine.cacherLock.RUnlock() - if !ok && !engine.disableGlobalCache { - cacher = engine.Cacher - } - return cacher -} - -func (engine *Engine) GetCacher(tableName string) core.Cacher { - return engine.getCacher(tableName) } // BufferSize sets buffer size for iterate @@ -274,7 +245,13 @@ func (engine *Engine) NoCascade() *Session { // MapCacher Set a table use a special cacher func (engine *Engine) MapCacher(bean interface{}, cacher core.Cacher) error { - engine.setCacher(engine.TableName(bean, true), cacher) + v := rValue(bean) + tb, err := engine.autoMapType(v) + if err != nil { + return err + } + + tb.Cacher = cacher return nil } @@ -559,6 +536,33 @@ func (engine *Engine) dumpTables(tables []*core.Table, w io.Writer, tp ...core.D return nil } +func (engine *Engine) tableName(beanOrTableName interface{}) (string, error) { + v := rValue(beanOrTableName) + if v.Type().Kind() == reflect.String { + return beanOrTableName.(string), nil + } else if v.Type().Kind() == reflect.Struct { + return engine.tbName(v), nil + } + return "", errors.New("bean should be a struct or struct's point") +} + +func (engine *Engine) tbName(v reflect.Value) string { + if tb, ok := v.Interface().(TableName); ok { + return tb.TableName() + } + + if v.Type().Kind() == reflect.Ptr { + if tb, ok := reflect.Indirect(v).Interface().(TableName); ok { + return tb.TableName() + } + } else if v.CanAddr() { + if tb, ok := v.Addr().Interface().(TableName); ok { + return tb.TableName() + } + } + return engine.TableMapper.Obj2Table(reflect.Indirect(v).Type().Name()) +} + // Cascade use cascade or not func (engine *Engine) Cascade(trueOrFalse ...bool) *Session { session := engine.NewSession() @@ -842,7 +846,7 @@ func (engine *Engine) TableInfo(bean interface{}) *Table { if err != nil { engine.logger.Error(err) } - return &Table{tb, engine.TableName(bean)} + return &Table{tb, engine.tbName(v)} } func addIndex(indexName string, table *core.Table, col *core.Column, indexType int) { @@ -857,6 +861,15 @@ func addIndex(indexName string, table *core.Table, col *core.Column, indexType i } } +func (engine *Engine) newTable() *core.Table { + table := core.NewEmptyTable() + + if !engine.disableGlobalCache { + table.Cacher = engine.Cacher + } + return table +} + // TableName table name interface to define customerize table name type TableName interface { TableName() string @@ -868,9 +881,21 @@ var ( func (engine *Engine) mapType(v reflect.Value) (*core.Table, error) { t := v.Type() - table := core.NewEmptyTable() + table := engine.newTable() + if tb, ok := v.Interface().(TableName); ok { + table.Name = tb.TableName() + } else { + if v.CanAddr() { + if tb, ok = v.Addr().Interface().(TableName); ok { + table.Name = tb.TableName() + } + } + if table.Name == "" { + table.Name = engine.TableMapper.Obj2Table(t.Name()) + } + } + table.Type = t - table.Name = engine.tbNameForMap(v) var idFieldColName string var hasCacheTag, hasNoCacheTag bool @@ -1024,15 +1049,15 @@ func (engine *Engine) mapType(v reflect.Value) (*core.Table, error) { if hasCacheTag { if engine.Cacher != nil { // !nash! use engine's cacher if provided engine.logger.Info("enable cache on table:", table.Name) - engine.setCacher(table.Name, engine.Cacher) + table.Cacher = engine.Cacher } else { engine.logger.Info("enable LRU cache on table:", table.Name) - engine.setCacher(table.Name, NewLRUCacher2(NewMemoryStore(), time.Hour, 10000)) + table.Cacher = NewLRUCacher2(NewMemoryStore(), time.Hour, 10000) // !nashtsai! HACK use LRU cacher for now } } if hasNoCacheTag { - engine.logger.Info("disable cache on table:", table.Name) - engine.setCacher(table.Name, nil) + engine.logger.Info("no cache on table:", table.Name) + table.Cacher = nil } return table, nil @@ -1137,10 +1162,26 @@ func (engine *Engine) CreateUniques(bean interface{}) error { return session.CreateUniques(bean) } +func (engine *Engine) getCacher2(table *core.Table) core.Cacher { + return table.Cacher +} + // ClearCacheBean if enabled cache, clear the cache bean func (engine *Engine) ClearCacheBean(bean interface{}, id string) error { - tableName := engine.TableName(bean) - cacher := engine.getCacher(tableName) + v := rValue(bean) + t := v.Type() + if t.Kind() != reflect.Struct { + return errors.New("error params") + } + tableName := engine.tbName(v) + table, err := engine.autoMapType(v) + if err != nil { + return err + } + cacher := table.Cacher + if cacher == nil { + cacher = engine.Cacher + } if cacher != nil { cacher.ClearIds(tableName) cacher.DelBean(tableName, id) @@ -1151,8 +1192,21 @@ func (engine *Engine) ClearCacheBean(bean interface{}, id string) error { // ClearCache if enabled cache, clear some tables' cache func (engine *Engine) ClearCache(beans ...interface{}) error { for _, bean := range beans { - tableName := engine.TableName(bean) - cacher := engine.getCacher(tableName) + v := rValue(bean) + t := v.Type() + if t.Kind() != reflect.Struct { + return errors.New("error params") + } + tableName := engine.tbName(v) + table, err := engine.autoMapType(v) + if err != nil { + return err + } + + cacher := table.Cacher + if cacher == nil { + cacher = engine.Cacher + } if cacher != nil { cacher.ClearIds(tableName) cacher.ClearBeans(tableName) @@ -1170,13 +1224,13 @@ func (engine *Engine) Sync(beans ...interface{}) error { for _, bean := range beans { v := rValue(bean) - tableNameNoSchema := engine.tbNameNoSchema(v.Interface()) + tableName := engine.tbName(v) table, err := engine.autoMapType(v) if err != nil { return err } - isExist, err := session.Table(bean).isTableExist(tableNameNoSchema) + isExist, err := session.Table(bean).isTableExist(tableName) if err != nil { return err } @@ -1202,12 +1256,12 @@ func (engine *Engine) Sync(beans ...interface{}) error { } } else { for _, col := range table.Columns() { - isExist, err := engine.dialect.IsColumnExist(tableNameNoSchema, col.Name) + isExist, err := engine.dialect.IsColumnExist(tableName, col.Name) if err != nil { return err } if !isExist { - if err := session.statement.setRefBean(bean); err != nil { + if err := session.statement.setRefValue(v); err != nil { return err } err = session.addColumn(col.Name) @@ -1218,35 +1272,35 @@ func (engine *Engine) Sync(beans ...interface{}) error { } for name, index := range table.Indexes { - if err := session.statement.setRefBean(bean); err != nil { + if err := session.statement.setRefValue(v); err != nil { return err } if index.Type == core.UniqueType { - isExist, err := session.isIndexExist2(tableNameNoSchema, index.Cols, true) + isExist, err := session.isIndexExist2(tableName, index.Cols, true) if err != nil { return err } if !isExist { - if err := session.statement.setRefBean(bean); err != nil { + if err := session.statement.setRefValue(v); err != nil { return err } - err = session.addUnique(tableNameNoSchema, name) + err = session.addUnique(tableName, name) if err != nil { return err } } } else if index.Type == core.IndexType { - isExist, err := session.isIndexExist2(tableNameNoSchema, index.Cols, false) + isExist, err := session.isIndexExist2(tableName, index.Cols, false) if err != nil { return err } if !isExist { - if err := session.statement.setRefBean(bean); err != nil { + if err := session.statement.setRefValue(v); err != nil { return err } - err = session.addIndex(tableNameNoSchema, name) + err = session.addIndex(tableName, name) if err != nil { return err } @@ -1399,13 +1453,6 @@ func (engine *Engine) Find(beans interface{}, condiBeans ...interface{}) error { return session.Find(beans, condiBeans...) } -// FindAndCount find the results and also return the counts -func (engine *Engine) FindAndCount(rowsSlicePtr interface{}, condiBean ...interface{}) (int64, error) { - session := engine.NewSession() - defer session.Close() - return session.FindAndCount(rowsSlicePtr, condiBean...) -} - // Iterate record by record handle records from table, bean's non-empty fields // are conditions. func (engine *Engine) Iterate(bean interface{}, fun IterFunc) error { @@ -1582,11 +1629,6 @@ func (engine *Engine) SetTZDatabase(tz *time.Location) { engine.DatabaseTZ = tz } -// SetSchema sets the schema of database -func (engine *Engine) SetSchema(schema string) { - engine.dialect.URI().Schema = schema -} - // Unscoped always disable struct tag "deleted" func (engine *Engine) Unscoped() *Session { session := engine.NewSession() diff --git a/vendor/github.com/go-xorm/xorm/engine_cond.go b/vendor/github.com/go-xorm/xorm/engine_cond.go index 4dde8662e13..6c8e3879cee 100644 --- a/vendor/github.com/go-xorm/xorm/engine_cond.go +++ b/vendor/github.com/go-xorm/xorm/engine_cond.go @@ -9,7 +9,6 @@ import ( "encoding/json" "fmt" "reflect" - "strings" "time" "github.com/go-xorm/builder" @@ -52,9 +51,7 @@ func (engine *Engine) buildConds(table *core.Table, bean interface{}, fieldValuePtr, err := col.ValueOf(bean) if err != nil { - if !strings.Contains(err.Error(), "is not valid") { - engine.logger.Warn(err) - } + engine.logger.Error(err) continue } diff --git a/vendor/github.com/go-xorm/xorm/engine_table.go b/vendor/github.com/go-xorm/xorm/engine_table.go deleted file mode 100644 index 94871a4bce5..00000000000 --- a/vendor/github.com/go-xorm/xorm/engine_table.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2018 The Xorm Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package xorm - -import ( - "fmt" - "reflect" - "strings" - - "github.com/go-xorm/core" -) - -// TableNameWithSchema will automatically add schema prefix on table name -func (engine *Engine) tbNameWithSchema(v string) string { - // Add schema name as prefix of table name. - // Only for postgres database. - if engine.dialect.DBType() == core.POSTGRES && - engine.dialect.URI().Schema != "" && - engine.dialect.URI().Schema != postgresPublicSchema && - strings.Index(v, ".") == -1 { - return engine.dialect.URI().Schema + "." + v - } - return v -} - -// TableName returns table name with schema prefix if has -func (engine *Engine) TableName(bean interface{}, includeSchema ...bool) string { - tbName := engine.tbNameNoSchema(bean) - if len(includeSchema) > 0 && includeSchema[0] { - tbName = engine.tbNameWithSchema(tbName) - } - - return tbName -} - -// tbName get some table's table name -func (session *Session) tbNameNoSchema(table *core.Table) string { - if len(session.statement.AltTableName) > 0 { - return session.statement.AltTableName - } - - return table.Name -} - -func (engine *Engine) tbNameForMap(v reflect.Value) string { - if v.Type().Implements(tpTableName) { - return v.Interface().(TableName).TableName() - } - if v.Kind() == reflect.Ptr { - v = v.Elem() - if v.Type().Implements(tpTableName) { - return v.Interface().(TableName).TableName() - } - } - - return engine.TableMapper.Obj2Table(v.Type().Name()) -} - -func (engine *Engine) tbNameNoSchema(tablename interface{}) string { - switch tablename.(type) { - case []string: - t := tablename.([]string) - if len(t) > 1 { - return fmt.Sprintf("%v AS %v", engine.Quote(t[0]), engine.Quote(t[1])) - } else if len(t) == 1 { - return engine.Quote(t[0]) - } - case []interface{}: - t := tablename.([]interface{}) - l := len(t) - var table string - if l > 0 { - f := t[0] - switch f.(type) { - case string: - table = f.(string) - case TableName: - table = f.(TableName).TableName() - default: - v := rValue(f) - t := v.Type() - if t.Kind() == reflect.Struct { - table = engine.tbNameForMap(v) - } else { - table = engine.Quote(fmt.Sprintf("%v", f)) - } - } - } - if l > 1 { - return fmt.Sprintf("%v AS %v", engine.Quote(table), - engine.Quote(fmt.Sprintf("%v", t[1]))) - } else if l == 1 { - return engine.Quote(table) - } - case TableName: - return tablename.(TableName).TableName() - case string: - return tablename.(string) - case reflect.Value: - v := tablename.(reflect.Value) - return engine.tbNameForMap(v) - default: - v := rValue(tablename) - t := v.Type() - if t.Kind() == reflect.Struct { - return engine.tbNameForMap(v) - } - return engine.Quote(fmt.Sprintf("%v", tablename)) - } - return "" -} diff --git a/vendor/github.com/go-xorm/xorm/error.go b/vendor/github.com/go-xorm/xorm/error.go index 1694683cf31..cfeefc31e8e 100644 --- a/vendor/github.com/go-xorm/xorm/error.go +++ b/vendor/github.com/go-xorm/xorm/error.go @@ -6,7 +6,6 @@ package xorm import ( "errors" - "fmt" ) var ( @@ -26,16 +25,4 @@ var ( ErrNotImplemented = errors.New("Not implemented") // ErrConditionType condition type unsupported ErrConditionType = errors.New("Unsupported conditon type") - // ErrColumnIsNotExist columns is not exist - ErrFieldIsNotExist = errors.New("Field is not exist") ) - -// ErrFieldIsNotValid is not valid -type ErrFieldIsNotValid struct { - FieldName string - TableName string -} - -func (e ErrFieldIsNotValid) Error() string { - return fmt.Sprintf("field %s is not valid on table %s", e.FieldName, e.TableName) -} diff --git a/vendor/github.com/go-xorm/xorm/helpers.go b/vendor/github.com/go-xorm/xorm/helpers.go index f1705782e3d..f39ed472560 100644 --- a/vendor/github.com/go-xorm/xorm/helpers.go +++ b/vendor/github.com/go-xorm/xorm/helpers.go @@ -11,6 +11,7 @@ import ( "sort" "strconv" "strings" + "time" "github.com/go-xorm/core" ) @@ -292,6 +293,19 @@ func structName(v reflect.Type) string { return v.Name() } +func col2NewCols(columns ...string) []string { + newColumns := make([]string, 0, len(columns)) + for _, col := range columns { + col = strings.Replace(col, "`", "", -1) + col = strings.Replace(col, `"`, "", -1) + ccols := strings.Split(col, ",") + for _, c := range ccols { + newColumns = append(newColumns, strings.TrimSpace(c)) + } + } + return newColumns +} + func sliceEq(left, right []string) bool { if len(left) != len(right) { return false @@ -306,6 +320,154 @@ func sliceEq(left, right []string) bool { return true } +func setColumnInt(bean interface{}, col *core.Column, t int64) { + v, err := col.ValueOf(bean) + if err != nil { + return + } + if v.CanSet() { + switch v.Type().Kind() { + case reflect.Int, reflect.Int64, reflect.Int32: + v.SetInt(t) + case reflect.Uint, reflect.Uint64, reflect.Uint32: + v.SetUint(uint64(t)) + } + } +} + +func setColumnTime(bean interface{}, col *core.Column, t time.Time) { + v, err := col.ValueOf(bean) + if err != nil { + return + } + if v.CanSet() { + switch v.Type().Kind() { + case reflect.Struct: + v.Set(reflect.ValueOf(t).Convert(v.Type())) + case reflect.Int, reflect.Int64, reflect.Int32: + v.SetInt(t.Unix()) + case reflect.Uint, reflect.Uint64, reflect.Uint32: + v.SetUint(uint64(t.Unix())) + } + } +} + +func genCols(table *core.Table, session *Session, bean interface{}, useCol bool, includeQuote bool) ([]string, []interface{}, error) { + colNames := make([]string, 0, len(table.ColumnsSeq())) + args := make([]interface{}, 0, len(table.ColumnsSeq())) + + for _, col := range table.Columns() { + if useCol && !col.IsVersion && !col.IsCreated && !col.IsUpdated { + if _, ok := getFlagForColumn(session.statement.columnMap, col); !ok { + continue + } + } + if col.MapType == core.ONLYFROMDB { + continue + } + + fieldValuePtr, err := col.ValueOf(bean) + if err != nil { + return nil, nil, err + } + fieldValue := *fieldValuePtr + + if col.IsAutoIncrement { + switch fieldValue.Type().Kind() { + case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int, reflect.Int64: + if fieldValue.Int() == 0 { + continue + } + case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint, reflect.Uint64: + if fieldValue.Uint() == 0 { + continue + } + case reflect.String: + if len(fieldValue.String()) == 0 { + continue + } + case reflect.Ptr: + if fieldValue.Pointer() == 0 { + continue + } + } + } + + if col.IsDeleted { + continue + } + + if session.statement.ColumnStr != "" { + if _, ok := getFlagForColumn(session.statement.columnMap, col); !ok { + continue + } else if _, ok := session.statement.incrColumns[col.Name]; ok { + continue + } else if _, ok := session.statement.decrColumns[col.Name]; ok { + continue + } + } + if session.statement.OmitStr != "" { + if _, ok := getFlagForColumn(session.statement.columnMap, col); ok { + continue + } + } + + // !evalphobia! set fieldValue as nil when column is nullable and zero-value + if _, ok := getFlagForColumn(session.statement.nullableMap, col); ok { + if col.Nullable && isZero(fieldValue.Interface()) { + var nilValue *int + fieldValue = reflect.ValueOf(nilValue) + } + } + + if (col.IsCreated || col.IsUpdated) && session.statement.UseAutoTime /*&& isZero(fieldValue.Interface())*/ { + // if time is non-empty, then set to auto time + val, t := session.engine.nowTime(col) + args = append(args, val) + + var colName = col.Name + session.afterClosures = append(session.afterClosures, func(bean interface{}) { + col := table.GetColumn(colName) + setColumnTime(bean, col, t) + }) + } else if col.IsVersion && session.statement.checkVersion { + args = append(args, 1) + } else { + arg, err := session.value2Interface(col, fieldValue) + if err != nil { + return colNames, args, err + } + args = append(args, arg) + } + + if includeQuote { + colNames = append(colNames, session.engine.Quote(col.Name)+" = ?") + } else { + colNames = append(colNames, col.Name) + } + } + return colNames, args, nil +} + func indexName(tableName, idxName string) string { return fmt.Sprintf("IDX_%v_%v", tableName, idxName) } + +func getFlagForColumn(m map[string]bool, col *core.Column) (val bool, has bool) { + if len(m) == 0 { + return false, false + } + + n := len(col.Name) + + for mk := range m { + if len(mk) != n { + continue + } + if strings.EqualFold(mk, col.Name) { + return m[mk], true + } + } + + return false, false +} diff --git a/vendor/github.com/go-xorm/xorm/interface.go b/vendor/github.com/go-xorm/xorm/interface.go index 0bc12ba0066..9a3b6da0b2b 100644 --- a/vendor/github.com/go-xorm/xorm/interface.go +++ b/vendor/github.com/go-xorm/xorm/interface.go @@ -30,7 +30,6 @@ type Interface interface { Exec(string, ...interface{}) (sql.Result, error) Exist(bean ...interface{}) (bool, error) Find(interface{}, ...interface{}) error - FindAndCount(interface{}, ...interface{}) (int64, error) Get(interface{}) (bool, error) GroupBy(keys string) *Session ID(interface{}) *Session @@ -42,7 +41,6 @@ type Interface interface { IsTableExist(beanOrTableName interface{}) (bool, error) Iterate(interface{}, IterFunc) error Limit(int, ...int) *Session - MustCols(columns ...string) *Session NoAutoCondition(...bool) *Session NotIn(string, ...interface{}) *Session Join(joinOperator string, tablename interface{}, condition string, args ...interface{}) *Session @@ -77,7 +75,6 @@ type EngineInterface interface { Dialect() core.Dialect DropTables(...interface{}) error DumpAllToFile(fp string, tp ...core.DbType) error - GetCacher(string) core.Cacher GetColumnMapper() core.IMapper GetDefaultCacher() core.Cacher GetTableMapper() core.IMapper @@ -86,11 +83,9 @@ type EngineInterface interface { NewSession() *Session NoAutoTime() *Session Quote(string) string - SetCacher(string, core.Cacher) SetDefaultCacher(core.Cacher) SetLogLevel(core.LogLevel) SetMapper(core.IMapper) - SetSchema(string) SetTZDatabase(tz *time.Location) SetTZLocation(tz *time.Location) ShowSQL(show ...bool) @@ -98,7 +93,6 @@ type EngineInterface interface { Sync2(...interface{}) error StoreEngine(storeEngine string) *Session TableInfo(bean interface{}) *Table - TableName(interface{}, ...bool) string UnMapType(reflect.Type) } diff --git a/vendor/github.com/go-xorm/xorm/rows.go b/vendor/github.com/go-xorm/xorm/rows.go index 54ec7f37a28..31e29ae26f6 100644 --- a/vendor/github.com/go-xorm/xorm/rows.go +++ b/vendor/github.com/go-xorm/xorm/rows.go @@ -32,7 +32,7 @@ func newRows(session *Session, bean interface{}) (*Rows, error) { var args []interface{} var err error - if err = rows.session.statement.setRefBean(bean); err != nil { + if err = rows.session.statement.setRefValue(rValue(bean)); err != nil { return nil, err } @@ -94,7 +94,8 @@ func (rows *Rows) Scan(bean interface{}) error { return fmt.Errorf("scan arg is incompatible type to [%v]", rows.beanType) } - if err := rows.session.statement.setRefBean(bean); err != nil { + dataStruct := rValue(bean) + if err := rows.session.statement.setRefValue(dataStruct); err != nil { return err } @@ -103,7 +104,6 @@ func (rows *Rows) Scan(bean interface{}) error { return err } - dataStruct := rValue(bean) _, err = rows.session.slice2Bean(scanResults, rows.fields, bean, &dataStruct, rows.session.statement.RefTable) if err != nil { return err diff --git a/vendor/github.com/go-xorm/xorm/session.go b/vendor/github.com/go-xorm/xorm/session.go index 48baf768eb5..5c6cb5f9def 100644 --- a/vendor/github.com/go-xorm/xorm/session.go +++ b/vendor/github.com/go-xorm/xorm/session.go @@ -278,22 +278,24 @@ func (session *Session) doPrepare(db *core.DB, sqlStr string) (stmt *core.Stmt, return } -func (session *Session) getField(dataStruct *reflect.Value, key string, table *core.Table, idx int) (*reflect.Value, error) { +func (session *Session) getField(dataStruct *reflect.Value, key string, table *core.Table, idx int) *reflect.Value { var col *core.Column if col = table.GetColumnIdx(key, idx); col == nil { - return nil, ErrFieldIsNotExist + //session.engine.logger.Warnf("table %v has no column %v. %v", table.Name, key, table.ColumnsSeq()) + return nil } fieldValue, err := col.ValueOfV(dataStruct) if err != nil { - return nil, err + session.engine.logger.Error(err) + return nil } if !fieldValue.IsValid() || !fieldValue.CanSet() { - return nil, ErrFieldIsNotValid{key, table.Name} + session.engine.logger.Warnf("table %v's column %v is not valid or cannot set", table.Name, key) + return nil } - - return fieldValue, nil + return fieldValue } // Cell cell is a result of one column field @@ -405,417 +407,409 @@ func (session *Session) slice2Bean(scanResults []interface{}, fields []string, b } tempMap[lKey] = idx - fieldValue, err := session.getField(dataStruct, key, table, idx) - if err != nil { - if !strings.Contains(err.Error(), "is not valid") { - session.engine.logger.Warn(err) + if fieldValue := session.getField(dataStruct, key, table, idx); fieldValue != nil { + rawValue := reflect.Indirect(reflect.ValueOf(scanResults[ii])) + + // if row is null then ignore + if rawValue.Interface() == nil { + continue } - continue - } - if fieldValue == nil { - continue - } - rawValue := reflect.Indirect(reflect.ValueOf(scanResults[ii])) - // if row is null then ignore - if rawValue.Interface() == nil { - continue - } - - if fieldValue.CanAddr() { - if structConvert, ok := fieldValue.Addr().Interface().(core.Conversion); ok { - if data, err := value2Bytes(&rawValue); err == nil { - if err := structConvert.FromDB(data); err != nil { + if fieldValue.CanAddr() { + if structConvert, ok := fieldValue.Addr().Interface().(core.Conversion); ok { + if data, err := value2Bytes(&rawValue); err == nil { + if err := structConvert.FromDB(data); err != nil { + return nil, err + } + } else { return nil, err } + continue + } + } + + if _, ok := fieldValue.Interface().(core.Conversion); ok { + if data, err := value2Bytes(&rawValue); err == nil { + if fieldValue.Kind() == reflect.Ptr && fieldValue.IsNil() { + fieldValue.Set(reflect.New(fieldValue.Type().Elem())) + } + fieldValue.Interface().(core.Conversion).FromDB(data) } else { return nil, err } continue } - } - if _, ok := fieldValue.Interface().(core.Conversion); ok { - if data, err := value2Bytes(&rawValue); err == nil { - if fieldValue.Kind() == reflect.Ptr && fieldValue.IsNil() { - fieldValue.Set(reflect.New(fieldValue.Type().Elem())) - } - fieldValue.Interface().(core.Conversion).FromDB(data) - } else { - return nil, err + rawValueType := reflect.TypeOf(rawValue.Interface()) + vv := reflect.ValueOf(rawValue.Interface()) + col := table.GetColumnIdx(key, idx) + if col.IsPrimaryKey { + pk = append(pk, rawValue.Interface()) } - continue - } + fieldType := fieldValue.Type() + hasAssigned := false - rawValueType := reflect.TypeOf(rawValue.Interface()) - vv := reflect.ValueOf(rawValue.Interface()) - col := table.GetColumnIdx(key, idx) - if col.IsPrimaryKey { - pk = append(pk, rawValue.Interface()) - } - fieldType := fieldValue.Type() - hasAssigned := false - - if col.SQLType.IsJson() { - var bs []byte - if rawValueType.Kind() == reflect.String { - bs = []byte(vv.String()) - } else if rawValueType.ConvertibleTo(core.BytesType) { - bs = vv.Bytes() - } else { - return nil, fmt.Errorf("unsupported database data type: %s %v", key, rawValueType.Kind()) - } - - hasAssigned = true - - if len(bs) > 0 { - if fieldType.Kind() == reflect.String { - fieldValue.SetString(string(bs)) - continue - } - if fieldValue.CanAddr() { - err := json.Unmarshal(bs, fieldValue.Addr().Interface()) - if err != nil { - return nil, err - } + if col.SQLType.IsJson() { + var bs []byte + if rawValueType.Kind() == reflect.String { + bs = []byte(vv.String()) + } else if rawValueType.ConvertibleTo(core.BytesType) { + bs = vv.Bytes() } else { - x := reflect.New(fieldType) - err := json.Unmarshal(bs, x.Interface()) - if err != nil { - return nil, err - } - fieldValue.Set(x.Elem()) + return nil, fmt.Errorf("unsupported database data type: %s %v", key, rawValueType.Kind()) } - } - continue - } + hasAssigned = true - switch fieldType.Kind() { - case reflect.Complex64, reflect.Complex128: - // TODO: reimplement this - var bs []byte - if rawValueType.Kind() == reflect.String { - bs = []byte(vv.String()) - } else if rawValueType.ConvertibleTo(core.BytesType) { - bs = vv.Bytes() - } - - hasAssigned = true - if len(bs) > 0 { - if fieldValue.CanAddr() { - err := json.Unmarshal(bs, fieldValue.Addr().Interface()) - if err != nil { - return nil, err + if len(bs) > 0 { + if fieldType.Kind() == reflect.String { + fieldValue.SetString(string(bs)) + continue } - } else { - x := reflect.New(fieldType) - err := json.Unmarshal(bs, x.Interface()) - if err != nil { - return nil, err + if fieldValue.CanAddr() { + err := json.Unmarshal(bs, fieldValue.Addr().Interface()) + if err != nil { + return nil, err + } + } else { + x := reflect.New(fieldType) + err := json.Unmarshal(bs, x.Interface()) + if err != nil { + return nil, err + } + fieldValue.Set(x.Elem()) } - fieldValue.Set(x.Elem()) } + + continue } - case reflect.Slice, reflect.Array: - switch rawValueType.Kind() { + + switch fieldType.Kind() { + case reflect.Complex64, reflect.Complex128: + // TODO: reimplement this + var bs []byte + if rawValueType.Kind() == reflect.String { + bs = []byte(vv.String()) + } else if rawValueType.ConvertibleTo(core.BytesType) { + bs = vv.Bytes() + } + + hasAssigned = true + if len(bs) > 0 { + if fieldValue.CanAddr() { + err := json.Unmarshal(bs, fieldValue.Addr().Interface()) + if err != nil { + return nil, err + } + } else { + x := reflect.New(fieldType) + err := json.Unmarshal(bs, x.Interface()) + if err != nil { + return nil, err + } + fieldValue.Set(x.Elem()) + } + } case reflect.Slice, reflect.Array: - switch rawValueType.Elem().Kind() { - case reflect.Uint8: - if fieldType.Elem().Kind() == reflect.Uint8 { + switch rawValueType.Kind() { + case reflect.Slice, reflect.Array: + switch rawValueType.Elem().Kind() { + case reflect.Uint8: + if fieldType.Elem().Kind() == reflect.Uint8 { + hasAssigned = true + if col.SQLType.IsText() { + x := reflect.New(fieldType) + err := json.Unmarshal(vv.Bytes(), x.Interface()) + if err != nil { + return nil, err + } + fieldValue.Set(x.Elem()) + } else { + if fieldValue.Len() > 0 { + for i := 0; i < fieldValue.Len(); i++ { + if i < vv.Len() { + fieldValue.Index(i).Set(vv.Index(i)) + } + } + } else { + for i := 0; i < vv.Len(); i++ { + fieldValue.Set(reflect.Append(*fieldValue, vv.Index(i))) + } + } + } + } + } + } + case reflect.String: + if rawValueType.Kind() == reflect.String { + hasAssigned = true + fieldValue.SetString(vv.String()) + } + case reflect.Bool: + if rawValueType.Kind() == reflect.Bool { + hasAssigned = true + fieldValue.SetBool(vv.Bool()) + } + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + switch rawValueType.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + hasAssigned = true + fieldValue.SetInt(vv.Int()) + } + case reflect.Float32, reflect.Float64: + switch rawValueType.Kind() { + case reflect.Float32, reflect.Float64: + hasAssigned = true + fieldValue.SetFloat(vv.Float()) + } + case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint: + switch rawValueType.Kind() { + case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint: + hasAssigned = true + fieldValue.SetUint(vv.Uint()) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + hasAssigned = true + fieldValue.SetUint(uint64(vv.Int())) + } + case reflect.Struct: + if fieldType.ConvertibleTo(core.TimeType) { + dbTZ := session.engine.DatabaseTZ + if col.TimeZone != nil { + dbTZ = col.TimeZone + } + + if rawValueType == core.TimeType { hasAssigned = true - if col.SQLType.IsText() { - x := reflect.New(fieldType) + + t := vv.Convert(core.TimeType).Interface().(time.Time) + + z, _ := t.Zone() + // set new location if database don't save timezone or give an incorrect timezone + if len(z) == 0 || t.Year() == 0 || t.Location().String() != dbTZ.String() { // !nashtsai! HACK tmp work around for lib/pq doesn't properly time with location + session.engine.logger.Debugf("empty zone key[%v] : %v | zone: %v | location: %+v\n", key, t, z, *t.Location()) + t = time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), + t.Minute(), t.Second(), t.Nanosecond(), dbTZ) + } + + t = t.In(session.engine.TZLocation) + fieldValue.Set(reflect.ValueOf(t).Convert(fieldType)) + } else if rawValueType == core.IntType || rawValueType == core.Int64Type || + rawValueType == core.Int32Type { + hasAssigned = true + + t := time.Unix(vv.Int(), 0).In(session.engine.TZLocation) + fieldValue.Set(reflect.ValueOf(t).Convert(fieldType)) + } else { + if d, ok := vv.Interface().([]uint8); ok { + hasAssigned = true + t, err := session.byte2Time(col, d) + if err != nil { + session.engine.logger.Error("byte2Time error:", err.Error()) + hasAssigned = false + } else { + fieldValue.Set(reflect.ValueOf(t).Convert(fieldType)) + } + } else if d, ok := vv.Interface().(string); ok { + hasAssigned = true + t, err := session.str2Time(col, d) + if err != nil { + session.engine.logger.Error("byte2Time error:", err.Error()) + hasAssigned = false + } else { + fieldValue.Set(reflect.ValueOf(t).Convert(fieldType)) + } + } else { + return nil, fmt.Errorf("rawValueType is %v, value is %v", rawValueType, vv.Interface()) + } + } + } else if nulVal, ok := fieldValue.Addr().Interface().(sql.Scanner); ok { + // !! 增加支持sql.Scanner接口的结构，如sql.NullString + hasAssigned = true + if err := nulVal.Scan(vv.Interface()); err != nil { + session.engine.logger.Error("sql.Sanner error:", err.Error()) + hasAssigned = false + } + } else if col.SQLType.IsJson() { + if rawValueType.Kind() == reflect.String { + hasAssigned = true + x := reflect.New(fieldType) + if len([]byte(vv.String())) > 0 { + err := json.Unmarshal([]byte(vv.String()), x.Interface()) + if err != nil { + return nil, err + } + fieldValue.Set(x.Elem()) + } + } else if rawValueType.Kind() == reflect.Slice { + hasAssigned = true + x := reflect.New(fieldType) + if len(vv.Bytes()) > 0 { err := json.Unmarshal(vv.Bytes(), x.Interface()) if err != nil { return nil, err } fieldValue.Set(x.Elem()) - } else { - if fieldValue.Len() > 0 { - for i := 0; i < fieldValue.Len(); i++ { - if i < vv.Len() { - fieldValue.Index(i).Set(vv.Index(i)) - } - } - } else { - for i := 0; i < vv.Len(); i++ { - fieldValue.Set(reflect.Append(*fieldValue, vv.Index(i))) - } - } } } - } - } - case reflect.String: - if rawValueType.Kind() == reflect.String { - hasAssigned = true - fieldValue.SetString(vv.String()) - } - case reflect.Bool: - if rawValueType.Kind() == reflect.Bool { - hasAssigned = true - fieldValue.SetBool(vv.Bool()) - } - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - switch rawValueType.Kind() { - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - hasAssigned = true - fieldValue.SetInt(vv.Int()) - } - case reflect.Float32, reflect.Float64: - switch rawValueType.Kind() { - case reflect.Float32, reflect.Float64: - hasAssigned = true - fieldValue.SetFloat(vv.Float()) - } - case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint: - switch rawValueType.Kind() { - case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint: - hasAssigned = true - fieldValue.SetUint(vv.Uint()) - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - hasAssigned = true - fieldValue.SetUint(uint64(vv.Int())) - } - case reflect.Struct: - if fieldType.ConvertibleTo(core.TimeType) { - dbTZ := session.engine.DatabaseTZ - if col.TimeZone != nil { - dbTZ = col.TimeZone - } - - if rawValueType == core.TimeType { - hasAssigned = true - - t := vv.Convert(core.TimeType).Interface().(time.Time) - - z, _ := t.Zone() - // set new location if database don't save timezone or give an incorrect timezone - if len(z) == 0 || t.Year() == 0 || t.Location().String() != dbTZ.String() { // !nashtsai! HACK tmp work around for lib/pq doesn't properly time with location - session.engine.logger.Debugf("empty zone key[%v] : %v | zone: %v | location: %+v\n", key, t, z, *t.Location()) - t = time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), - t.Minute(), t.Second(), t.Nanosecond(), dbTZ) + } else if session.statement.UseCascade { + table, err := session.engine.autoMapType(*fieldValue) + if err != nil { + return nil, err } - t = t.In(session.engine.TZLocation) - fieldValue.Set(reflect.ValueOf(t).Convert(fieldType)) - } else if rawValueType == core.IntType || rawValueType == core.Int64Type || - rawValueType == core.Int32Type { hasAssigned = true + if len(table.PrimaryKeys) != 1 { + return nil, errors.New("unsupported non or composited primary key cascade") + } + var pk = make(core.PK, len(table.PrimaryKeys)) + pk[0], err = asKind(vv, rawValueType) + if err != nil { + return nil, err + } - t := time.Unix(vv.Int(), 0).In(session.engine.TZLocation) - fieldValue.Set(reflect.ValueOf(t).Convert(fieldType)) - } else { - if d, ok := vv.Interface().([]uint8); ok { - hasAssigned = true - t, err := session.byte2Time(col, d) + if !isPKZero(pk) { + // !nashtsai! TODO for hasOne relationship, it's preferred to use join query for eager fetch + // however, also need to consider adding a 'lazy' attribute to xorm tag which allow hasOne + // property to be fetched lazily + structInter := reflect.New(fieldValue.Type()) + has, err := session.ID(pk).NoCascade().get(structInter.Interface()) if err != nil { - session.engine.logger.Error("byte2Time error:", err.Error()) - hasAssigned = false - } else { - fieldValue.Set(reflect.ValueOf(t).Convert(fieldType)) + return nil, err } - } else if d, ok := vv.Interface().(string); ok { - hasAssigned = true - t, err := session.str2Time(col, d) - if err != nil { - session.engine.logger.Error("byte2Time error:", err.Error()) - hasAssigned = false + if has { + fieldValue.Set(structInter.Elem()) } else { - fieldValue.Set(reflect.ValueOf(t).Convert(fieldType)) + return nil, errors.New("cascade obj is not exist") } - } else { - return nil, fmt.Errorf("rawValueType is %v, value is %v", rawValueType, vv.Interface()) } } - } else if nulVal, ok := fieldValue.Addr().Interface().(sql.Scanner); ok { - // !! 增加支持sql.Scanner接口的结构，如sql.NullString - hasAssigned = true - if err := nulVal.Scan(vv.Interface()); err != nil { - session.engine.logger.Error("sql.Sanner error:", err.Error()) - hasAssigned = false - } - } else if col.SQLType.IsJson() { - if rawValueType.Kind() == reflect.String { - hasAssigned = true - x := reflect.New(fieldType) + case reflect.Ptr: + // !nashtsai! TODO merge duplicated codes above + switch fieldType { + // following types case matching ptr's native type, therefore assign ptr directly + case core.PtrStringType: + if rawValueType.Kind() == reflect.String { + x := vv.String() + hasAssigned = true + fieldValue.Set(reflect.ValueOf(&x)) + } + case core.PtrBoolType: + if rawValueType.Kind() == reflect.Bool { + x := vv.Bool() + hasAssigned = true + fieldValue.Set(reflect.ValueOf(&x)) + } + case core.PtrTimeType: + if rawValueType == core.PtrTimeType { + hasAssigned = true + var x = rawValue.Interface().(time.Time) + fieldValue.Set(reflect.ValueOf(&x)) + } + case core.PtrFloat64Type: + if rawValueType.Kind() == reflect.Float64 { + x := vv.Float() + hasAssigned = true + fieldValue.Set(reflect.ValueOf(&x)) + } + case core.PtrUint64Type: + if rawValueType.Kind() == reflect.Int64 { + var x = uint64(vv.Int()) + hasAssigned = true + fieldValue.Set(reflect.ValueOf(&x)) + } + case core.PtrInt64Type: + if rawValueType.Kind() == reflect.Int64 { + x := vv.Int() + hasAssigned = true + fieldValue.Set(reflect.ValueOf(&x)) + } + case core.PtrFloat32Type: + if rawValueType.Kind() == reflect.Float64 { + var x = float32(vv.Float()) + hasAssigned = true + fieldValue.Set(reflect.ValueOf(&x)) + } + case core.PtrIntType: + if rawValueType.Kind() == reflect.Int64 { + var x = int(vv.Int()) + hasAssigned = true + fieldValue.Set(reflect.ValueOf(&x)) + } + case core.PtrInt32Type: + if rawValueType.Kind() == reflect.Int64 { + var x = int32(vv.Int()) + hasAssigned = true + fieldValue.Set(reflect.ValueOf(&x)) + } + case core.PtrInt8Type: + if rawValueType.Kind() == reflect.Int64 { + var x = int8(vv.Int()) + hasAssigned = true + fieldValue.Set(reflect.ValueOf(&x)) + } + case core.PtrInt16Type: + if rawValueType.Kind() == reflect.Int64 { + var x = int16(vv.Int()) + hasAssigned = true + fieldValue.Set(reflect.ValueOf(&x)) + } + case core.PtrUintType: + if rawValueType.Kind() == reflect.Int64 { + var x = uint(vv.Int()) + hasAssigned = true + fieldValue.Set(reflect.ValueOf(&x)) + } + case core.PtrUint32Type: + if rawValueType.Kind() == reflect.Int64 { + var x = uint32(vv.Int()) + hasAssigned = true + fieldValue.Set(reflect.ValueOf(&x)) + } + case core.Uint8Type: + if rawValueType.Kind() == reflect.Int64 { + var x = uint8(vv.Int()) + hasAssigned = true + fieldValue.Set(reflect.ValueOf(&x)) + } + case core.Uint16Type: + if rawValueType.Kind() == reflect.Int64 { + var x = uint16(vv.Int()) + hasAssigned = true + fieldValue.Set(reflect.ValueOf(&x)) + } + case core.Complex64Type: + var x complex64 if len([]byte(vv.String())) > 0 { - err := json.Unmarshal([]byte(vv.String()), x.Interface()) + err := json.Unmarshal([]byte(vv.String()), &x) if err != nil { return nil, err } - fieldValue.Set(x.Elem()) + fieldValue.Set(reflect.ValueOf(&x)) } - } else if rawValueType.Kind() == reflect.Slice { hasAssigned = true - x := reflect.New(fieldType) - if len(vv.Bytes()) > 0 { - err := json.Unmarshal(vv.Bytes(), x.Interface()) + case core.Complex128Type: + var x complex128 + if len([]byte(vv.String())) > 0 { + err := json.Unmarshal([]byte(vv.String()), &x) if err != nil { return nil, err } - fieldValue.Set(x.Elem()) + fieldValue.Set(reflect.ValueOf(&x)) } - } - } else if session.statement.UseCascade { - table, err := session.engine.autoMapType(*fieldValue) + hasAssigned = true + } // switch fieldType + } // switch fieldType.Kind() + + // !nashtsai! for value can't be assigned directly fallback to convert to []byte then back to value + if !hasAssigned { + data, err := value2Bytes(&rawValue) if err != nil { return nil, err } - hasAssigned = true - if len(table.PrimaryKeys) != 1 { - return nil, errors.New("unsupported non or composited primary key cascade") - } - var pk = make(core.PK, len(table.PrimaryKeys)) - pk[0], err = asKind(vv, rawValueType) - if err != nil { + if err = session.bytes2Value(col, fieldValue, data); err != nil { return nil, err } - - if !isPKZero(pk) { - // !nashtsai! TODO for hasOne relationship, it's preferred to use join query for eager fetch - // however, also need to consider adding a 'lazy' attribute to xorm tag which allow hasOne - // property to be fetched lazily - structInter := reflect.New(fieldValue.Type()) - has, err := session.ID(pk).NoCascade().get(structInter.Interface()) - if err != nil { - return nil, err - } - if has { - fieldValue.Set(structInter.Elem()) - } else { - return nil, errors.New("cascade obj is not exist") - } - } - } - case reflect.Ptr: - // !nashtsai! TODO merge duplicated codes above - switch fieldType { - // following types case matching ptr's native type, therefore assign ptr directly - case core.PtrStringType: - if rawValueType.Kind() == reflect.String { - x := vv.String() - hasAssigned = true - fieldValue.Set(reflect.ValueOf(&x)) - } - case core.PtrBoolType: - if rawValueType.Kind() == reflect.Bool { - x := vv.Bool() - hasAssigned = true - fieldValue.Set(reflect.ValueOf(&x)) - } - case core.PtrTimeType: - if rawValueType == core.PtrTimeType { - hasAssigned = true - var x = rawValue.Interface().(time.Time) - fieldValue.Set(reflect.ValueOf(&x)) - } - case core.PtrFloat64Type: - if rawValueType.Kind() == reflect.Float64 { - x := vv.Float() - hasAssigned = true - fieldValue.Set(reflect.ValueOf(&x)) - } - case core.PtrUint64Type: - if rawValueType.Kind() == reflect.Int64 { - var x = uint64(vv.Int()) - hasAssigned = true - fieldValue.Set(reflect.ValueOf(&x)) - } - case core.PtrInt64Type: - if rawValueType.Kind() == reflect.Int64 { - x := vv.Int() - hasAssigned = true - fieldValue.Set(reflect.ValueOf(&x)) - } - case core.PtrFloat32Type: - if rawValueType.Kind() == reflect.Float64 { - var x = float32(vv.Float()) - hasAssigned = true - fieldValue.Set(reflect.ValueOf(&x)) - } - case core.PtrIntType: - if rawValueType.Kind() == reflect.Int64 { - var x = int(vv.Int()) - hasAssigned = true - fieldValue.Set(reflect.ValueOf(&x)) - } - case core.PtrInt32Type: - if rawValueType.Kind() == reflect.Int64 { - var x = int32(vv.Int()) - hasAssigned = true - fieldValue.Set(reflect.ValueOf(&x)) - } - case core.PtrInt8Type: - if rawValueType.Kind() == reflect.Int64 { - var x = int8(vv.Int()) - hasAssigned = true - fieldValue.Set(reflect.ValueOf(&x)) - } - case core.PtrInt16Type: - if rawValueType.Kind() == reflect.Int64 { - var x = int16(vv.Int()) - hasAssigned = true - fieldValue.Set(reflect.ValueOf(&x)) - } - case core.PtrUintType: - if rawValueType.Kind() == reflect.Int64 { - var x = uint(vv.Int()) - hasAssigned = true - fieldValue.Set(reflect.ValueOf(&x)) - } - case core.PtrUint32Type: - if rawValueType.Kind() == reflect.Int64 { - var x = uint32(vv.Int()) - hasAssigned = true - fieldValue.Set(reflect.ValueOf(&x)) - } - case core.Uint8Type: - if rawValueType.Kind() == reflect.Int64 { - var x = uint8(vv.Int()) - hasAssigned = true - fieldValue.Set(reflect.ValueOf(&x)) - } - case core.Uint16Type: - if rawValueType.Kind() == reflect.Int64 { - var x = uint16(vv.Int()) - hasAssigned = true - fieldValue.Set(reflect.ValueOf(&x)) - } - case core.Complex64Type: - var x complex64 - if len([]byte(vv.String())) > 0 { - err := json.Unmarshal([]byte(vv.String()), &x) - if err != nil { - return nil, err - } - fieldValue.Set(reflect.ValueOf(&x)) - } - hasAssigned = true - case core.Complex128Type: - var x complex128 - if len([]byte(vv.String())) > 0 { - err := json.Unmarshal([]byte(vv.String()), &x) - if err != nil { - return nil, err - } - fieldValue.Set(reflect.ValueOf(&x)) - } - hasAssigned = true - } // switch fieldType - } // switch fieldType.Kind() - - // !nashtsai! for value can't be assigned directly fallback to convert to []byte then back to value - if !hasAssigned { - data, err := value2Bytes(&rawValue) - if err != nil { - return nil, err - } - - if err = session.bytes2Value(col, fieldValue, data); err != nil { - return nil, err } } } @@ -834,6 +828,15 @@ func (session *Session) LastSQL() (string, []interface{}) { return session.lastSQL, session.lastSQLArgs } +// tbName get some table's table name +func (session *Session) tbNameNoSchema(table *core.Table) string { + if len(session.statement.AltTableName) > 0 { + return session.statement.AltTableName + } + + return table.Name +} + // Unscoped always disable struct tag "deleted" func (session *Session) Unscoped() *Session { session.statement.Unscoped() diff --git a/vendor/github.com/go-xorm/xorm/session_cols.go b/vendor/github.com/go-xorm/xorm/session_cols.go index 1c2b023d82a..9972cb0ae4b 100644 --- a/vendor/github.com/go-xorm/xorm/session_cols.go +++ b/vendor/github.com/go-xorm/xorm/session_cols.go @@ -4,113 +4,6 @@ package xorm -import ( - "reflect" - "strings" - "time" - - "github.com/go-xorm/core" -) - -type incrParam struct { - colName string - arg interface{} -} - -type decrParam struct { - colName string - arg interface{} -} - -type exprParam struct { - colName string - expr string -} - -type columnMap []string - -func (m columnMap) contain(colName string) bool { - if len(m) == 0 { - return false - } - - n := len(colName) - for _, mk := range m { - if len(mk) != n { - continue - } - if strings.EqualFold(mk, colName) { - return true - } - } - - return false -} - -func setColumnInt(bean interface{}, col *core.Column, t int64) { - v, err := col.ValueOf(bean) - if err != nil { - return - } - if v.CanSet() { - switch v.Type().Kind() { - case reflect.Int, reflect.Int64, reflect.Int32: - v.SetInt(t) - case reflect.Uint, reflect.Uint64, reflect.Uint32: - v.SetUint(uint64(t)) - } - } -} - -func setColumnTime(bean interface{}, col *core.Column, t time.Time) { - v, err := col.ValueOf(bean) - if err != nil { - return - } - if v.CanSet() { - switch v.Type().Kind() { - case reflect.Struct: - v.Set(reflect.ValueOf(t).Convert(v.Type())) - case reflect.Int, reflect.Int64, reflect.Int32: - v.SetInt(t.Unix()) - case reflect.Uint, reflect.Uint64, reflect.Uint32: - v.SetUint(uint64(t.Unix())) - } - } -} - -func getFlagForColumn(m map[string]bool, col *core.Column) (val bool, has bool) { - if len(m) == 0 { - return false, false - } - - n := len(col.Name) - - for mk := range m { - if len(mk) != n { - continue - } - if strings.EqualFold(mk, col.Name) { - return m[mk], true - } - } - - return false, false -} - -func col2NewCols(columns ...string) []string { - newColumns := make([]string, 0, len(columns)) - for _, col := range columns { - col = strings.Replace(col, "`", "", -1) - col = strings.Replace(col, `"`, "", -1) - ccols := strings.Split(col, ",") - for _, c := range ccols { - newColumns = append(newColumns, strings.TrimSpace(c)) - } - } - return newColumns -} - // Incr provides a query string like "count = count + 1" func (session *Session) Incr(column string, arg ...interface{}) *Session { session.statement.Incr(column, arg...) diff --git a/vendor/github.com/go-xorm/xorm/session_delete.go b/vendor/github.com/go-xorm/xorm/session_delete.go index d9cf3ea9373..688b122ca6d 100644 --- a/vendor/github.com/go-xorm/xorm/session_delete.go +++ b/vendor/github.com/go-xorm/xorm/session_delete.go @@ -27,7 +27,7 @@ func (session *Session) cacheDelete(table *core.Table, tableName, sqlStr string, return ErrCacheFailed } - cacher := session.engine.getCacher(tableName) + cacher := session.engine.getCacher2(table) pkColumns := table.PKColumns() ids, err := core.GetCacheSql(cacher, tableName, newsql, args) if err != nil { @@ -79,7 +79,7 @@ func (session *Session) Delete(bean interface{}) (int64, error) { defer session.Close() } - if err := session.statement.setRefBean(bean); err != nil { + if err := session.statement.setRefValue(rValue(bean)); err != nil { return 0, err } @@ -199,7 +199,7 @@ func (session *Session) Delete(bean interface{}) (int64, error) { }) } - if cacher := session.engine.getCacher(tableName); cacher != nil && session.statement.UseCache { + if cacher := session.engine.getCacher2(table); cacher != nil && session.statement.UseCache { session.cacheDelete(table, tableNameNoQuote, deleteSQL, argsForCache...) } diff --git a/vendor/github.com/go-xorm/xorm/session_exist.go b/vendor/github.com/go-xorm/xorm/session_exist.go index 74a660e852b..049c1ddff14 100644 --- a/vendor/github.com/go-xorm/xorm/session_exist.go +++ b/vendor/github.com/go-xorm/xorm/session_exist.go @@ -10,7 +10,6 @@ import ( "reflect" "github.com/go-xorm/builder" - "github.com/go-xorm/core" ) // Exist returns true if the record exist otherwise return false @@ -36,18 +35,10 @@ func (session *Session) Exist(bean ...interface{}) (bool, error) { return false, err } - if session.engine.dialect.DBType() == core.MSSQL { - sqlStr = fmt.Sprintf("SELECT top 1 * FROM %s WHERE %s", tableName, condSQL) - } else { - sqlStr = fmt.Sprintf("SELECT * FROM %s WHERE %s LIMIT 1", tableName, condSQL) - } + sqlStr = fmt.Sprintf("SELECT * FROM %s WHERE %s LIMIT 1", tableName, condSQL) args = condArgs } else { - if session.engine.dialect.DBType() == core.MSSQL { - sqlStr = fmt.Sprintf("SELECT top 1 * FROM %s", tableName) - } else { - sqlStr = fmt.Sprintf("SELECT * FROM %s LIMIT 1", tableName) - } + sqlStr = fmt.Sprintf("SELECT * FROM %s LIMIT 1", tableName) args = []interface{}{} } } else { @@ -57,7 +48,7 @@ func (session *Session) Exist(bean ...interface{}) (bool, error) { } if beanValue.Elem().Kind() == reflect.Struct { - if err := session.statement.setRefBean(bean[0]); err != nil { + if err := session.statement.setRefValue(beanValue.Elem()); err != nil { return false, err } } diff --git a/vendor/github.com/go-xorm/xorm/session_find.go b/vendor/github.com/go-xorm/xorm/session_find.go index 46bbf26c98d..f95dcfef2cb 100644 --- a/vendor/github.com/go-xorm/xorm/session_find.go +++ b/vendor/github.com/go-xorm/xorm/session_find.go @@ -29,39 +29,6 @@ func (session *Session) Find(rowsSlicePtr interface{}, condiBean ...interface{}) return session.find(rowsSlicePtr, condiBean...) } -// FindAndCount find the results and also return the counts -func (session *Session) FindAndCount(rowsSlicePtr interface{}, condiBean ...interface{}) (int64, error) { - if session.isAutoClose { - defer session.Close() - } - - session.autoResetStatement = false - err := session.find(rowsSlicePtr, condiBean...) - if err != nil { - return 0, err - } - - sliceValue := reflect.Indirect(reflect.ValueOf(rowsSlicePtr)) - if sliceValue.Kind() != reflect.Slice && sliceValue.Kind() != reflect.Map { - return 0, errors.New("needs a pointer to a slice or a map") - } - - sliceElementType := sliceValue.Type().Elem() - if sliceElementType.Kind() == reflect.Ptr { - sliceElementType = sliceElementType.Elem() - } - session.autoResetStatement = true - - if session.statement.selectStr != "" { - session.statement.selectStr = "" - } - if session.statement.OrderStr != "" { - session.statement.OrderStr = "" - } - - return session.Count(reflect.New(sliceElementType).Interface()) -} - func (session *Session) find(rowsSlicePtr interface{}, condiBean ...interface{}) error { sliceValue := reflect.Indirect(reflect.ValueOf(rowsSlicePtr)) if sliceValue.Kind() != reflect.Slice && sliceValue.Kind() != reflect.Map { @@ -75,7 +42,7 @@ func (session *Session) find(rowsSlicePtr interface{}, condiBean ...interface{}) if sliceElementType.Kind() == reflect.Ptr { if sliceElementType.Elem().Kind() == reflect.Struct { pv := reflect.New(sliceElementType.Elem()) - if err := session.statement.setRefValue(pv); err != nil { + if err := session.statement.setRefValue(pv.Elem()); err != nil { return err } } else { @@ -83,7 +50,7 @@ func (session *Session) find(rowsSlicePtr interface{}, condiBean ...interface{}) } } else if sliceElementType.Kind() == reflect.Struct { pv := reflect.New(sliceElementType) - if err := session.statement.setRefValue(pv); err != nil { + if err := session.statement.setRefValue(pv.Elem()); err != nil { return err } } else { @@ -161,7 +128,7 @@ func (session *Session) find(rowsSlicePtr interface{}, condiBean ...interface{}) } args = append(session.statement.joinArgs, condArgs...) - sqlStr, err = session.statement.genSelectSQL(columnStr, condSQL, true, true) + sqlStr, err = session.statement.genSelectSQL(columnStr, condSQL) if err != nil { return err } @@ -176,7 +143,7 @@ func (session *Session) find(rowsSlicePtr interface{}, condiBean ...interface{}) } if session.canCache() { - if cacher := session.engine.getCacher(table.Name); cacher != nil && + if cacher := session.engine.getCacher2(table); cacher != nil && !session.statement.IsDistinct && !session.statement.unscoped { err = session.cacheFind(sliceElementType, sqlStr, rowsSlicePtr, args...) @@ -321,12 +288,6 @@ func (session *Session) cacheFind(t reflect.Type, sqlStr string, rowsSlicePtr in return ErrCacheFailed } - tableName := session.statement.TableName() - cacher := session.engine.getCacher(tableName) - if cacher == nil { - return nil - } - for _, filter := range session.engine.dialect.Filters() { sqlStr = filter.Do(sqlStr, session.engine.dialect, session.statement.RefTable) } @@ -336,7 +297,9 @@ func (session *Session) cacheFind(t reflect.Type, sqlStr string, rowsSlicePtr in return ErrCacheFailed } + tableName := session.statement.TableName() table := session.statement.RefTable + cacher := session.engine.getCacher2(table) ids, err := core.GetCacheSql(cacher, tableName, newsql, args) if err != nil { rows, err := session.queryRows(newsql, args...) diff --git a/vendor/github.com/go-xorm/xorm/session_get.go b/vendor/github.com/go-xorm/xorm/session_get.go index 3b2c9493c26..8faf53c02c7 100644 --- a/vendor/github.com/go-xorm/xorm/session_get.go +++ b/vendor/github.com/go-xorm/xorm/session_get.go @@ -5,7 +5,6 @@ package xorm import ( - "database/sql" "errors" "reflect" "strconv" @@ -31,7 +30,7 @@ func (session *Session) get(bean interface{}) (bool, error) { } if beanValue.Elem().Kind() == reflect.Struct { - if err := session.statement.setRefBean(bean); err != nil { + if err := session.statement.setRefValue(beanValue.Elem()); err != nil { return false, err } } @@ -57,7 +56,7 @@ func (session *Session) get(bean interface{}) (bool, error) { table := session.statement.RefTable if session.canCache() && beanValue.Elem().Kind() == reflect.Struct { - if cacher := session.engine.getCacher(table.Name); cacher != nil && + if cacher := session.engine.getCacher2(table); cacher != nil && !session.statement.unscoped { has, err := session.cacheGet(bean, sqlStr, args...) if err != ErrCacheFailed { @@ -80,13 +79,6 @@ func (session *Session) nocacheGet(beanKind reflect.Kind, table *core.Table, bea return false, nil } - switch bean.(type) { - case sql.NullInt64, sql.NullBool, sql.NullFloat64, sql.NullString: - return true, rows.Scan(&bean) - case *sql.NullInt64, *sql.NullBool, *sql.NullFloat64, *sql.NullString: - return true, rows.Scan(bean) - } - switch beanKind { case reflect.Struct: fields, err := rows.Columns() @@ -134,9 +126,8 @@ func (session *Session) cacheGet(bean interface{}, sqlStr string, args ...interf return false, ErrCacheFailed } + cacher := session.engine.getCacher2(session.statement.RefTable) tableName := session.statement.TableName() - cacher := session.engine.getCacher(tableName) - session.engine.logger.Debug("[cacheGet] find sql:", newsql, args) table := session.statement.RefTable ids, err := core.GetCacheSql(cacher, tableName, newsql, args) diff --git a/vendor/github.com/go-xorm/xorm/session_insert.go b/vendor/github.com/go-xorm/xorm/session_insert.go index c1182fe64f8..129ee23098a 100644 --- a/vendor/github.com/go-xorm/xorm/session_insert.go +++ b/vendor/github.com/go-xorm/xorm/session_insert.go @@ -66,12 +66,11 @@ func (session *Session) innerInsertMulti(rowsSlicePtr interface{}) (int64, error return 0, errors.New("could not insert a empty slice") } - if err := session.statement.setRefBean(sliceValue.Index(0).Interface()); err != nil { + if err := session.statement.setRefValue(reflect.ValueOf(sliceValue.Index(0).Interface())); err != nil { return 0, err } - tableName := session.statement.TableName() - if len(tableName) <= 0 { + if len(session.statement.TableName()) <= 0 { return 0, ErrTableNotFound } @@ -116,11 +115,15 @@ func (session *Session) innerInsertMulti(rowsSlicePtr interface{}) (int64, error if col.IsDeleted { continue } - if session.statement.omitColumnMap.contain(col.Name) { - continue + if session.statement.ColumnStr != "" { + if _, ok := getFlagForColumn(session.statement.columnMap, col); !ok { + continue + } } - if len(session.statement.columnMap) > 0 && !session.statement.columnMap.contain(col.Name) { - continue + if session.statement.OmitStr != "" { + if _, ok := getFlagForColumn(session.statement.columnMap, col); ok { + continue + } } if (col.IsCreated || col.IsUpdated) && session.statement.UseAutoTime { val, t := session.engine.nowTime(col) @@ -167,11 +170,15 @@ func (session *Session) innerInsertMulti(rowsSlicePtr interface{}) (int64, error if col.IsDeleted { continue } - if session.statement.omitColumnMap.contain(col.Name) { - continue + if session.statement.ColumnStr != "" { + if _, ok := getFlagForColumn(session.statement.columnMap, col); !ok { + continue + } } - if len(session.statement.columnMap) > 0 && !session.statement.columnMap.contain(col.Name) { - continue + if session.statement.OmitStr != "" { + if _, ok := getFlagForColumn(session.statement.columnMap, col); ok { + continue + } } if (col.IsCreated || col.IsUpdated) && session.statement.UseAutoTime { val, t := session.engine.nowTime(col) @@ -206,6 +213,7 @@ func (session *Session) innerInsertMulti(rowsSlicePtr interface{}) (int64, error var sql = "INSERT INTO %s (%v%v%v) VALUES (%v)" var statement string + var tableName = session.statement.TableName() if session.engine.dialect.DBType() == core.ORACLE { sql = "INSERT ALL INTO %s (%v%v%v) VALUES (%v) SELECT 1 FROM DUAL" temp := fmt.Sprintf(") INTO %s (%v%v%v) VALUES (", @@ -232,7 +240,9 @@ func (session *Session) innerInsertMulti(rowsSlicePtr interface{}) (int64, error return 0, err } - session.cacheInsert(tableName) + if cacher := session.engine.getCacher2(table); cacher != nil && session.statement.UseCache { + session.cacheInsert(table, tableName) + } lenAfterClosures := len(session.afterClosures) for i := 0; i < size; i++ { @@ -288,7 +298,7 @@ func (session *Session) InsertMulti(rowsSlicePtr interface{}) (int64, error) { } func (session *Session) innerInsert(bean interface{}) (int64, error) { - if err := session.statement.setRefBean(bean); err != nil { + if err := session.statement.setRefValue(rValue(bean)); err != nil { return 0, err } if len(session.statement.TableName()) <= 0 { @@ -306,8 +316,8 @@ func (session *Session) innerInsert(bean interface{}) (int64, error) { if processor, ok := interface{}(bean).(BeforeInsertProcessor); ok { processor.BeforeInsert() } - - colNames, args, err := session.genInsertColumns(bean) + // -- + colNames, args, err := genCols(session.statement.RefTable, session, bean, false, false) if err != nil { return 0, err } @@ -392,7 +402,9 @@ func (session *Session) innerInsert(bean interface{}) (int64, error) { defer handleAfterInsertProcessorFunc(bean) - session.cacheInsert(tableName) + if cacher := session.engine.getCacher2(table); cacher != nil && session.statement.UseCache { + session.cacheInsert(table, tableName) + } if table.Version != "" && session.statement.checkVersion { verValue, err := table.VersionColumn().ValueOf(bean) @@ -435,7 +447,9 @@ func (session *Session) innerInsert(bean interface{}) (int64, error) { } defer handleAfterInsertProcessorFunc(bean) - session.cacheInsert(tableName) + if cacher := session.engine.getCacher2(table); cacher != nil && session.statement.UseCache { + session.cacheInsert(table, tableName) + } if table.Version != "" && session.statement.checkVersion { verValue, err := table.VersionColumn().ValueOf(bean) @@ -476,7 +490,9 @@ func (session *Session) innerInsert(bean interface{}) (int64, error) { defer handleAfterInsertProcessorFunc(bean) - session.cacheInsert(tableName) + if cacher := session.engine.getCacher2(table); cacher != nil && session.statement.UseCache { + session.cacheInsert(table, tableName) + } if table.Version != "" && session.statement.checkVersion { verValue, err := table.VersionColumn().ValueOf(bean) @@ -523,104 +539,16 @@ func (session *Session) InsertOne(bean interface{}) (int64, error) { return session.innerInsert(bean) } -func (session *Session) cacheInsert(table string) error { - if !session.statement.UseCache { - return nil +func (session *Session) cacheInsert(table *core.Table, tables ...string) error { + if table == nil { + return ErrCacheFailed } - cacher := session.engine.getCacher(table) - if cacher == nil { - return nil + + cacher := session.engine.getCacher2(table) + for _, t := range tables { + session.engine.logger.Debug("[cache] clear sql:", t) + cacher.ClearIds(t) } - session.engine.logger.Debug("[cache] clear sql:", table) - cacher.ClearIds(table) + return nil } - -// genInsertColumns generates insert needed columns -func (session *Session) genInsertColumns(bean interface{}) ([]string, []interface{}, error) { - table := session.statement.RefTable - colNames := make([]string, 0, len(table.ColumnsSeq())) - args := make([]interface{}, 0, len(table.ColumnsSeq())) - - for _, col := range table.Columns() { - if col.MapType == core.ONLYFROMDB { - continue - } - - if col.IsDeleted { - continue - } - - if session.statement.omitColumnMap.contain(col.Name) { - continue - } - - if len(session.statement.columnMap) > 0 && !session.statement.columnMap.contain(col.Name) { - continue - } - - if _, ok := session.statement.incrColumns[col.Name]; ok { - continue - } else if _, ok := session.statement.decrColumns[col.Name]; ok { - continue - } - - fieldValuePtr, err := col.ValueOf(bean) - if err != nil { - return nil, nil, err - } - fieldValue := *fieldValuePtr - - if col.IsAutoIncrement { - switch fieldValue.Type().Kind() { - case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int, reflect.Int64: - if fieldValue.Int() == 0 { - continue - } - case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint, reflect.Uint64: - if fieldValue.Uint() == 0 { - continue - } - case reflect.String: - if len(fieldValue.String()) == 0 { - continue - } - case reflect.Ptr: - if fieldValue.Pointer() == 0 { - continue - } - } - } - - // !evalphobia! set fieldValue as nil when column is nullable and zero-value - if _, ok := getFlagForColumn(session.statement.nullableMap, col); ok { - if col.Nullable && isZero(fieldValue.Interface()) { - var nilValue *int - fieldValue = reflect.ValueOf(nilValue) - } - } - - if (col.IsCreated || col.IsUpdated) && session.statement.UseAutoTime /*&& isZero(fieldValue.Interface())*/ { - // if time is non-empty, then set to auto time - val, t := session.engine.nowTime(col) - args = append(args, val) - - var colName = col.Name - session.afterClosures = append(session.afterClosures, func(bean interface{}) { - col := table.GetColumn(colName) - setColumnTime(bean, col, t) - }) - } else if col.IsVersion && session.statement.checkVersion { - args = append(args, 1) - } else { - arg, err := session.value2Interface(col, fieldValue) - if err != nil { - return colNames, args, err - } - args = append(args, arg) - } - - colNames = append(colNames, col.Name) - } - return colNames, args, nil -} diff --git a/vendor/github.com/go-xorm/xorm/session_query.go b/vendor/github.com/go-xorm/xorm/session_query.go index 5c9aeb3916c..5b4e0dc45d0 100644 --- a/vendor/github.com/go-xorm/xorm/session_query.go +++ b/vendor/github.com/go-xorm/xorm/session_query.go @@ -17,17 +17,7 @@ import ( func (session *Session) genQuerySQL(sqlorArgs ...interface{}) (string, []interface{}, error) { if len(sqlorArgs) > 0 { - switch sqlorArgs[0].(type) { - case string: - return sqlorArgs[0].(string), sqlorArgs[1:], nil - case *builder.Builder: - return sqlorArgs[0].(*builder.Builder).ToSQL() - case builder.Builder: - bd := sqlorArgs[0].(builder.Builder) - return bd.ToSQL() - default: - return "", nil, ErrUnSupportedType - } + return sqlorArgs[0].(string), sqlorArgs[1:], nil } if session.statement.RawSQL != "" { @@ -64,17 +54,13 @@ func (session *Session) genQuerySQL(sqlorArgs ...interface{}) (string, []interfa } } - if err := session.statement.processIDParam(); err != nil { - return "", nil, err - } - condSQL, condArgs, err := builder.ToSQL(session.statement.cond) if err != nil { return "", nil, err } args := append(session.statement.joinArgs, condArgs...) - sqlStr, err := session.statement.genSelectSQL(columnStr, condSQL, true, true) + sqlStr, err := session.statement.genSelectSQL(columnStr, condSQL) if err != nil { return "", nil, err } diff --git a/vendor/github.com/go-xorm/xorm/session_schema.go b/vendor/github.com/go-xorm/xorm/session_schema.go index f06286614e3..a2708b736c0 100644 --- a/vendor/github.com/go-xorm/xorm/session_schema.go +++ b/vendor/github.com/go-xorm/xorm/session_schema.go @@ -6,7 +6,9 @@ package xorm import ( "database/sql" + "errors" "fmt" + "reflect" "strings" "github.com/go-xorm/core" @@ -32,7 +34,8 @@ func (session *Session) CreateTable(bean interface{}) error { } func (session *Session) createTable(bean interface{}) error { - if err := session.statement.setRefBean(bean); err != nil { + v := rValue(bean) + if err := session.statement.setRefValue(v); err != nil { return err } @@ -51,7 +54,8 @@ func (session *Session) CreateIndexes(bean interface{}) error { } func (session *Session) createIndexes(bean interface{}) error { - if err := session.statement.setRefBean(bean); err != nil { + v := rValue(bean) + if err := session.statement.setRefValue(v); err != nil { return err } @@ -74,7 +78,8 @@ func (session *Session) CreateUniques(bean interface{}) error { } func (session *Session) createUniques(bean interface{}) error { - if err := session.statement.setRefBean(bean); err != nil { + v := rValue(bean) + if err := session.statement.setRefValue(v); err != nil { return err } @@ -98,7 +103,8 @@ func (session *Session) DropIndexes(bean interface{}) error { } func (session *Session) dropIndexes(bean interface{}) error { - if err := session.statement.setRefBean(bean); err != nil { + v := rValue(bean) + if err := session.statement.setRefValue(v); err != nil { return err } @@ -122,7 +128,11 @@ func (session *Session) DropTable(beanOrTableName interface{}) error { } func (session *Session) dropTable(beanOrTableName interface{}) error { - tableName := session.engine.tbNameNoSchema(beanOrTableName) + tableName, err := session.engine.tableName(beanOrTableName) + if err != nil { + return err + } + var needDrop = true if !session.engine.dialect.SupportDropIfExists() { sqlStr, args := session.engine.dialect.TableCheckSql(tableName) @@ -134,8 +144,8 @@ func (session *Session) dropTable(beanOrTableName interface{}) error { } if needDrop { - sqlStr := session.engine.Dialect().DropTableSql(session.engine.TableName(tableName, true)) - _, err := session.exec(sqlStr) + sqlStr := session.engine.Dialect().DropTableSql(tableName) + _, err = session.exec(sqlStr) return err } return nil @@ -147,7 +157,10 @@ func (session *Session) IsTableExist(beanOrTableName interface{}) (bool, error) defer session.Close() } - tableName := session.engine.tbNameNoSchema(beanOrTableName) + tableName, err := session.engine.tableName(beanOrTableName) + if err != nil { + return false, err + } return session.isTableExist(tableName) } @@ -160,15 +173,24 @@ func (session *Session) isTableExist(tableName string) (bool, error) { // IsTableEmpty if table have any records func (session *Session) IsTableEmpty(bean interface{}) (bool, error) { - if session.isAutoClose { - defer session.Close() + v := rValue(bean) + t := v.Type() + + if t.Kind() == reflect.String { + if session.isAutoClose { + defer session.Close() + } + return session.isTableEmpty(bean.(string)) + } else if t.Kind() == reflect.Struct { + rows, err := session.Count(bean) + return rows == 0, err } - return session.isTableEmpty(session.engine.tbNameNoSchema(bean)) + return false, errors.New("bean should be a struct or struct's point") } func (session *Session) isTableEmpty(tableName string) (bool, error) { var total int64 - sqlStr := fmt.Sprintf("select count(*) from %s", session.engine.Quote(session.engine.TableName(tableName, true))) + sqlStr := fmt.Sprintf("select count(*) from %s", session.engine.Quote(tableName)) err := session.queryRow(sqlStr).Scan(&total) if err != nil { if err == sql.ErrNoRows { @@ -233,12 +255,6 @@ func (session *Session) Sync2(beans ...interface{}) error { return err } - session.autoResetStatement = false - defer func() { - session.autoResetStatement = true - session.resetStatement() - }() - var structTables []*core.Table for _, bean := range beans { @@ -248,8 +264,7 @@ func (session *Session) Sync2(beans ...interface{}) error { return err } structTables = append(structTables, table) - tbName := session.tbNameNoSchema(table) - tbNameWithSchema := engine.TableName(tbName, true) + var tbName = session.tbNameNoSchema(table) var oriTable *core.Table for _, tb := range tables { @@ -294,32 +309,32 @@ func (session *Session) Sync2(beans ...interface{}) error { if engine.dialect.DBType() == core.MYSQL || engine.dialect.DBType() == core.POSTGRES { engine.logger.Infof("Table %s column %s change type from %s to %s\n", - tbNameWithSchema, col.Name, curType, expectedType) - _, err = session.exec(engine.dialect.ModifyColumnSql(tbNameWithSchema, col)) + tbName, col.Name, curType, expectedType) + _, err = session.exec(engine.dialect.ModifyColumnSql(table.Name, col)) } else { engine.logger.Warnf("Table %s column %s db type is %s, struct type is %s\n", - tbNameWithSchema, col.Name, curType, expectedType) + tbName, col.Name, curType, expectedType) } } else if strings.HasPrefix(curType, core.Varchar) && strings.HasPrefix(expectedType, core.Varchar) { if engine.dialect.DBType() == core.MYSQL { if oriCol.Length < col.Length { engine.logger.Infof("Table %s column %s change type from varchar(%d) to varchar(%d)\n", - tbNameWithSchema, col.Name, oriCol.Length, col.Length) - _, err = session.exec(engine.dialect.ModifyColumnSql(tbNameWithSchema, col)) + tbName, col.Name, oriCol.Length, col.Length) + _, err = session.exec(engine.dialect.ModifyColumnSql(table.Name, col)) } } } else { if !(strings.HasPrefix(curType, expectedType) && curType[len(expectedType)] == '(') { engine.logger.Warnf("Table %s column %s db type is %s, struct type is %s", - tbNameWithSchema, col.Name, curType, expectedType) + tbName, col.Name, curType, expectedType) } } } else if expectedType == core.Varchar { if engine.dialect.DBType() == core.MYSQL { if oriCol.Length < col.Length { engine.logger.Infof("Table %s column %s change type from varchar(%d) to varchar(%d)\n", - tbNameWithSchema, col.Name, oriCol.Length, col.Length) - _, err = session.exec(engine.dialect.ModifyColumnSql(tbNameWithSchema, col)) + tbName, col.Name, oriCol.Length, col.Length) + _, err = session.exec(engine.dialect.ModifyColumnSql(table.Name, col)) } } } @@ -333,7 +348,7 @@ func (session *Session) Sync2(beans ...interface{}) error { } } else { session.statement.RefTable = table - session.statement.tableName = tbNameWithSchema + session.statement.tableName = tbName err = session.addColumn(col.Name) } if err != nil { @@ -356,7 +371,7 @@ func (session *Session) Sync2(beans ...interface{}) error { if oriIndex != nil { if oriIndex.Type != index.Type { - sql := engine.dialect.DropIndexSql(tbNameWithSchema, oriIndex) + sql := engine.dialect.DropIndexSql(tbName, oriIndex) _, err = session.exec(sql) if err != nil { return err @@ -372,7 +387,7 @@ func (session *Session) Sync2(beans ...interface{}) error { for name2, index2 := range oriTable.Indexes { if _, ok := foundIndexNames[name2]; !ok { - sql := engine.dialect.DropIndexSql(tbNameWithSchema, index2) + sql := engine.dialect.DropIndexSql(tbName, index2) _, err = session.exec(sql) if err != nil { return err @@ -383,12 +398,12 @@ func (session *Session) Sync2(beans ...interface{}) error { for name, index := range addedNames { if index.Type == core.UniqueType { session.statement.RefTable = table - session.statement.tableName = tbNameWithSchema - err = session.addUnique(tbNameWithSchema, name) + session.statement.tableName = tbName + err = session.addUnique(tbName, name) } else if index.Type == core.IndexType { session.statement.RefTable = table - session.statement.tableName = tbNameWithSchema - err = session.addIndex(tbNameWithSchema, name) + session.statement.tableName = tbName + err = session.addIndex(tbName, name) } if err != nil { return err @@ -413,7 +428,7 @@ func (session *Session) Sync2(beans ...interface{}) error { for _, colName := range table.ColumnsSeq() { if oriTable.GetColumn(colName) == nil { - engine.logger.Warnf("Table %s has column %s but struct has not related field", engine.TableName(table.Name, true), colName) + engine.logger.Warnf("Table %s has column %s but struct has not related field", table.Name, colName) } } } diff --git a/vendor/github.com/go-xorm/xorm/session_update.go b/vendor/github.com/go-xorm/xorm/session_update.go index 84c7e7fecff..f558745667f 100644 --- a/vendor/github.com/go-xorm/xorm/session_update.go +++ b/vendor/github.com/go-xorm/xorm/session_update.go @@ -40,7 +40,7 @@ func (session *Session) cacheUpdate(table *core.Table, tableName, sqlStr string, } } - cacher := session.engine.getCacher(tableName) + cacher := session.engine.getCacher2(table) session.engine.logger.Debug("[cacheUpdate] get cache sql", newsql, args[nStart:]) ids, err := core.GetCacheSql(cacher, tableName, newsql, args[nStart:]) if err != nil { @@ -167,7 +167,7 @@ func (session *Session) Update(bean interface{}, condiBean ...interface{}) (int6 var isMap = t.Kind() == reflect.Map var isStruct = t.Kind() == reflect.Struct if isStruct { - if err := session.statement.setRefBean(bean); err != nil { + if err := session.statement.setRefValue(v); err != nil { return 0, err } @@ -176,10 +176,12 @@ func (session *Session) Update(bean interface{}, condiBean ...interface{}) (int6 } if session.statement.ColumnStr == "" { - colNames, args = session.statement.buildUpdates(bean, false, false, - false, false, true) + colNames, args = buildUpdates(session.engine, session.statement.RefTable, bean, false, false, + false, false, session.statement.allUseBool, session.statement.useAllCols, + session.statement.mustColumnMap, session.statement.nullableMap, + session.statement.columnMap, true, session.statement.unscoped) } else { - colNames, args, err = session.genUpdateColumns(bean) + colNames, args, err = genCols(session.statement.RefTable, session, bean, true, true) if err != nil { return 0, err } @@ -200,8 +202,7 @@ func (session *Session) Update(bean interface{}, condiBean ...interface{}) (int6 table := session.statement.RefTable if session.statement.UseAutoTime && table != nil && table.Updated != "" { - if !session.statement.columnMap.contain(table.Updated) && - !session.statement.omitColumnMap.contain(table.Updated) { + if _, ok := session.statement.columnMap[strings.ToLower(table.Updated)]; !ok { colNames = append(colNames, session.engine.Quote(table.Updated)+" = ?") col := table.UpdatedColumn() val, t := session.engine.nowTime(col) @@ -361,11 +362,12 @@ func (session *Session) Update(bean interface{}, condiBean ...interface{}) (int6 } } - if cacher := session.engine.getCacher(tableName); cacher != nil && session.statement.UseCache { - //session.cacheUpdate(table, tableName, sqlStr, args...) - session.engine.logger.Debug("[cacheUpdate] clear table ", tableName) - cacher.ClearIds(tableName) - cacher.ClearBeans(tableName) + if table != nil { + if cacher := session.engine.getCacher2(table); cacher != nil && session.statement.UseCache { + //session.cacheUpdate(table, tableName, sqlStr, args...) + cacher.ClearIds(tableName) + cacher.ClearBeans(tableName) + } } // handle after update processors @@ -400,92 +402,3 @@ func (session *Session) Update(bean interface{}, condiBean ...interface{}) (int6 return res.RowsAffected() } - -func (session *Session) genUpdateColumns(bean interface{}) ([]string, []interface{}, error) { - table := session.statement.RefTable - colNames := make([]string, 0, len(table.ColumnsSeq())) - args := make([]interface{}, 0, len(table.ColumnsSeq())) - - for _, col := range table.Columns() { - if !col.IsVersion && !col.IsCreated && !col.IsUpdated { - if session.statement.omitColumnMap.contain(col.Name) { - continue - } - } - if col.MapType == core.ONLYFROMDB { - continue - } - - fieldValuePtr, err := col.ValueOf(bean) - if err != nil { - return nil, nil, err - } - fieldValue := *fieldValuePtr - - if col.IsAutoIncrement { - switch fieldValue.Type().Kind() { - case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int, reflect.Int64: - if fieldValue.Int() == 0 { - continue - } - case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint, reflect.Uint64: - if fieldValue.Uint() == 0 { - continue - } - case reflect.String: - if len(fieldValue.String()) == 0 { - continue - } - case reflect.Ptr: - if fieldValue.Pointer() == 0 { - continue - } - } - } - - if col.IsDeleted || col.IsCreated { - continue - } - - if len(session.statement.columnMap) > 0 { - if !session.statement.columnMap.contain(col.Name) { - continue - } else if _, ok := session.statement.incrColumns[col.Name]; ok { - continue - } else if _, ok := session.statement.decrColumns[col.Name]; ok { - continue - } - } - - // !evalphobia! set fieldValue as nil when column is nullable and zero-value - if _, ok := getFlagForColumn(session.statement.nullableMap, col); ok { - if col.Nullable && isZero(fieldValue.Interface()) { - var nilValue *int - fieldValue = reflect.ValueOf(nilValue) - } - } - - if col.IsUpdated && session.statement.UseAutoTime /*&& isZero(fieldValue.Interface())*/ { - // if time is non-empty, then set to auto time - val, t := session.engine.nowTime(col) - args = append(args, val) - - var colName = col.Name - session.afterClosures = append(session.afterClosures, func(bean interface{}) { - col := table.GetColumn(colName) - setColumnTime(bean, col, t) - }) - } else if col.IsVersion && session.statement.checkVersion { - args = append(args, 1) - } else { - arg, err := session.value2Interface(col, fieldValue) - if err != nil { - return colNames, args, err - } - args = append(args, arg) - } - - colNames = append(colNames, session.engine.Quote(col.Name)+" = ?") - } - return colNames, args, nil -} diff --git a/vendor/github.com/go-xorm/xorm/statement.go b/vendor/github.com/go-xorm/xorm/statement.go index 38fa26d2e92..6400425b20e 100644 --- a/vendor/github.com/go-xorm/xorm/statement.go +++ b/vendor/github.com/go-xorm/xorm/statement.go @@ -18,6 +18,21 @@ import ( "github.com/go-xorm/core" ) +type incrParam struct { + colName string + arg interface{} +} + +type decrParam struct { + colName string + arg interface{} +} + +type exprParam struct { + colName string + expr string +} + // Statement save all the sql info for executing SQL type Statement struct { RefTable *core.Table @@ -32,6 +47,7 @@ type Statement struct { HavingStr string ColumnStr string selectStr string + columnMap map[string]bool useAllCols bool OmitStr string AltTableName string @@ -51,8 +67,6 @@ type Statement struct { allUseBool bool checkVersion bool unscoped bool - columnMap columnMap - omitColumnMap columnMap mustColumnMap map[string]bool nullableMap map[string]bool incrColumns map[string]incrParam @@ -75,8 +89,7 @@ func (statement *Statement) Init() { statement.HavingStr = "" statement.ColumnStr = "" statement.OmitStr = "" - statement.columnMap = columnMap{} - statement.omitColumnMap = columnMap{} + statement.columnMap = make(map[string]bool) statement.AltTableName = "" statement.tableName = "" statement.idParam = nil @@ -208,33 +221,34 @@ func (statement *Statement) setRefValue(v reflect.Value) error { if err != nil { return err } - statement.tableName = statement.Engine.TableName(v, true) + statement.tableName = statement.Engine.tbName(v) return nil } -func (statement *Statement) setRefBean(bean interface{}) error { - var err error - statement.RefTable, err = statement.Engine.autoMapType(rValue(bean)) - if err != nil { - return err +// Table tempororily set table name, the parameter could be a string or a pointer of struct +func (statement *Statement) Table(tableNameOrBean interface{}) *Statement { + v := rValue(tableNameOrBean) + t := v.Type() + if t.Kind() == reflect.String { + statement.AltTableName = tableNameOrBean.(string) + } else if t.Kind() == reflect.Struct { + var err error + statement.RefTable, err = statement.Engine.autoMapType(v) + if err != nil { + statement.Engine.logger.Error(err) + return statement + } + statement.AltTableName = statement.Engine.tbName(v) } - statement.tableName = statement.Engine.TableName(bean, true) - return nil + return statement } // Auto generating update columnes and values according a struct -func (statement *Statement) buildUpdates(bean interface{}, - includeVersion, includeUpdated, includeNil, - includeAutoIncr, update bool) ([]string, []interface{}) { - engine := statement.Engine - table := statement.RefTable - allUseBool := statement.allUseBool - useAllCols := statement.useAllCols - mustColumnMap := statement.mustColumnMap - nullableMap := statement.nullableMap - columnMap := statement.columnMap - omitColumnMap := statement.omitColumnMap - unscoped := statement.unscoped +func buildUpdates(engine *Engine, table *core.Table, bean interface{}, + includeVersion bool, includeUpdated bool, includeNil bool, + includeAutoIncr bool, allUseBool bool, useAllCols bool, + mustColumnMap map[string]bool, nullableMap map[string]bool, + columnMap map[string]bool, update, unscoped bool) ([]string, []interface{}) { var colNames = make([]string, 0) var args = make([]interface{}, 0) @@ -254,10 +268,7 @@ func (statement *Statement) buildUpdates(bean interface{}, if col.IsDeleted && !unscoped { continue } - if omitColumnMap.contain(col.Name) { - continue - } - if len(columnMap) > 0 && !columnMap.contain(col.Name) { + if use, ok := columnMap[strings.ToLower(col.Name)]; ok && !use { continue } @@ -593,10 +604,17 @@ func (statement *Statement) col2NewColsWithQuote(columns ...string) []string { } func (statement *Statement) colmap2NewColsWithQuote() []string { - newColumns := make([]string, len(statement.columnMap), len(statement.columnMap)) - copy(newColumns, statement.columnMap) - for i := 0; i < len(statement.columnMap); i++ { - newColumns[i] = statement.Engine.Quote(newColumns[i]) + newColumns := make([]string, 0, len(statement.columnMap)) + for col := range statement.columnMap { + fields := strings.Split(strings.TrimSpace(col), ".") + if len(fields) == 1 { + newColumns = append(newColumns, statement.Engine.quote(fields[0])) + } else if len(fields) == 2 { + newColumns = append(newColumns, statement.Engine.quote(fields[0])+"."+ + statement.Engine.quote(fields[1])) + } else { + panic(errors.New("unwanted colnames")) + } } return newColumns } @@ -624,11 +642,10 @@ func (statement *Statement) Select(str string) *Statement { func (statement *Statement) Cols(columns ...string) *Statement { cols := col2NewCols(columns...) for _, nc := range cols { - statement.columnMap = append(statement.columnMap, nc) + statement.columnMap[strings.ToLower(nc)] = true } newColumns := statement.colmap2NewColsWithQuote() - statement.ColumnStr = strings.Join(newColumns, ", ") statement.ColumnStr = strings.Replace(statement.ColumnStr, statement.Engine.quote("*"), "*", -1) return statement @@ -663,7 +680,7 @@ func (statement *Statement) UseBool(columns ...string) *Statement { func (statement *Statement) Omit(columns ...string) { newColumns := col2NewCols(columns...) for _, nc := range newColumns { - statement.omitColumnMap = append(statement.omitColumnMap, nc) + statement.columnMap[strings.ToLower(nc)] = false } statement.OmitStr = statement.Engine.Quote(strings.Join(newColumns, statement.Engine.Quote(", "))) } @@ -726,23 +743,6 @@ func (statement *Statement) Asc(colNames ...string) *Statement { return statement } -// Table tempororily set table name, the parameter could be a string or a pointer of struct -func (statement *Statement) Table(tableNameOrBean interface{}) *Statement { - v := rValue(tableNameOrBean) - t := v.Type() - if t.Kind() == reflect.Struct { - var err error - statement.RefTable, err = statement.Engine.autoMapType(v) - if err != nil { - statement.Engine.logger.Error(err) - return statement - } - } - - statement.AltTableName = statement.Engine.TableName(tableNameOrBean, true) - return statement -} - // Join The joinOP should be one of INNER, LEFT OUTER, CROSS etc - this will be prepended to JOIN func (statement *Statement) Join(joinOP string, tablename interface{}, condition string, args ...interface{}) *Statement { var buf bytes.Buffer @@ -752,9 +752,39 @@ func (statement *Statement) Join(joinOP string, tablename interface{}, condition fmt.Fprintf(&buf, "%v JOIN ", joinOP) } - tbName := statement.Engine.TableName(tablename, true) + switch tablename.(type) { + case []string: + t := tablename.([]string) + if len(t) > 1 { + fmt.Fprintf(&buf, "%v AS %v", statement.Engine.Quote(t[0]), statement.Engine.Quote(t[1])) + } else if len(t) == 1 { + fmt.Fprintf(&buf, statement.Engine.Quote(t[0])) + } + case []interface{}: + t := tablename.([]interface{}) + l := len(t) + var table string + if l > 0 { + f := t[0] + v := rValue(f) + t := v.Type() + if t.Kind() == reflect.String { + table = f.(string) + } else if t.Kind() == reflect.Struct { + table = statement.Engine.tbName(v) + } + } + if l > 1 { + fmt.Fprintf(&buf, "%v AS %v", statement.Engine.Quote(table), + statement.Engine.Quote(fmt.Sprintf("%v", t[1]))) + } else if l == 1 { + fmt.Fprintf(&buf, statement.Engine.Quote(table)) + } + default: + fmt.Fprintf(&buf, statement.Engine.Quote(fmt.Sprintf("%v", tablename))) + } - fmt.Fprintf(&buf, "%s ON %v", tbName, condition) + fmt.Fprintf(&buf, " ON %v", condition) statement.JoinStr = buf.String() statement.joinArgs = append(statement.joinArgs, args...) return statement @@ -787,12 +817,10 @@ func (statement *Statement) genColumnStr() string { columns := statement.RefTable.Columns() for _, col := range columns { - if statement.omitColumnMap.contain(col.Name) { - continue - } - - if len(statement.columnMap) > 0 && !statement.columnMap.contain(col.Name) { - continue + if statement.OmitStr != "" { + if _, ok := getFlagForColumn(statement.columnMap, col); ok { + continue + } } if col.MapType == core.ONLYTODB { @@ -803,6 +831,10 @@ func (statement *Statement) genColumnStr() string { buf.WriteString(", ") } + if col.IsPrimaryKey && statement.Engine.Dialect().DBType() == "ql" { + buf.WriteString("id() AS ") + } + if statement.JoinStr != "" { if statement.TableAlias != "" { buf.WriteString(statement.TableAlias) @@ -827,13 +859,11 @@ func (statement *Statement) genCreateTableSQL() string { func (statement *Statement) genIndexSQL() []string { var sqls []string tbName := statement.TableName() - for _, index := range statement.RefTable.Indexes { + quote := statement.Engine.Quote + for idxName, index := range statement.RefTable.Indexes { if index.Type == core.IndexType { - sql := statement.Engine.dialect.CreateIndexSql(tbName, index) - /*idxTBName := strings.Replace(tbName, ".", "_", -1) - idxTBName = strings.Replace(idxTBName, `"`, "", -1) - sql := fmt.Sprintf("CREATE INDEX %v ON %v (%v);", quote(indexName(idxTBName, idxName)), - quote(tbName), quote(strings.Join(index.Cols, quote(","))))*/ + sql := fmt.Sprintf("CREATE INDEX %v ON %v (%v);", quote(indexName(tbName, idxName)), + quote(tbName), quote(strings.Join(index.Cols, quote(",")))) sqls = append(sqls, sql) } } @@ -859,18 +889,16 @@ func (statement *Statement) genUniqueSQL() []string { func (statement *Statement) genDelIndexSQL() []string { var sqls []string tbName := statement.TableName() - idxPrefixName := strings.Replace(tbName, `"`, "", -1) - idxPrefixName = strings.Replace(idxPrefixName, `.`, "_", -1) for idxName, index := range statement.RefTable.Indexes { var rIdxName string if index.Type == core.UniqueType { - rIdxName = uniqueName(idxPrefixName, idxName) + rIdxName = uniqueName(tbName, idxName) } else if index.Type == core.IndexType { - rIdxName = indexName(idxPrefixName, idxName) + rIdxName = indexName(tbName, idxName) } - sql := fmt.Sprintf("DROP INDEX %v", statement.Engine.Quote(statement.Engine.TableName(rIdxName, true))) + sql := fmt.Sprintf("DROP INDEX %v", statement.Engine.Quote(rIdxName)) if statement.Engine.dialect.IndexOnTable() { - sql += fmt.Sprintf(" ON %v", statement.Engine.Quote(tbName)) + sql += fmt.Sprintf(" ON %v", statement.Engine.Quote(statement.TableName())) } sqls = append(sqls, sql) } @@ -921,7 +949,7 @@ func (statement *Statement) genGetSQL(bean interface{}) (string, []interface{}, v := rValue(bean) isStruct := v.Kind() == reflect.Struct if isStruct { - statement.setRefBean(bean) + statement.setRefValue(v) } var columnStr = statement.ColumnStr @@ -954,17 +982,13 @@ func (statement *Statement) genGetSQL(bean interface{}) (string, []interface{}, if err := statement.mergeConds(bean); err != nil { return "", nil, err } - } else { - if err := statement.processIDParam(); err != nil { - return "", nil, err - } } condSQL, condArgs, err := builder.ToSQL(statement.cond) if err != nil { return "", nil, err } - sqlStr, err := statement.genSelectSQL(columnStr, condSQL, true, true) + sqlStr, err := statement.genSelectSQL(columnStr, condSQL) if err != nil { return "", nil, err } @@ -977,7 +1001,7 @@ func (statement *Statement) genCountSQL(beans ...interface{}) (string, []interfa var condArgs []interface{} var err error if len(beans) > 0 { - statement.setRefBean(beans[0]) + statement.setRefValue(rValue(beans[0])) condSQL, condArgs, err = statement.genConds(beans[0]) } else { condSQL, condArgs, err = builder.ToSQL(statement.cond) @@ -994,7 +1018,7 @@ func (statement *Statement) genCountSQL(beans ...interface{}) (string, []interfa selectSQL = "count(*)" } } - sqlStr, err := statement.genSelectSQL(selectSQL, condSQL, false, false) + sqlStr, err := statement.genSelectSQL(selectSQL, condSQL) if err != nil { return "", nil, err } @@ -1003,7 +1027,7 @@ func (statement *Statement) genCountSQL(beans ...interface{}) (string, []interfa } func (statement *Statement) genSumSQL(bean interface{}, columns ...string) (string, []interface{}, error) { - statement.setRefBean(bean) + statement.setRefValue(rValue(bean)) var sumStrs = make([]string, 0, len(columns)) for _, colName := range columns { @@ -1019,7 +1043,7 @@ func (statement *Statement) genSumSQL(bean interface{}, columns ...string) (stri return "", nil, err } - sqlStr, err := statement.genSelectSQL(sumSelect, condSQL, true, true) + sqlStr, err := statement.genSelectSQL(sumSelect, condSQL) if err != nil { return "", nil, err } @@ -1027,7 +1051,7 @@ func (statement *Statement) genSumSQL(bean interface{}, columns ...string) (stri return sqlStr, append(statement.joinArgs, condArgs...), nil } -func (statement *Statement) genSelectSQL(columnStr, condSQL string, needLimit, needOrderBy bool) (a string, err error) { +func (statement *Statement) genSelectSQL(columnStr, condSQL string) (a string, err error) { var distinct string if statement.IsDistinct && !strings.HasPrefix(columnStr, "count") { distinct = "DISTINCT " @@ -1038,6 +1062,10 @@ func (statement *Statement) genSelectSQL(columnStr, condSQL string, needLimit, n var top string var mssqlCondi string + if err := statement.processIDParam(); err != nil { + return "", err + } + var buf bytes.Buffer if len(condSQL) > 0 { fmt.Fprintf(&buf, " WHERE %v", condSQL) @@ -1090,10 +1118,9 @@ func (statement *Statement) genSelectSQL(columnStr, condSQL string, needLimit, n } var orderStr string - if needOrderBy && len(statement.OrderStr) > 0 { + if len(statement.OrderStr) > 0 { orderStr = " ORDER BY " + statement.OrderStr } - var groupStr string if len(statement.GroupByStr) > 0 { groupStr = " GROUP BY " + statement.GroupByStr @@ -1119,20 +1146,18 @@ func (statement *Statement) genSelectSQL(columnStr, condSQL string, needLimit, n if statement.HavingStr != "" { a = fmt.Sprintf("%v %v", a, statement.HavingStr) } - if needOrderBy && statement.OrderStr != "" { + if statement.OrderStr != "" { a = fmt.Sprintf("%v ORDER BY %v", a, statement.OrderStr) } - if needLimit { - if dialect.DBType() != core.MSSQL && dialect.DBType() != core.ORACLE { - if statement.Start > 0 { - a = fmt.Sprintf("%v LIMIT %v OFFSET %v", a, statement.LimitN, statement.Start) - } else if statement.LimitN > 0 { - a = fmt.Sprintf("%v LIMIT %v", a, statement.LimitN) - } - } else if dialect.DBType() == core.ORACLE { - if statement.Start != 0 || statement.LimitN != 0 { - a = fmt.Sprintf("SELECT %v FROM (SELECT %v,ROWNUM RN FROM (%v) at WHERE ROWNUM <= %d) aat WHERE RN > %d", columnStr, columnStr, a, statement.Start+statement.LimitN, statement.Start) - } + if dialect.DBType() != core.MSSQL && dialect.DBType() != core.ORACLE { + if statement.Start > 0 { + a = fmt.Sprintf("%v LIMIT %v OFFSET %v", a, statement.LimitN, statement.Start) + } else if statement.LimitN > 0 { + a = fmt.Sprintf("%v LIMIT %v", a, statement.LimitN) + } + } else if dialect.DBType() == core.ORACLE { + if statement.Start != 0 || statement.LimitN != 0 { + a = fmt.Sprintf("SELECT %v FROM (SELECT %v,ROWNUM RN FROM (%v) at WHERE ROWNUM <= %d) aat WHERE RN > %d", columnStr, columnStr, a, statement.Start+statement.LimitN, statement.Start) } } if statement.IsForUpdate { @@ -1143,7 +1168,7 @@ func (statement *Statement) genSelectSQL(columnStr, condSQL string, needLimit, n } func (statement *Statement) processIDParam() error { - if statement.idParam == nil || statement.RefTable == nil { + if statement.idParam == nil { return nil } diff --git a/vendor/github.com/go-xorm/xorm/xorm.go b/vendor/github.com/go-xorm/xorm/xorm.go index b1032b52637..4fdadf2fade 100644 --- a/vendor/github.com/go-xorm/xorm/xorm.go +++ b/vendor/github.com/go-xorm/xorm/xorm.go @@ -17,7 +17,7 @@ import ( const ( // Version show the xorm's version - Version string = "0.6.6.0413" + Version string = "0.6.4.0910" ) func regDrvsNDialects() bool { @@ -90,7 +90,6 @@ func NewEngine(driverName string, dataSourceName string) (*Engine, error) { TagIdentifier: "xorm", TZLocation: time.Local, tagHandlers: defaultTagHandlers, - cachers: make(map[string]core.Cacher), } if uri.DbType == core.SQLITE { @@ -109,13 +108,6 @@ func NewEngine(driverName string, dataSourceName string) (*Engine, error) { return engine, nil } -// NewEngineWithParams new a db manager with params. The params will be passed to dialect. -func NewEngineWithParams(driverName string, dataSourceName string, params map[string]string) (*Engine, error) { - engine, err := NewEngine(driverName, dataSourceName) - engine.dialect.SetParams(params) - return engine, err -} - // Clone clone an engine func (engine *Engine) Clone() (*Engine, error) { return NewEngine(engine.DriverName(), engine.DataSourceName()) From ced8c5f0e41486b5cfd7afdfe50827c20206e2ec Mon Sep 17 00:00:00 2001 From: David Date: Mon, 28 May 2018 13:46:57 +0200 Subject: [PATCH 37/87] Upgrade webpack loaders (#12081) * upgrade all webpack loaders to recent version * keep TS loader at 4.0.0 (5.0.0 requires webpack 4) * remove unused json-loader --- package.json | 15 +++++------ yarn.lock | 75 ++++++++++++++++------------------------------------ 2 files changed, 29 insertions(+), 61 deletions(-) diff --git a/package.json b/package.json index 06883a6c7ec..3b22d29beec 100644 --- a/package.json +++ b/package.json @@ -18,10 +18,9 @@ "@types/react-dom": "^16.0.3", "angular-mocks": "^1.6.6", "autoprefixer": "^6.4.0", - "awesome-typescript-loader": "^3.2.3", + "awesome-typescript-loader": "^4.0.0", "axios": "^0.17.1", "babel-core": "^6.26.0", - "babel-loader": "^7.1.2", "babel-plugin-syntax-dynamic-import": "^6.18.0", "babel-preset-es2015": "^6.24.1", "clean-webpack-plugin": "^0.1.19", @@ -34,7 +33,7 @@ "expect.js": "~0.2.0", "expose-loader": "^0.7.3", "extract-text-webpack-plugin": "^3.0.0", - "file-loader": "^0.11.2", + "file-loader": "^1.1.11", "gaze": "^1.1.2", "glob": "~7.0.0", "grunt": "1.0.1", @@ -61,7 +60,6 @@ "husky": "^0.14.3", "jest": "^22.0.4", "jshint-stylish": "~2.2.1", - "json-loader": "^0.5.7", "karma": "1.7.0", "karma-chrome-launcher": "~2.2.0", "karma-expect": "~1.1.3", @@ -83,16 +81,15 @@ "postcss-loader": "^2.0.6", "postcss-reporter": "^5.0.0", "prettier": "1.9.2", - "react-hot-loader": "^4.0.1", + "react-hot-loader": "^4.2.0", "react-test-renderer": "^16.0.0", "sass-lint": "^1.10.2", - "sass-loader": "^6.0.6", + "sass-loader": "^7.0.1", "sinon": "1.17.6", - "style-loader": "^0.20.3", + "style-loader": "^0.21.0", "systemjs": "0.20.19", "systemjs-plugin-css": "^0.1.36", "ts-jest": "^22.0.0", - "ts-loader": "^3.2.0", "tslint": "^5.8.0", "tslint-loader": "^3.5.3", "typescript": "^2.6.2", @@ -183,4 +180,4 @@ "resolutions": { "caniuse-db": "1.0.30000772" } -} +} \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 8d742b34ba1..cdd71528baa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -728,9 +728,9 @@ autoprefixer@^6.3.1, autoprefixer@^6.4.0: postcss "^5.2.16" postcss-value-parser "^3.2.3" -awesome-typescript-loader@^3.2.3: - version "3.5.0" - resolved "https://registry.yarnpkg.com/awesome-typescript-loader/-/awesome-typescript-loader-3.5.0.tgz#4d4d10cba7a04ed433dfa0334250846fb11a1a5a" +awesome-typescript-loader@^4.0.0: + version "4.0.1" + resolved "https://registry.yarnpkg.com/awesome-typescript-loader/-/awesome-typescript-loader-4.0.1.tgz#bddae8183f06eb65184390d596e4342ca2089281" dependencies: chalk "^2.3.1" enhanced-resolve "3.3.0" @@ -886,14 +886,6 @@ babel-jest@^22.4.3: babel-plugin-istanbul "^4.1.5" babel-preset-jest "^22.4.3" -babel-loader@^7.1.2: - version "7.1.4" - resolved "https://registry.yarnpkg.com/babel-loader/-/babel-loader-7.1.4.tgz#e3463938bd4e6d55d1c174c5485d406a188ed015" - dependencies: - find-cache-dir "^1.0.0" - loader-utils "^1.0.2" - mkdirp "^0.5.1" - babel-messages@^6.23.0: version "6.23.0" resolved "https://registry.yarnpkg.com/babel-messages/-/babel-messages-6.23.0.tgz#f3cdf4703858035b2a2951c6ec5edf6c62f2630e" @@ -2069,10 +2061,6 @@ comment-parser@^0.3.1: dependencies: readable-stream "^2.0.4" -commondir@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/commondir/-/commondir-1.0.1.tgz#ddd800da0c66127393cca5950ea968a3aaf1253b" - compare-versions@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/compare-versions/-/compare-versions-3.1.0.tgz#43310256a5c555aaed4193c04d8f154cf9c6efd5" @@ -3270,7 +3258,7 @@ enhanced-resolve@3.3.0: object-assign "^4.0.1" tapable "^0.2.5" -enhanced-resolve@^3.0.0, enhanced-resolve@^3.4.0: +enhanced-resolve@^3.4.0: version "3.4.1" resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-3.4.1.tgz#0421e339fd71419b3da13d129b3979040230476e" dependencies: @@ -3870,11 +3858,12 @@ file-entry-cache@^1.1.1: flat-cache "^1.2.1" object-assign "^4.0.1" -file-loader@^0.11.2: - version "0.11.2" - resolved "https://registry.yarnpkg.com/file-loader/-/file-loader-0.11.2.tgz#4ff1df28af38719a6098093b88c82c71d1794a34" +file-loader@^1.1.11: + version "1.1.11" + resolved "https://registry.yarnpkg.com/file-loader/-/file-loader-1.1.11.tgz#6fe886449b0f2a936e43cabaac0cdbfb369506f8" dependencies: loader-utils "^1.0.2" + schema-utils "^0.4.5" file-saver@^1.3.3: version "1.3.8" @@ -3942,14 +3931,6 @@ finalhandler@1.1.1: statuses "~1.4.0" unpipe "~1.0.0" -find-cache-dir@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/find-cache-dir/-/find-cache-dir-1.0.0.tgz#9288e3e9e3cc3748717d39eade17cf71fc30ee6f" - dependencies: - commondir "^1.0.1" - make-dir "^1.0.0" - pkg-dir "^2.0.0" - find-index@^0.1.1: version "0.1.1" resolved "https://registry.yarnpkg.com/find-index/-/find-index-0.1.1.tgz#675d358b2ca3892d795a1ab47232f8b6e2e0dde4" @@ -6091,7 +6072,7 @@ jshint@~2.9.4: shelljs "0.3.x" strip-json-comments "1.0.x" -json-loader@^0.5.4, json-loader@^0.5.7: +json-loader@^0.5.4: version "0.5.7" resolved "https://registry.yarnpkg.com/json-loader/-/json-loader-0.5.7.tgz#dca14a70235ff82f0ac9a3abeb60d337a365185d" @@ -8941,15 +8922,15 @@ react-highlight-words@^0.10.0: highlight-words-core "^1.1.0" prop-types "^15.5.8" -react-hot-loader@^4.0.1: - version "4.1.2" - resolved "https://registry.yarnpkg.com/react-hot-loader/-/react-hot-loader-4.1.2.tgz#5e8025f5bc5605506586b46eb2c6cc4006fd54d7" +react-hot-loader@^4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/react-hot-loader/-/react-hot-loader-4.2.0.tgz#4a2ec79114f872e28ea786e04889d643ad3dfb7c" dependencies: fast-levenshtein "^2.0.6" global "^4.3.0" hoist-non-react-statics "^2.5.0" prop-types "^15.6.1" - react-lifecycles-compat "^3.0.2" + react-lifecycles-compat "^3.0.4" shallowequal "^1.0.2" react-immutable-proptypes@^2.1.0: @@ -8966,9 +8947,9 @@ react-is@^16.3.2: version "16.3.2" resolved "https://registry.yarnpkg.com/react-is/-/react-is-16.3.2.tgz#f4d3d0e2f5fbb6ac46450641eb2e25bf05d36b22" -react-lifecycles-compat@^3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/react-lifecycles-compat/-/react-lifecycles-compat-3.0.2.tgz#7279047275bd727a912e25f734c0559527e84eff" +react-lifecycles-compat@^3.0.4: + version "3.0.4" + resolved "https://registry.yarnpkg.com/react-lifecycles-compat/-/react-lifecycles-compat-3.0.4.tgz#4f1a273afdfc8f3488a8c516bfda78f872352362" react-popper@^0.7.5: version "0.7.5" @@ -9668,9 +9649,9 @@ sass-lint@^1.10.2, sass-lint@^1.12.0: path-is-absolute "^1.0.0" util "^0.10.3" -sass-loader@^6.0.6: - version "6.0.7" - resolved "https://registry.yarnpkg.com/sass-loader/-/sass-loader-6.0.7.tgz#dd2fdb3e7eeff4a53f35ba6ac408715488353d00" +sass-loader@^7.0.1: + version "7.0.1" + resolved "https://registry.yarnpkg.com/sass-loader/-/sass-loader-7.0.1.tgz#fd937259ccba3a9cfe0d5f8a98746d48adfcc261" dependencies: clone-deep "^2.0.1" loader-utils "^1.0.1" @@ -9726,7 +9707,7 @@ semver-diff@^2.0.0: dependencies: semver "^5.0.3" -"semver@2 >=2.2.1 || 3.x || 4 || 5", "semver@2 || 3 || 4 || 5", "semver@2.x || 3.x || 4 || 5", "semver@^2.3.0 || 3.x || 4 || 5", semver@^5.0.1, semver@^5.0.3, semver@^5.1.0, semver@^5.3.0, semver@^5.4.1, semver@^5.5.0: +"semver@2 >=2.2.1 || 3.x || 4 || 5", "semver@2 || 3 || 4 || 5", "semver@2.x || 3.x || 4 || 5", "semver@^2.3.0 || 3.x || 4 || 5", semver@^5.0.3, semver@^5.1.0, semver@^5.3.0, semver@^5.4.1, semver@^5.5.0: version "5.5.0" resolved "https://registry.yarnpkg.com/semver/-/semver-5.5.0.tgz#dc4bbc7a6ca9d916dee5d43516f0092b58f7b8ab" @@ -10470,9 +10451,9 @@ strip-json-comments@~2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" -style-loader@^0.20.3: - version "0.20.3" - resolved "https://registry.yarnpkg.com/style-loader/-/style-loader-0.20.3.tgz#ebef06b89dec491bcb1fdb3452e913a6fd1c10c4" +style-loader@^0.21.0: + version "0.21.0" + resolved "https://registry.yarnpkg.com/style-loader/-/style-loader-0.21.0.tgz#68c52e5eb2afc9ca92b6274be277ee59aea3a852" dependencies: loader-utils "^1.1.0" schema-utils "^0.4.5" @@ -10799,16 +10780,6 @@ ts-jest@^22.0.0: pkg-dir "^2.0.0" yargs "^11.0.0" -ts-loader@^3.2.0: - version "3.5.0" - resolved "https://registry.yarnpkg.com/ts-loader/-/ts-loader-3.5.0.tgz#151d004dcddb4cf8e381a3bf9d6b74c2d957a9c0" - dependencies: - chalk "^2.3.0" - enhanced-resolve "^3.0.0" - loader-utils "^1.0.2" - micromatch "^3.1.4" - semver "^5.0.1" - tslib@^1.8.0, tslib@^1.8.1: version "1.9.0" resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.9.0.tgz#e37a86fda8cbbaf23a057f473c9f4dc64e5fc2e8" From ebe8e62bd52f9831a8f6df68385eeb2ebed103e9 Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Mon, 28 May 2018 13:49:15 +0200 Subject: [PATCH 38/87] Split webpack dev config into dev and hot Motivation: * too many conditionals for config, better to be explicit * different priorities: faster build for hot mode * working SCSS sources for styles in hot mode The biggest differences: * removed linter from TS loader in hot (should be editor or precommit or responsibility) * simplified styles loading * hot needs more extensions to resolve * removed commons chunking for hot * removed devServer from dev Reduced HMR time from 8s to 4s on my machine. --- package.json | 4 +- scripts/webpack/webpack.dev.js | 78 +++++------------------------ scripts/webpack/webpack.hot.js | 91 ++++++++++++++++++++++++++++++++++ 3 files changed, 106 insertions(+), 67 deletions(-) create mode 100644 scripts/webpack/webpack.hot.js diff --git a/package.json b/package.json index 06883a6c7ec..402fdc823a7 100644 --- a/package.json +++ b/package.json @@ -105,7 +105,7 @@ }, "scripts": { "dev": "webpack --progress --colors --config scripts/webpack/webpack.dev.js", - "start": "webpack-dev-server --progress --colors --config scripts/webpack/webpack.dev.js", + "start": "webpack-dev-server --progress --colors --config scripts/webpack/webpack.hot.js", "watch": "webpack --progress --colors --watch --config scripts/webpack/webpack.dev.js", "build": "grunt build", "test": "grunt test", @@ -183,4 +183,4 @@ "resolutions": { "caniuse-db": "1.0.30000772" } -} +} \ No newline at end of file diff --git a/scripts/webpack/webpack.dev.js b/scripts/webpack/webpack.dev.js index 625f921a388..7e43e8179ac 100644 --- a/scripts/webpack/webpack.dev.js +++ b/scripts/webpack/webpack.dev.js @@ -5,61 +5,29 @@ const common = require('./webpack.common.js'); const path = require('path'); const webpack = require('webpack'); const HtmlWebpackPlugin = require("html-webpack-plugin"); -const HtmlWebpackHarddiskPlugin = require('html-webpack-harddisk-plugin'); const ExtractTextPlugin = require("extract-text-webpack-plugin"); const CleanWebpackPlugin = require('clean-webpack-plugin'); const BundleAnalyzerPlugin = require('webpack-bundle-analyzer').BundleAnalyzerPlugin; -const TARGET = process.env.npm_lifecycle_event; -const HOT = TARGET === 'start'; - const extractSass = new ExtractTextPlugin({ - filename: "grafana.[name].css", - disable: HOT + filename: "grafana.[name].css" }); -const entries = HOT ? { - app: [ - 'webpack-dev-server/client?http://localhost:3333', - './public/app/dev.ts', - ], - vendor: require('./dependencies'), -} : { - app: './public/app/index.ts', - dark: './public/sass/grafana.dark.scss', - light: './public/sass/grafana.light.scss', - vendor: require('./dependencies'), - }; - -const output = HOT ? { - path: path.resolve(__dirname, '../../public/build'), - filename: '[name].[hash].js', - publicPath: "/public/build/", -} : { - path: path.resolve(__dirname, '../../public/build'), - filename: '[name].[hash].js', - // Keep publicPath relative for host.com/grafana/ deployments - publicPath: "public/build/", - }; - module.exports = merge(common, { devtool: "cheap-module-source-map", - entry: entries, - - output: output, - - resolve: { - extensions: ['.scss', '.ts', '.tsx', '.es6', '.js', '.json', '.svg', '.woff2', '.png'], + entry: { + app: './public/app/index.ts', + dark: './public/sass/grafana.dark.scss', + light: './public/sass/grafana.light.scss', + vendor: require('./dependencies'), }, - devServer: { - publicPath: '/public/build/', - hot: HOT, - port: 3333, - proxy: { - '!/public/build': 'http://localhost:3000' - } + output: { + path: path.resolve(__dirname, '../../public/build'), + filename: '[name].[hash].js', + // Keep publicPath relative for host.com/grafana/ deployments + publicPath: "public/build/", }, module: { @@ -83,33 +51,16 @@ module.exports = merge(common, { loader: 'awesome-typescript-loader', options: { useCache: true, - useBabel: HOT, - babelOptions: { - babelrc: false, - plugins: [ - 'syntax-dynamic-import', - 'react-hot-loader/babel' - ] - } }, } }, require('./sass.rule.js')({ - sourceMap: true, minimize: false, preserveUrl: HOT + sourceMap: true, minimize: false, preserveUrl: false }, extractSass), { - test: /\.(ttf|eot|svg|woff(2)?)(\?[a-z0-9=&.]+)?$/, + test: /\.(png|jpg|gif|ttf|eot|svg|woff(2)?)(\?[a-z0-9=&.]+)?$/, loader: 'file-loader' }, - { - test: /\.(png|jpg|gif)$/, - use: [ - { - loader: 'file-loader', - options: {} - } - ] - }, ] }, @@ -121,13 +72,10 @@ module.exports = merge(common, { template: path.resolve(__dirname, '../../public/views/index.template.html'), inject: 'body', chunks: ['manifest', 'vendor', 'app'], - alwaysWriteToDisk: HOT }), - new HtmlWebpackHarddiskPlugin(), new webpack.NamedModulesPlugin(), new webpack.HotModuleReplacementPlugin(), new webpack.DefinePlugin({ - 'GRAFANA_THEME': JSON.stringify(process.env.GRAFANA_THEME || 'dark'), 'process.env': { 'NODE_ENV': JSON.stringify('development') } diff --git a/scripts/webpack/webpack.hot.js b/scripts/webpack/webpack.hot.js new file mode 100644 index 00000000000..e0f078b71fb --- /dev/null +++ b/scripts/webpack/webpack.hot.js @@ -0,0 +1,91 @@ +'use strict'; + +const merge = require('webpack-merge'); +const common = require('./webpack.common.js'); +const path = require('path'); +const webpack = require('webpack'); +const HtmlWebpackPlugin = require("html-webpack-plugin"); +const HtmlWebpackHarddiskPlugin = require('html-webpack-harddisk-plugin'); +const CleanWebpackPlugin = require('clean-webpack-plugin'); + +module.exports = merge(common, { + entry: { + app: [ + 'webpack-dev-server/client?http://localhost:3333', + './public/app/dev.ts', + ], + }, + + output: { + path: path.resolve(__dirname, '../../public/build'), + filename: '[name].[hash].js', + publicPath: "/public/build/", + }, + + resolve: { + extensions: ['.scss', '.ts', '.tsx', '.es6', '.js', '.json', '.svg', '.woff2', '.png'], + }, + + devServer: { + publicPath: '/public/build/', + hot: true, + port: 3333, + proxy: { + '!/public/build': 'http://localhost:3000' + } + }, + + module: { + rules: [ + { + test: /\.tsx?$/, + exclude: /node_modules/, + use: { + loader: 'awesome-typescript-loader', + options: { + useCache: true, + useBabel: true, + babelOptions: { + babelrc: false, + plugins: [ + 'syntax-dynamic-import', + 'react-hot-loader/babel' + ] + } + }, + } + }, + { + test: /\.scss$/, + use: [ + "style-loader", // creates style nodes from JS strings + "css-loader", // translates CSS into CommonJS + "sass-loader" // compiles Sass to CSS + ] + }, + { + test: /\.(png|jpg|gif|ttf|eot|svg|woff(2)?)(\?[a-z0-9=&.]+)?$/, + loader: 'file-loader' + }, + ] + }, + + plugins: [ + new CleanWebpackPlugin('../public/build', { allowExternal: true }), + new HtmlWebpackPlugin({ + filename: path.resolve(__dirname, '../../public/views/index.html'), + template: path.resolve(__dirname, '../../public/views/index.template.html'), + inject: 'body', + alwaysWriteToDisk: true + }), + new HtmlWebpackHarddiskPlugin(), + new webpack.NamedModulesPlugin(), + new webpack.HotModuleReplacementPlugin(), + new webpack.DefinePlugin({ + 'GRAFANA_THEME': JSON.stringify(process.env.GRAFANA_THEME || 'dark'), + 'process.env': { + 'NODE_ENV': JSON.stringify('development') + } + }), + ] +}); From 17b96092097cdbe6b93e3eaf9536c72343ba416d Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Mon, 28 May 2018 07:30:44 -0700 Subject: [PATCH 39/87] Sparklines should scale to the data range (#12010) * Add a full range option to sparklines * line zero=false --- public/app/plugins/panel/singlestat/module.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/public/app/plugins/panel/singlestat/module.ts b/public/app/plugins/panel/singlestat/module.ts index b1996d8ffc9..b73a3bb32bd 100644 --- a/public/app/plugins/panel/singlestat/module.ts +++ b/public/app/plugins/panel/singlestat/module.ts @@ -580,6 +580,7 @@ class SingleStatCtrl extends MetricsPanelCtrl { lines: { show: true, fill: 1, + zero: false, lineWidth: 1, fillColor: panel.sparkline.fillColor, }, From 8d400b8f7bc89271877ba4984b209d1fb032b9f9 Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 28 May 2018 08:07:45 +0200 Subject: [PATCH 40/87] changelog: adds note about closing #9847 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5906cf567e..85164192aa5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ * **Security**: Fix XSS vulnerabilities in dashboard links [#11813](https://github.com/grafana/grafana/pull/11813) * **Singlestat**: Fix "time of last point" shows local time when dashboard timezone set to UTC [#10338](https://github.com/grafana/grafana/issues/10338) * **Prometheus**: Add support for passing timeout parameter to Prometheus [#11788](https://github.com/grafana/grafana/pull/11788), thx [@mtanda](https://github.com/mtanda) +* **Login**: Add optional option sign out url for generic oauth [#9847](https://github.com/grafana/grafana/issues/9847), thx [@roidelapluie](https://github.com/roidelapluie) # 5.1.3 (2018-05-16) From 7cb0403faa07ea66c35e1cafae0661c4726bc3a5 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Mon, 28 May 2018 19:45:18 +0200 Subject: [PATCH 41/87] elasticsearch: handle if alert query contains template variable If datasource handles targetContainsTemplate function it can evaluate if a certain query contains template variables and this is used for show an error message that template variables not is supported in alert queries. --- .../datasource/elasticsearch/datasource.ts | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/public/app/plugins/datasource/elasticsearch/datasource.ts b/public/app/plugins/datasource/elasticsearch/datasource.ts index e3eccfb8029..5a8e83a16cb 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.ts +++ b/public/app/plugins/datasource/elasticsearch/datasource.ts @@ -408,4 +408,65 @@ export class ElasticDatasource { getTagValues(options) { return this.getTerms({ field: options.key, query: '*' }); } + + targetContainsTemplate(target) { + if (this.templateSrv.variableExists(target.query) || this.templateSrv.variableExists(target.alias)) { + return true; + } + + for (let bucketAgg of target.bucketAggs) { + if (this.templateSrv.variableExists(bucketAgg.field) || this.objectContainsTemplate(bucketAgg.settings)) { + return true; + } + } + + for (let metric of target.metrics) { + if ( + this.templateSrv.variableExists(metric.field) || + this.objectContainsTemplate(metric.settings) || + this.objectContainsTemplate(metric.meta) + ) { + return true; + } + } + + return false; + } + + private isPrimitive(obj) { + if (obj === null || obj === undefined) { + return true; + } + if (['string', 'number', 'boolean'].some(type => type === typeof true)) { + return true; + } + + return false; + } + + private objectContainsTemplate(obj) { + if (!obj) { + return false; + } + + for (let key of Object.keys(obj)) { + if (this.isPrimitive(obj[key])) { + if (this.templateSrv.variableExists(obj[key])) { + return true; + } + } else if (Array.isArray(obj[key])) { + for (let item of obj[key]) { + if (this.objectContainsTemplate(item)) { + return true; + } + } + } else { + if (this.objectContainsTemplate(obj[key])) { + return true; + } + } + } + + return false; + } } From b487aa3e6afd3e109e9b619bf51d787e07c56608 Mon Sep 17 00:00:00 2001 From: thurt Date: Mon, 28 May 2018 17:49:31 +0000 Subject: [PATCH 42/87] return better error message when err is ErrSmtpNotEnabled fixes #12087 --- pkg/api/org_invite.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/api/org_invite.go b/pkg/api/org_invite.go index d6ab1c9d372..dfb2cf045ed 100644 --- a/pkg/api/org_invite.go +++ b/pkg/api/org_invite.go @@ -74,6 +74,9 @@ func AddOrgInvite(c *m.ReqContext, inviteDto dtos.AddInviteForm) Response { } if err := bus.Dispatch(&emailCmd); err != nil { + if err == m.ErrSmtpNotEnabled { + return Error(412, err.Error(), err) + } return Error(500, "Failed to send email invite", err) } From 6d8d6cdb57d03af2bdd1d86813ff2add733ce8fd Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Mon, 28 May 2018 17:38:09 +0200 Subject: [PATCH 43/87] Fix sourcemaps for webpack hot config --- scripts/webpack/webpack.hot.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/webpack/webpack.hot.js b/scripts/webpack/webpack.hot.js index e0f078b71fb..9a2845372ea 100644 --- a/scripts/webpack/webpack.hot.js +++ b/scripts/webpack/webpack.hot.js @@ -26,6 +26,8 @@ module.exports = merge(common, { extensions: ['.scss', '.ts', '.tsx', '.es6', '.js', '.json', '.svg', '.woff2', '.png'], }, + devtool: 'eval-source-map', + devServer: { publicPath: '/public/build/', hot: true, From e708e9ac3cc0ff60ae335c9a5fc42314500c6d4b Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 28 May 2018 15:57:12 +0200 Subject: [PATCH 44/87] graphite: avoid dtracing headers in direct mode closes #11494 --- .../plugins/datasource/graphite/datasource.ts | 12 +++- .../graphite/specs/datasource.jest.ts | 61 ++++++++++++++++--- 2 files changed, 62 insertions(+), 11 deletions(-) diff --git a/public/app/plugins/datasource/graphite/datasource.ts b/public/app/plugins/datasource/graphite/datasource.ts index 0b79673a14c..bc1c5722c3f 100644 --- a/public/app/plugins/datasource/graphite/datasource.ts +++ b/public/app/plugins/datasource/graphite/datasource.ts @@ -50,11 +50,11 @@ export function GraphiteDatasource(instanceSettings, $q, backendSrv, templateSrv data: params.join('&'), headers: { 'Content-Type': 'application/x-www-form-urlencoded', - 'X-Dashboard-Id': options.dashboardId, // enables distributed tracing in ds_proxy - 'X-Panel-Id': options.panelId, // enables distributed tracing in ds_proxy }, }; + this.addTracingHeaders(httpOptions, options); + if (options.panelId) { httpOptions.requestId = this.name + '.panelId.' + options.panelId; } @@ -62,6 +62,14 @@ export function GraphiteDatasource(instanceSettings, $q, backendSrv, templateSrv return this.doGraphiteRequest(httpOptions).then(this.convertDataPointsToMs); }; + this.addTracingHeaders = function(httpOptions, options) { + var proxyMode = !this.url.match(/^http/); + if (proxyMode) { + httpOptions.headers['X-Dashboard-Id'] = options.dashboardId; + httpOptions.headers['X-Panel-Id'] = options.panelId; + } + }; + this.convertDataPointsToMs = function(result) { if (!result || !result.data) { return []; diff --git a/public/app/plugins/datasource/graphite/specs/datasource.jest.ts b/public/app/plugins/datasource/graphite/specs/datasource.jest.ts index dac6c2252d8..f94378c57a6 100644 --- a/public/app/plugins/datasource/graphite/specs/datasource.jest.ts +++ b/public/app/plugins/datasource/graphite/specs/datasource.jest.ts @@ -9,16 +9,18 @@ describe('graphiteDatasource', () => { backendSrv: {}, $q: $q, templateSrv: new TemplateSrvStub(), + instanceSettings: { url: 'url', name: 'graphiteProd', jsonData: {} }, }; beforeEach(function() { - ctx.instanceSettings = { url: [''], name: 'graphiteProd', jsonData: {} }; + ctx.instanceSettings.url = '/api/datasources/proxy/1'; ctx.ds = new GraphiteDatasource(ctx.instanceSettings, ctx.$q, ctx.backendSrv, ctx.templateSrv); }); describe('When querying graphite with one target using query editor target spec', function() { let query = { panelId: 3, + dashboardId: 5, rangeRaw: { from: 'now-1h', to: 'now' }, targets: [{ target: 'prod1.count' }, { target: 'prod2.count' }], maxDataPoints: 500, @@ -40,8 +42,13 @@ describe('graphiteDatasource', () => { }); }); + it('X-Dashboard and X-Panel headers to be set!', () => { + expect(requestOptions.headers['X-Dashboard-Id']).toBe(5); + expect(requestOptions.headers['X-Panel-Id']).toBe(3); + }); + it('should generate the correct query', function() { - expect(requestOptions.url).toBe('/render'); + expect(requestOptions.url).toBe('/api/datasources/proxy/1/render'); }); it('should set unique requestId', function() { @@ -228,7 +235,7 @@ describe('graphiteDatasource', () => { results = data; }); - expect(requestOptions.url).toBe('/tags/autoComplete/tags'); + expect(requestOptions.url).toBe('/api/datasources/proxy/1/tags/autoComplete/tags'); expect(requestOptions.params.expr).toEqual([]); expect(results).not.toBe(null); }); @@ -238,7 +245,7 @@ describe('graphiteDatasource', () => { results = data; }); - expect(requestOptions.url).toBe('/tags/autoComplete/tags'); + expect(requestOptions.url).toBe('/api/datasources/proxy/1/tags/autoComplete/tags'); expect(requestOptions.params.expr).toEqual(['server=backend_01']); expect(results).not.toBe(null); }); @@ -248,7 +255,7 @@ describe('graphiteDatasource', () => { results = data; }); - expect(requestOptions.url).toBe('/tags/autoComplete/tags'); + expect(requestOptions.url).toBe('/api/datasources/proxy/1/tags/autoComplete/tags'); expect(requestOptions.params.expr).toEqual(['server=backend_01']); expect(results).not.toBe(null); }); @@ -258,7 +265,7 @@ describe('graphiteDatasource', () => { results = data; }); - expect(requestOptions.url).toBe('/tags/autoComplete/values'); + expect(requestOptions.url).toBe('/api/datasources/proxy/1/tags/autoComplete/values'); expect(requestOptions.params.tag).toBe('server'); expect(requestOptions.params.expr).toEqual([]); expect(results).not.toBe(null); @@ -269,7 +276,7 @@ describe('graphiteDatasource', () => { results = data; }); - expect(requestOptions.url).toBe('/tags/autoComplete/values'); + expect(requestOptions.url).toBe('/api/datasources/proxy/1/tags/autoComplete/values'); expect(requestOptions.params.tag).toBe('server'); expect(requestOptions.params.expr).toEqual(['server=~backend*']); expect(results).not.toBe(null); @@ -280,7 +287,7 @@ describe('graphiteDatasource', () => { results = data; }); - expect(requestOptions.url).toBe('/tags/autoComplete/values'); + expect(requestOptions.url).toBe('/api/datasources/proxy/1/tags/autoComplete/values'); expect(requestOptions.params.tag).toBe('server'); expect(requestOptions.params.expr).toEqual([]); expect(results).not.toBe(null); @@ -291,10 +298,46 @@ describe('graphiteDatasource', () => { results = data; }); - expect(requestOptions.url).toBe('/tags/autoComplete/values'); + expect(requestOptions.url).toBe('/api/datasources/proxy/1/tags/autoComplete/values'); expect(requestOptions.params.tag).toBe('server'); expect(requestOptions.params.expr).toEqual(['server=~backend*']); expect(results).not.toBe(null); }); }); }); + +function accessScenario(name, url, fn) { + describe('access scenario ' + name, function() { + let ctx: any = { + backendSrv: {}, + $q: $q, + templateSrv: new TemplateSrvStub(), + instanceSettings: { url: 'url', name: 'graphiteProd', jsonData: {} }, + }; + + let httpOptions = { + headers: {}, + }; + + describe('when using proxy mode', () => { + let options = { dashboardId: 1, panelId: 2 }; + + it('tracing headers should be added', () => { + ctx.instanceSettings.url = url; + var ds = new GraphiteDatasource(ctx.instanceSettings, ctx.$q, ctx.backendSrv, ctx.templateSrv); + ds.addTracingHeaders(httpOptions, options); + fn(httpOptions); + }); + }); + }); +} + +accessScenario('with proxy access', '/api/datasources/proxy/1', function(httpOptions) { + expect(httpOptions.headers['X-Dashboard-Id']).toBe(1); + expect(httpOptions.headers['X-Panel-Id']).toBe(2); +}); + +accessScenario('with direct access', 'http://localhost:8080', function(httpOptions) { + expect(httpOptions.headers['X-Dashboard-Id']).toBe(undefined); + expect(httpOptions.headers['X-Panel-Id']).toBe(undefined); +}); From 7548d6f6d1485b8cbb552a080d5410045dc6ef75 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Mon, 28 May 2018 20:37:39 +0200 Subject: [PATCH 45/87] Show create dashboard link if at least editor in one folder --- pkg/api/index.go | 25 +++++++++++-------- .../manage_dashboards/manage_dashboards.html | 2 +- .../manage_dashboards/manage_dashboards.ts | 6 +++++ public/app/core/components/search/search.html | 6 ++--- public/app/core/components/search/search.ts | 2 ++ 5 files changed, 27 insertions(+), 14 deletions(-) diff --git a/pkg/api/index.go b/pkg/api/index.go index 2a905b474ce..f082f03b5f6 100644 --- a/pkg/api/index.go +++ b/pkg/api/index.go @@ -92,17 +92,22 @@ func setIndexViewData(c *m.ReqContext) (*dtos.IndexViewData, error) { data.Theme = "light" } - if c.OrgRole == m.ROLE_ADMIN || c.OrgRole == m.ROLE_EDITOR { + if hasEditPermissionInFoldersQuery.Result { + children := []*dtos.NavLink{ + {Text: "Dashboard", Icon: "gicon gicon-dashboard-new", Url: setting.AppSubUrl + "/dashboard/new"}, + } + + if c.OrgRole == m.ROLE_ADMIN || c.OrgRole == m.ROLE_EDITOR { + children = append(children, &dtos.NavLink{Text: "Folder", SubTitle: "Create a new folder to organize your dashboards", Id: "folder", Icon: "gicon gicon-folder-new", Url: setting.AppSubUrl + "/dashboards/folder/new"}) + children = append(children, &dtos.NavLink{Text: "Import", SubTitle: "Import dashboard from file or Grafana.com", Id: "import", Icon: "gicon gicon-dashboard-import", Url: setting.AppSubUrl + "/dashboard/import"}) + } + data.NavTree = append(data.NavTree, &dtos.NavLink{ - Text: "Create", - Id: "create", - Icon: "fa fa-fw fa-plus", - Url: setting.AppSubUrl + "/dashboard/new", - Children: []*dtos.NavLink{ - {Text: "Dashboard", Icon: "gicon gicon-dashboard-new", Url: setting.AppSubUrl + "/dashboard/new"}, - {Text: "Folder", SubTitle: "Create a new folder to organize your dashboards", Id: "folder", Icon: "gicon gicon-folder-new", Url: setting.AppSubUrl + "/dashboards/folder/new"}, - {Text: "Import", SubTitle: "Import dashboard from file or Grafana.com", Id: "import", Icon: "gicon gicon-dashboard-import", Url: setting.AppSubUrl + "/dashboard/import"}, - }, + Text: "Create", + Id: "create", + Icon: "fa fa-fw fa-plus", + Url: setting.AppSubUrl + "/dashboard/new", + Children: children, }) } diff --git a/public/app/core/components/manage_dashboards/manage_dashboards.html b/public/app/core/components/manage_dashboards/manage_dashboards.html index 2dfb9c96d1b..aac30d2ce02 100644 --- a/public/app/core/components/manage_dashboards/manage_dashboards.html +++ b/public/app/core/components/manage_dashboards/manage_dashboards.html @@ -5,7 +5,7 @@
- + Dashboard diff --git a/public/app/core/components/manage_dashboards/manage_dashboards.ts b/public/app/core/components/manage_dashboards/manage_dashboards.ts index 545119a80d7..db73d84fd58 100644 --- a/public/app/core/components/manage_dashboards/manage_dashboards.ts +++ b/public/app/core/components/manage_dashboards/manage_dashboards.ts @@ -42,9 +42,12 @@ export class ManageDashboardsCtrl { // if user has editor role or higher isEditor: boolean; + hasEditPermissionInFolders: boolean; + /** @ngInject */ constructor(private backendSrv, navModelSrv, private searchSrv: SearchSrv, private contextSrv) { this.isEditor = this.contextSrv.isEditor; + this.hasEditPermissionInFolders = this.contextSrv.hasEditPermissionInFolders; this.query = { query: '', @@ -80,6 +83,9 @@ export class ManageDashboardsCtrl { return this.backendSrv.getFolderByUid(this.folderUid).then(folder => { this.canSave = folder.canSave; + if (!this.canSave) { + this.hasEditPermissionInFolders = false; + } }); }); } diff --git a/public/app/core/components/search/search.html b/public/app/core/components/search/search.html index afb9e723cad..561c752208e 100644 --- a/public/app/core/components/search/search.html +++ b/public/app/core/components/search/search.html @@ -45,14 +45,14 @@ -
+
New dashboard - + New folder - + Import dashboard diff --git a/public/app/core/components/search/search.ts b/public/app/core/components/search/search.ts index 25e05c2139d..162eeb1b9f3 100644 --- a/public/app/core/components/search/search.ts +++ b/public/app/core/components/search/search.ts @@ -17,6 +17,7 @@ export class SearchCtrl { isLoading: boolean; initialFolderFilterTitle: string; isEditor: string; + hasEditPermissionInFolders: boolean; /** @ngInject */ constructor($scope, private $location, private $timeout, private searchSrv: SearchSrv) { @@ -27,6 +28,7 @@ export class SearchCtrl { this.getTags = this.getTags.bind(this); this.onTagSelect = this.onTagSelect.bind(this); this.isEditor = contextSrv.isEditor; + this.hasEditPermissionInFolders = contextSrv.hasEditPermissionInFolders; } closeSearch() { From bafe25fbd969070dd62ba87583eae37d44279367 Mon Sep 17 00:00:00 2001 From: iyeonok Date: Mon, 28 May 2018 16:12:36 +0900 Subject: [PATCH 46/87] configure proxy environments for Transport property related issue: https://github.com/grafana/grafana/issues/9703 --- pkg/api/login_oauth.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/api/login_oauth.go b/pkg/api/login_oauth.go index c4a5f8fdacf..510c31c4ee1 100644 --- a/pkg/api/login_oauth.go +++ b/pkg/api/login_oauth.go @@ -75,9 +75,10 @@ func OAuthLogin(ctx *m.ReqContext) { ctx.Handle(500, "login.OAuthLogin(state mismatch)", nil) return } - + // handle call back tr := &http.Transport{ + Proxy: http.ProxyFromEnvironment, TLSClientConfig: &tls.Config{ InsecureSkipVerify: setting.OAuthService.OAuthInfos[name].TlsSkipVerify, }, From d7b5fb4604057cd757e00f8ce77ed1e2568060fb Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 28 May 2018 20:47:48 +0200 Subject: [PATCH 47/87] go fmt fixes --- pkg/api/login_oauth.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/api/login_oauth.go b/pkg/api/login_oauth.go index 510c31c4ee1..fe4fa93b621 100644 --- a/pkg/api/login_oauth.go +++ b/pkg/api/login_oauth.go @@ -75,7 +75,7 @@ func OAuthLogin(ctx *m.ReqContext) { ctx.Handle(500, "login.OAuthLogin(state mismatch)", nil) return } - + // handle call back tr := &http.Transport{ Proxy: http.ProxyFromEnvironment, From 83b7bbd60bc2be28e2f891f92da84e3dac74e81f Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 28 May 2018 20:55:11 +0200 Subject: [PATCH 48/87] changelog: adds note about closing #9703 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85164192aa5..9d70d30dc66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ * **Singlestat**: Fix "time of last point" shows local time when dashboard timezone set to UTC [#10338](https://github.com/grafana/grafana/issues/10338) * **Prometheus**: Add support for passing timeout parameter to Prometheus [#11788](https://github.com/grafana/grafana/pull/11788), thx [@mtanda](https://github.com/mtanda) * **Login**: Add optional option sign out url for generic oauth [#9847](https://github.com/grafana/grafana/issues/9847), thx [@roidelapluie](https://github.com/roidelapluie) +* **Login**: Use proxy server from environment variable if available [#9703](https://github.com/grafana/grafana/issues/9703), thx [@iyeonok](https://github.com/iyeonok) # 5.1.3 (2018-05-16) From 01f80950dec09e086a070db9cd0ed2e032be7849 Mon Sep 17 00:00:00 2001 From: Brice Maron Date: Sat, 19 May 2018 22:29:02 +0200 Subject: [PATCH 49/87] fix: add track by name in annotation list to avoid $$hashKey in json --- public/app/features/annotations/partials/editor.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/app/features/annotations/partials/editor.html b/public/app/features/annotations/partials/editor.html index 289f368ad0e..e1410ad0fea 100644 --- a/public/app/features/annotations/partials/editor.html +++ b/public/app/features/annotations/partials/editor.html @@ -21,7 +21,7 @@
- + - - + +
  {{annotation.name}} From 7c3e8afd82447cdece9897161e474ccdb46767d9 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Mon, 28 May 2018 22:19:14 +0200 Subject: [PATCH 50/87] add validation of uid when importing dashboards --- .../dashboard/dashboard_import_ctrl.ts | 31 ++++++++++++++++++ .../dashboard/partials/dashboard_import.html | 32 +++++++++++++++++-- .../specs/dashboard_import_ctrl.jest.ts | 1 + 3 files changed, 62 insertions(+), 2 deletions(-) diff --git a/public/app/features/dashboard/dashboard_import_ctrl.ts b/public/app/features/dashboard/dashboard_import_ctrl.ts index d127e628a77..fe61d3f7a55 100644 --- a/public/app/features/dashboard/dashboard_import_ctrl.ts +++ b/public/app/features/dashboard/dashboard_import_ctrl.ts @@ -7,6 +7,7 @@ export class DashboardImportCtrl { jsonText: string; parseError: string; nameExists: boolean; + uidExists: boolean; dash: any; inputs: any[]; inputsValid: boolean; @@ -16,6 +17,10 @@ export class DashboardImportCtrl { titleTouched: boolean; hasNameValidationError: boolean; nameValidationError: any; + hasUidValidationError: boolean; + uidValidationError: any; + autoGenerateUid: boolean; + autoGenerateUidValue: string; /** @ngInject */ constructor(private backendSrv, private validationSrv, navModelSrv, private $location, $routeParams) { @@ -23,6 +28,9 @@ export class DashboardImportCtrl { this.step = 1; this.nameExists = false; + this.uidExists = false; + this.autoGenerateUid = true; + this.autoGenerateUidValue = 'auto-generated'; // check gnetId in url if ($routeParams.gnetId) { @@ -61,6 +69,7 @@ export class DashboardImportCtrl { this.inputsValid = this.inputs.length === 0; this.titleChanged(); + this.uidChanged(true); } setDatasourceOptions(input, inputModel) { @@ -107,6 +116,28 @@ export class DashboardImportCtrl { }); } + uidChanged(initial) { + this.uidExists = false; + this.hasUidValidationError = false; + + if (initial === true && this.dash.uid) { + this.autoGenerateUidValue = 'value set'; + } + + this.backendSrv + .getDashboardByUid(this.dash.uid) + .then(res => { + this.uidExists = true; + this.hasUidValidationError = true; + this.uidValidationError = `Dashboard named '${res.dashboard.title}' in folder '${ + res.meta.folderTitle + }' has the same uid`; + }) + .catch(err => { + err.isHandled = true; + }); + } + saveDashboard() { var inputs = this.inputs.map(input => { return { diff --git a/public/app/features/dashboard/partials/dashboard_import.html b/public/app/features/dashboard/partials/dashboard_import.html index 020bb98e8b0..51011ae2c3d 100644 --- a/public/app/features/dashboard/partials/dashboard_import.html +++ b/public/app/features/dashboard/partials/dashboard_import.html @@ -80,6 +80,34 @@ +
+
+ + Unique identifier (uid) + + The unique identifier (uid) of a dashboard can be used for uniquely identify a dashboard between multiple Grafana installs. + The uid allows having consistent URL’s for accessing dashboards so changing the title of a dashboard will not break any + bookmarked links to that dashboard. + + + + change + + +
+
+ +
+
+ +
+
+
- - Cancel diff --git a/public/app/features/dashboard/specs/dashboard_import_ctrl.jest.ts b/public/app/features/dashboard/specs/dashboard_import_ctrl.jest.ts index 737eb360461..d75bd42f0c1 100644 --- a/public/app/features/dashboard/specs/dashboard_import_ctrl.jest.ts +++ b/public/app/features/dashboard/specs/dashboard_import_ctrl.jest.ts @@ -15,6 +15,7 @@ describe('DashboardImportCtrl', function() { backendSrv = { search: jest.fn().mockReturnValue(Promise.resolve([])), + getDashboardByUid: jest.fn().mockReturnValue(Promise.resolve([])), get: jest.fn(), }; From e6f2811b21c3efcd8faeac43f638c07f10bd45da Mon Sep 17 00:00:00 2001 From: Leonard Gram Date: Mon, 28 May 2018 16:57:51 +0200 Subject: [PATCH 51/87] sql: seconds epochs are now correctly converted to ms. Closes #12061 --- pkg/tsdb/sql_engine.go | 4 ++-- pkg/tsdb/sql_engine_test.go | 13 ++++++++++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/pkg/tsdb/sql_engine.go b/pkg/tsdb/sql_engine.go index 274e5b05dc1..82a9b8f0d88 100644 --- a/pkg/tsdb/sql_engine.go +++ b/pkg/tsdb/sql_engine.go @@ -144,10 +144,10 @@ func ConvertSqlTimeColumnToEpochMs(values RowValues, timeIndex int) { if timeIndex >= 0 { switch value := values[timeIndex].(type) { case time.Time: - values[timeIndex] = EpochPrecisionToMs(float64(value.UnixNano())) + values[timeIndex] = float64(value.UnixNano()) / float64(time.Millisecond) case *time.Time: if value != nil { - values[timeIndex] = EpochPrecisionToMs(float64((*value).UnixNano())) + values[timeIndex] = float64((*value).UnixNano()) / float64(time.Millisecond) } case int64: values[timeIndex] = int64(EpochPrecisionToMs(float64(value))) diff --git a/pkg/tsdb/sql_engine_test.go b/pkg/tsdb/sql_engine_test.go index ce1fb45de21..854734fac31 100644 --- a/pkg/tsdb/sql_engine_test.go +++ b/pkg/tsdb/sql_engine_test.go @@ -12,14 +12,17 @@ import ( func TestSqlEngine(t *testing.T) { Convey("SqlEngine", t, func() { dt := time.Date(2018, 3, 14, 21, 20, 6, int(527345*time.Microsecond), time.UTC) + earlyDt := time.Date(1970, 3, 14, 21, 20, 6, int(527345*time.Microsecond), time.UTC) Convey("Given row values with time.Time as time columns", func() { var nilPointer *time.Time - fixtures := make([]interface{}, 3) + fixtures := make([]interface{}, 5) fixtures[0] = dt fixtures[1] = &dt - fixtures[2] = nilPointer + fixtures[2] = earlyDt + fixtures[3] = &earlyDt + fixtures[4] = nilPointer for i := range fixtures { ConvertSqlTimeColumnToEpochMs(fixtures, i) @@ -27,9 +30,13 @@ func TestSqlEngine(t *testing.T) { Convey("When converting them should return epoch time with millisecond precision ", func() { expected := float64(dt.UnixNano()) / float64(time.Millisecond) + expectedEarly := float64(earlyDt.UnixNano()) / float64(time.Millisecond) + So(fixtures[0].(float64), ShouldEqual, expected) So(fixtures[1].(float64), ShouldEqual, expected) - So(fixtures[2], ShouldBeNil) + So(fixtures[2].(float64), ShouldEqual, expectedEarly) + So(fixtures[3].(float64), ShouldEqual, expectedEarly) + So(fixtures[4], ShouldBeNil) }) }) From 2d5ec9b9e42a9bf898f611dd554116319787d4be Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Tue, 29 May 2018 10:49:41 +0200 Subject: [PATCH 52/87] changelog: add notes about closing #12087 [skip ci] --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d70d30dc66..d0d650c1aee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ * **Prometheus**: Add support for passing timeout parameter to Prometheus [#11788](https://github.com/grafana/grafana/pull/11788), thx [@mtanda](https://github.com/mtanda) * **Login**: Add optional option sign out url for generic oauth [#9847](https://github.com/grafana/grafana/issues/9847), thx [@roidelapluie](https://github.com/roidelapluie) * **Login**: Use proxy server from environment variable if available [#9703](https://github.com/grafana/grafana/issues/9703), thx [@iyeonok](https://github.com/iyeonok) +* **Invite users**: Friendlier error message when smtp is not configured [#12087](https://github.com/grafana/grafana/issues/12087), thx [@thurt](https://github.com/thurt) # 5.1.3 (2018-05-16) From fb41048dd7a061b32a86a6a8c6462fb53cb40290 Mon Sep 17 00:00:00 2001 From: Daniel Lee Date: Tue, 29 May 2018 11:04:16 +0200 Subject: [PATCH 53/87] docker: new block for elasticsearch6 --- docker/blocks/elastic6/docker-compose.yaml | 15 +++++++++++++++ docker/blocks/elastic6/elasticsearch.yml | 2 ++ 2 files changed, 17 insertions(+) create mode 100644 docker/blocks/elastic6/docker-compose.yaml create mode 100644 docker/blocks/elastic6/elasticsearch.yml diff --git a/docker/blocks/elastic6/docker-compose.yaml b/docker/blocks/elastic6/docker-compose.yaml new file mode 100644 index 00000000000..dd2439f88e4 --- /dev/null +++ b/docker/blocks/elastic6/docker-compose.yaml @@ -0,0 +1,15 @@ +# You need to run 'sysctl -w vm.max_map_count=262144' on the host machine + + elasticsearch6: + image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.2.4 + command: elasticsearch + ports: + - "11200:9200" + - "11300:9300" + + fake-elastic6-data: + image: grafana/fake-data-gen + network_mode: bridge + environment: + FD_DATASOURCE: elasticsearch6 + FD_PORT: 11200 diff --git a/docker/blocks/elastic6/elasticsearch.yml b/docker/blocks/elastic6/elasticsearch.yml new file mode 100644 index 00000000000..c57b2c12908 --- /dev/null +++ b/docker/blocks/elastic6/elasticsearch.yml @@ -0,0 +1,2 @@ +script.inline: on +script.indexed: on From 6a82098ddf9fd925c42cfef6599a8e893c0fa275 Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 29 May 2018 11:52:00 +0200 Subject: [PATCH 54/87] devenv: scripts for generating many unique dashboards --- .gitignore | 2 + .../dashboards/bulk-testing/bulkdash.jsonnet | 1140 +++++++++++++++++ devenv/dashboards/generate-bulk-dashboards.sh | 15 + 3 files changed, 1157 insertions(+) create mode 100644 devenv/dashboards/bulk-testing/bulkdash.jsonnet create mode 100755 devenv/dashboards/generate-bulk-dashboards.sh diff --git a/.gitignore b/.gitignore index cbc85835a36..45dcb52e8d8 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,5 @@ debug.test /vendor/**/.editorconfig /vendor/**/appengine* *.orig + +/devenv/dashboards/bulk-testing/*.json diff --git a/devenv/dashboards/bulk-testing/bulkdash.jsonnet b/devenv/dashboards/bulk-testing/bulkdash.jsonnet new file mode 100644 index 00000000000..17b3f8983af --- /dev/null +++ b/devenv/dashboards/bulk-testing/bulkdash.jsonnet @@ -0,0 +1,1140 @@ +{ + "annotations": { + "enable": false, + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "aliasColors": { + "cpu": "#E24D42", + "memory": "#1f78c1", + "statsd.fakesite.counters.session_start.desktop.count": "#6ED0E0" + }, + "annotate": { + "enable": false + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "editable": true, + "fill": 3, + "grid": { + "max": null, + "min": 0 + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 4, + "interactive": true, + "legend": { + "avg": false, + "current": true, + "max": false, + "min": true, + "show": true, + "total": false, + "values": false + }, + "legend_counts": true, + "lines": true, + "linewidth": 2, + "nullPointMode": "connected", + "options": false, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "resolution": 100, + "scale": 1, + "seriesOverrides": [ + { + "alias": "cpu", + "fill": 0, + "lines": true, + "yaxis": 2, + "zindex": 2 + }, + { + "alias": "memory", + "pointradius": 2, + "points": true + } + ], + "spaceLength": 10, + "spyable": true, + "stack": false, + "steppedLine": false, + "targets": [ + { + "hide": false, + "refId": "A", + "target": "alias(movingAverage(scaleToSeconds(apps.fakesite.web_server_01.counters.request_status.code_302.count, 10), 20), 'cpu')" + }, + { + "refId": "B", + "target": "alias(statsd.fakesite.counters.session_start.desktop.count, 'memory')" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "timezone": "browser", + "title": "Memory / CPU", + "tooltip": { + "msResolution": false, + "query_as_alias": true, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percent", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + }, + "zerofill": true + }, + { + "aliasColors": { + "logins": "#5195ce", + "logins (-1 day)": "#447EBC", + "logins (-1 hour)": "#705da0" + }, + "annotate": { + "enable": false + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "editable": true, + "fill": 1, + "grid": { + "max": null, + "min": 0 + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 3, + "interactive": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "legend_counts": true, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": false, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "resolution": 100, + "scale": 1, + "seriesOverrides": [], + "spaceLength": 10, + "spyable": true, + "stack": true, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "alias(movingAverage(scaleToSeconds(apps.fakesite.web_server_01.counters.requests.count, 1), 2), 'logins')" + }, + { + "refId": "B", + "target": "alias(movingAverage(timeShift(scaleToSeconds(apps.fakesite.web_server_01.counters.requests.count, 1), '1h'), 2), 'logins (-1 hour)')" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": "1h", + "timezone": "browser", + "title": "logins", + "tooltip": { + "msResolution": false, + "query_as_alias": true, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + }, + "zerofill": true + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#629e51", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": null, + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 300, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 16, + "y": 0 + }, + "id": 22, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "refId": "A", + "target": "scale(apps.backend.backend_01.counters.requests.count, 0.4)" + } + ], + "thresholds": "200,270", + "title": "Memory", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 16, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "refId": "A", + "target": "apps.backend.backend_02.counters.requests.count" + } + ], + "thresholds": "100,270", + "title": "Sign ups", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 3 + }, + "id": 17, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "refId": "A", + "target": "apps.backend.backend_04.counters.requests.count" + } + ], + "thresholds": "100,270", + "title": "Sign outs", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 6 + }, + "id": 15, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "refId": "A", + "target": "scale(apps.backend.backend_01.counters.requests.count, 0.7)" + } + ], + "thresholds": "100,270", + "title": "Logins", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + "web_server_01": "#badff4", + "web_server_02": "#5195ce", + "web_server_03": "#1f78c1", + "web_server_04": "#0a437c" + }, + "annotate": { + "enable": false + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "editable": true, + "fill": 6, + "grid": { + "max": null, + "min": 0 + }, + "gridPos": { + "h": 11, + "w": 16, + "x": 0, + "y": 7 + }, + "id": 2, + "interactive": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "legend_counts": true, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": false, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "resolution": 100, + "scale": 1, + "seriesOverrides": [], + "spaceLength": 10, + "spyable": true, + "stack": true, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "aliasByNode(movingAverage(scaleToSeconds(apps.fakesite.*.counters.requests.count, 1), 2), 2)" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "timezone": "browser", + "title": "server requests", + "tooltip": { + "msResolution": false, + "query_as_alias": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + }, + "zerofill": true + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#629e51", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": null, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 300, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 7 + }, + "id": 21, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "refId": "A", + "target": "scale(apps.backend.backend_01.counters.requests.count, 0.8)" + } + ], + "thresholds": "200,270", + "title": "Logouts", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 9 + }, + "id": 18, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "refId": "A", + "target": "scale(apps.backend.backend_03.counters.requests.count, 0.3)" + } + ], + "thresholds": "100,270", + "title": "Support calls", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#629e51", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": null, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 300, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 16, + "y": 12 + }, + "id": 26, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "refId": "A", + "target": "scale(apps.backend.backend_01.counters.requests.count, 0.2)" + } + ], + "thresholds": "200,270", + "title": "Google hits", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#629e51", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": null, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 300, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 20, + "y": 12 + }, + "id": 24, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "refId": "A", + "target": "scale(apps.backend.backend_01.counters.requests.count, 0.2)" + } + ], + "thresholds": "200,270", + "title": "Google hits", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + "upper_25": "#F9E2D2", + "upper_50": "#F2C96D", + "upper_75": "#EAB839" + }, + "annotate": { + "enable": false + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": null, + "editable": true, + "fill": 1, + "grid": { + "max": null, + "min": 0 + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 5, + "interactive": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "legend_counts": true, + "lines": false, + "linewidth": 2, + "nullPointMode": "connected", + "options": false, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "resolution": 100, + "scale": 1, + "seriesOverrides": [], + "spaceLength": 10, + "spyable": true, + "stack": true, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "aliasByNode(summarize(statsd.fakesite.timers.ads_timer.*, '4min', 'avg'), 4)" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "timezone": "browser", + "title": "client side full page load", + "tooltip": { + "msResolution": false, + "query_as_alias": true, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + }, + "zerofill": true + } + ], + "refresh": false, + "schemaVersion": 16, + "style": "dark", + "tags": [ + "demo" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "collapse": false, + "enable": true, + "notice": false, + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "status": "Stable", + "time_options": [ + "5m", + "15m", + "1h", + "2h", + " 6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "type": "timepicker" + }, + "timezone": "browser", + "title": "Big Dashboard", + "uid": "000000003", + "version": 16 +} \ No newline at end of file diff --git a/devenv/dashboards/generate-bulk-dashboards.sh b/devenv/dashboards/generate-bulk-dashboards.sh new file mode 100755 index 00000000000..079a5a9c520 --- /dev/null +++ b/devenv/dashboards/generate-bulk-dashboards.sh @@ -0,0 +1,15 @@ +#/bin/bash + +if ! type "jsonnet" > /dev/null; then + echo "you need you install jsonnet to run this script" + echo "follow the instructions on https://github.com/google/jsonnet" + exit 1 +fi + +COUNTER=0 +MAX=400 +while [ $COUNTER -lt $MAX ]; do + jsonnet -o "bulk-testing/dashboard${COUNTER}.json" -e "local bulkDash = import 'bulk-testing/bulkdash.jsonnet'; bulkDash + { uid: 'uid-${COUNTER}', title: 'title-${COUNTER}' }" + let COUNTER=COUNTER+1 +done + From 8bcd55d2213375dd00930c09f13ae020bd94d787 Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Tue, 29 May 2018 12:01:10 +0200 Subject: [PATCH 55/87] Fix cache busting for systemjs imports for plugins * everything imported via systemjs in the path `plugin/` will get a timestamp appended for cache busting * timestamp is set once on page load * plugin css loader gets cache buster too --- public/app/features/plugins/plugin_loader.ts | 20 +++++++--------- public/vendor/plugin-css/css.js | 25 ++++++++++---------- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/public/app/features/plugins/plugin_loader.ts b/public/app/features/plugins/plugin_loader.ts index 57edfb35885..03b77e4e870 100644 --- a/public/app/features/plugins/plugin_loader.ts +++ b/public/app/features/plugins/plugin_loader.ts @@ -27,6 +27,13 @@ import 'rxjs/add/observable/from'; import 'rxjs/add/operator/map'; import 'rxjs/add/operator/combineAll'; +// add cache busting +const bust = `?_cache=${Date.now()}`; +function locate(load) { + return load.address + bust; +} +System.registry.set('plugin-loader', System.newModule({ locate: locate })); + System.config({ baseURL: 'public', defaultExtension: 'js', @@ -40,23 +47,14 @@ System.config({ css: 'vendor/plugin-css/css.js', }, meta: { - '*': { + 'plugin*': { esModule: true, authorization: true, + loader: 'plugin-loader', }, }, }); -// add cache busting -var systemLocate = System.locate; -System.cacheBust = '?bust=' + Date.now(); -System.locate = function(load) { - var System = this; - return Promise.resolve(systemLocate.call(this, load)).then(function(address) { - return address + System.cacheBust; - }); -}; - function exposeToPlugin(name: string, component: any) { System.registerDynamic(name, [], true, function(require, exports, module) { module.exports = component; diff --git a/public/vendor/plugin-css/css.js b/public/vendor/plugin-css/css.js index 44839808385..09f28d23b3a 100644 --- a/public/vendor/plugin-css/css.js +++ b/public/vendor/plugin-css/css.js @@ -1,6 +1,7 @@ "use strict"; if (typeof window !== 'undefined') { + var bust = '?_cache=' + Date.now(); var waitSeconds = 100; var head = document.getElementsByTagName('head')[0]; @@ -13,8 +14,8 @@ if (typeof window !== 'undefined') { } var isWebkit = !!window.navigator.userAgent.match(/AppleWebKit\/([^ ;]*)/); - var webkitLoadCheck = function(link, callback) { - setTimeout(function() { + var webkitLoadCheck = function (link, callback) { + setTimeout(function () { for (var i = 0; i < document.styleSheets.length; i++) { var sheet = document.styleSheets[i]; if (sheet.href === link.href) { @@ -25,17 +26,17 @@ if (typeof window !== 'undefined') { }, 10); }; - var noop = function() {}; + var noop = function () { }; - var loadCSS = function(url) { - return new Promise(function(resolve, reject) { - var timeout = setTimeout(function() { + var loadCSS = function (url) { + return new Promise(function (resolve, reject) { + var timeout = setTimeout(function () { reject('Unable to load CSS'); }, waitSeconds * 1000); - var _callback = function(error) { + var _callback = function (error) { clearTimeout(timeout); link.onload = link.onerror = noop; - setTimeout(function() { + setTimeout(function () { if (error) { reject(error); } @@ -47,22 +48,22 @@ if (typeof window !== 'undefined') { var link = document.createElement('link'); link.type = 'text/css'; link.rel = 'stylesheet'; - link.href = url; + link.href = url + bust; if (!isWebkit) { - link.onload = function() { + link.onload = function () { _callback(); } } else { webkitLoadCheck(link, _callback); } - link.onerror = function(event) { + link.onerror = function (event) { _callback(event.error || new Error('Error loading CSS file.')); }; head.appendChild(link); }); }; - exports.fetch = function(load) { + exports.fetch = function (load) { // dont reload styles loaded in the head for (var i = 0; i < linkHrefs.length; i++) if (load.address == linkHrefs[i]) From ddd5e5ae70283d0b8e695daa5e1fe4b91cf89de9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Tue, 29 May 2018 13:23:07 +0200 Subject: [PATCH 56/87] tech: updated react-grid-layout to latest official release, closes #12100 --- package.json | 10 +++--- .../dashboard/dashgrid/DashboardGrid.tsx | 2 +- public/img/resize-handle-white.svg | 11 ------- public/sass/components/_dashboard_grid.scss | 7 +--- yarn.lock | 32 +++++++++---------- 5 files changed, 23 insertions(+), 39 deletions(-) delete mode 100644 public/img/resize-handle-white.svg diff --git a/package.json b/package.json index 84d404829eb..df3da5812c1 100644 --- a/package.json +++ b/package.json @@ -134,11 +134,11 @@ }, "license": "Apache-2.0", "dependencies": { - "angular": "^1.6.6", + "angular": "1.6.6", "angular-bindonce": "^0.3.1", "angular-native-dragdrop": "^1.2.2", - "angular-route": "^1.6.6", - "angular-sanitize": "^1.6.6", + "angular-route": "1.6.6", + "angular-sanitize": "1.6.6", "babel-polyfill": "^6.26.0", "baron": "^3.0.3", "brace": "^0.10.0", @@ -161,7 +161,7 @@ "prop-types": "^15.6.0", "react": "^16.2.0", "react-dom": "^16.2.0", - "react-grid-layout-grafana": "0.16.0", + "react-grid-layout": "0.16.6", "react-highlight-words": "^0.10.0", "react-popper": "^0.7.5", "react-select": "^1.1.0", @@ -180,4 +180,4 @@ "resolutions": { "caniuse-db": "1.0.30000772" } -} \ No newline at end of file +} diff --git a/public/app/features/dashboard/dashgrid/DashboardGrid.tsx b/public/app/features/dashboard/dashgrid/DashboardGrid.tsx index 03bf65afc6e..290e587eace 100644 --- a/public/app/features/dashboard/dashgrid/DashboardGrid.tsx +++ b/public/app/features/dashboard/dashgrid/DashboardGrid.tsx @@ -1,5 +1,5 @@ import React from 'react'; -import ReactGridLayout from 'react-grid-layout-grafana'; +import ReactGridLayout from 'react-grid-layout'; import { GRID_CELL_HEIGHT, GRID_CELL_VMARGIN, GRID_COLUMN_COUNT } from 'app/core/constants'; import { DashboardPanel } from './DashboardPanel'; import { DashboardModel } from '../dashboard_model'; diff --git a/public/img/resize-handle-white.svg b/public/img/resize-handle-white.svg deleted file mode 100644 index 110ff8edfbb..00000000000 --- a/public/img/resize-handle-white.svg +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - diff --git a/public/sass/components/_dashboard_grid.scss b/public/sass/components/_dashboard_grid.scss index aec08d72258..0a27df75164 100644 --- a/public/sass/components/_dashboard_grid.scss +++ b/public/sass/components/_dashboard_grid.scss @@ -1,4 +1,4 @@ -@import '~react-grid-layout-grafana/css/styles.css'; +@import '~react-grid-layout/css/styles.css'; @import '~react-resizable/css/styles.css'; .panel-in-fullscreen { @@ -44,11 +44,6 @@ border-right: 2px solid $gray-1; border-bottom: 2px solid $gray-1; } - // temp fix since we use old commit of grid component - // this can be removed when we revert to non fork grid component - .react-grid-item > .react-resizable-handle { - background-image: url('../img/resize-handle-white.svg'); - } } .theme-light { diff --git a/yarn.lock b/yarn.lock index cdd71528baa..f58731040c6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -405,17 +405,17 @@ angular-native-dragdrop@^1.2.2: version "1.2.2" resolved "https://registry.yarnpkg.com/angular-native-dragdrop/-/angular-native-dragdrop-1.2.2.tgz#d646c6b75b131c48073c3f6e36a225b2726d8bae" -angular-route@^1.6.6: - version "1.6.10" - resolved "https://registry.yarnpkg.com/angular-route/-/angular-route-1.6.10.tgz#4247a32eab19495624623e96c1626dfba17ebf21" +angular-route@1.6.6: + version "1.6.6" + resolved "https://registry.yarnpkg.com/angular-route/-/angular-route-1.6.6.tgz#8c11748aa195c717b1b615a7e746442bfc7c61f4" -angular-sanitize@^1.6.6: - version "1.6.10" - resolved "https://registry.yarnpkg.com/angular-sanitize/-/angular-sanitize-1.6.10.tgz#635a362afb2dd040179f17d3a5455962b2c1918f" +angular-sanitize@1.6.6: + version "1.6.6" + resolved "https://registry.yarnpkg.com/angular-sanitize/-/angular-sanitize-1.6.6.tgz#0fd065a19931517fbece66596d325d72b6e06041" -angular@^1.6.6: - version "1.6.10" - resolved "https://registry.yarnpkg.com/angular/-/angular-1.6.10.tgz#eed3080a34d29d0f681ff119b18ce294e3f74826" +angular@1.6.6: + version "1.6.6" + resolved "https://registry.yarnpkg.com/angular/-/angular-1.6.6.tgz#fd5a3cfb437ce382d854ee01120797978527cb64" ansi-align@^2.0.0: version "2.0.0" @@ -8898,22 +8898,22 @@ react-dom@^16.2.0: object-assign "^4.1.1" prop-types "^15.6.0" -"react-draggable@^2.2.6 || ^3.0.3", react-draggable@^3.0.3: +react-draggable@3.x, "react-draggable@^2.2.6 || ^3.0.3": version "3.0.5" resolved "https://registry.yarnpkg.com/react-draggable/-/react-draggable-3.0.5.tgz#c031e0ed4313531f9409d6cd84c8ebcec0ddfe2d" dependencies: classnames "^2.2.5" prop-types "^15.6.0" -react-grid-layout-grafana@0.16.0: - version "0.16.0" - resolved "https://registry.yarnpkg.com/react-grid-layout-grafana/-/react-grid-layout-grafana-0.16.0.tgz#12242153fcd0bb80a26af8e41694bc2fde788b3a" +react-grid-layout@0.16.6: + version "0.16.6" + resolved "https://registry.yarnpkg.com/react-grid-layout/-/react-grid-layout-0.16.6.tgz#9b2407a2b946c2260ebaf66f13b556e1da4efeb2" dependencies: classnames "2.x" lodash.isequal "^4.0.0" prop-types "15.x" - react-draggable "^3.0.3" - react-resizable "^1.7.5" + react-draggable "3.x" + react-resizable "1.x" react-highlight-words@^0.10.0: version "0.10.0" @@ -8973,7 +8973,7 @@ react-reconciler@^0.7.0: object-assign "^4.1.1" prop-types "^15.6.0" -react-resizable@^1.7.5: +react-resizable@1.x: version "1.7.5" resolved "https://registry.yarnpkg.com/react-resizable/-/react-resizable-1.7.5.tgz#83eb75bb3684da6989bbbf4f826e1470f0af902e" dependencies: From a1e6c31ec12a43f7cd7605031e9cbce7c2c667d6 Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 29 May 2018 14:00:46 +0200 Subject: [PATCH 57/87] devenv: script for setting up default datasources --- .../bulk-testing/bulk-dashboards.yaml | 9 ++ devenv/dashboards/generate-bulk-dashboards.sh | 15 ---- devenv/datasources/default/default.yaml | 82 +++++++++++++++++++ devenv/setup.sh | 61 ++++++++++++++ 4 files changed, 152 insertions(+), 15 deletions(-) create mode 100644 devenv/dashboards/bulk-testing/bulk-dashboards.yaml delete mode 100755 devenv/dashboards/generate-bulk-dashboards.sh create mode 100644 devenv/datasources/default/default.yaml create mode 100755 devenv/setup.sh diff --git a/devenv/dashboards/bulk-testing/bulk-dashboards.yaml b/devenv/dashboards/bulk-testing/bulk-dashboards.yaml new file mode 100644 index 00000000000..7838e4bc342 --- /dev/null +++ b/devenv/dashboards/bulk-testing/bulk-dashboards.yaml @@ -0,0 +1,9 @@ +apiVersion: 1 + +providers: + - name: 'Bulk dashboards' + folder: 'Bulk dashboards' + type: file + options: + path: /home/carl/go/src/github.com/grafana/grafana/devenv/dashboards/bulk-testing + diff --git a/devenv/dashboards/generate-bulk-dashboards.sh b/devenv/dashboards/generate-bulk-dashboards.sh deleted file mode 100755 index 079a5a9c520..00000000000 --- a/devenv/dashboards/generate-bulk-dashboards.sh +++ /dev/null @@ -1,15 +0,0 @@ -#/bin/bash - -if ! type "jsonnet" > /dev/null; then - echo "you need you install jsonnet to run this script" - echo "follow the instructions on https://github.com/google/jsonnet" - exit 1 -fi - -COUNTER=0 -MAX=400 -while [ $COUNTER -lt $MAX ]; do - jsonnet -o "bulk-testing/dashboard${COUNTER}.json" -e "local bulkDash = import 'bulk-testing/bulkdash.jsonnet'; bulkDash + { uid: 'uid-${COUNTER}', title: 'title-${COUNTER}' }" - let COUNTER=COUNTER+1 -done - diff --git a/devenv/datasources/default/default.yaml b/devenv/datasources/default/default.yaml new file mode 100644 index 00000000000..b721c093f3a --- /dev/null +++ b/devenv/datasources/default/default.yaml @@ -0,0 +1,82 @@ +apiVersion: 1 + +datasources: + - name: Graphite + type: graphite + access: proxy + url: http://localhost:8080 + jsonData: + graphiteVersion: "1.1" + + - name: Prometheus + type: prometheus + access: proxy + isDefault: true + url: http://localhost:9090 + + - name: InfluxDB + type: influxdb + access: proxy + database: site + user: grafana + password: grafana + url: http://localhost:8086 + jsonData: + timeInterval: "15s" + + - name: OpenTsdb + type: opentsdb + access: proxy + url: http://localhost:4242 + jsonData: + tsdbResolution: 1 + tsdbVersion: 1 + + - name: Elastic + type: elasticsearch + access: proxy + database: "[metrics-]YYYY.MM.DD" + url: http://localhost:9200 + jsonData: + interval: Daily + timeField: "@timestamp" + + - name: MySQL + type: mysql + url: localhost:3306 + database: grafana + user: grafana + password: password + + - name: MSSQL + type: mssql + url: localhost:1433 + database: grafana + user: grafana + password: "Password!" + + - name: Postgres + type: postgres + url: localhost:5432 + database: grafana + user: grafana + password: password + jsonData: + sslmode: "disable" + + - name: Cloudwatch + type: cloudwatch + editable: true + jsonData: + authType: credentials + defaultRegion: eu-west-2 + + - name: Cloudwatch keys + type: cloudwatch + editable: true + jsonData: + authType: keys + defaultRegion: eu-west-2 + secureJsonData: + accessKey: AKIAJL347VWN6MK63N2A + secretKey: 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 diff --git a/devenv/setup.sh b/devenv/setup.sh new file mode 100755 index 00000000000..d6f8f969e75 --- /dev/null +++ b/devenv/setup.sh @@ -0,0 +1,61 @@ +#/bin/bash + +bulkDashboard() { + + requiresJsonnet + + COUNTER=0 + MAX=400 + while [ $COUNTER -lt $MAX ]; do + jsonnet -o "dashboards/bulk-testing/dashboard${COUNTER}.json" -e "local bulkDash = import 'dashboards/bulk-testing/bulkdash.jsonnet'; bulkDash + { uid: 'uid-${COUNTER}', title: 'title-${COUNTER}' }" + let COUNTER=COUNTER+1 + done + + ln -s -f -r ./dashboards/bulk-testing/bulk-dashboards.yaml ../conf/provisioning/dashboards/custom.yaml +} + +requiresJsonnet() { + if ! type "jsonnet" > /dev/null; then + echo "you need you install jsonnet to run this script" + echo "follow the instructions on https://github.com/google/jsonnet" + exit 1 + fi +} + +defaultDashboards() { + echo "not implemented yet" +} + +defaultDatasources() { + echo "setting up all default datasources using provisioning" + + ln -s -f -r ./datasources/default/default.yaml ../conf/provisioning/datasources/custom.yaml +} + +usage() { + echo -e "install.sh\n\tThis script installs my basic setup for a debian laptop\n" + echo "Usage:" + echo " bulk-dashboards - create and provisioning 400 dashboards" + echo " default-datasources - provisiong all core datasources" +} + +main() { + local cmd=$1 + + if [[ -z "$cmd" ]]; then + usage + exit 1 + fi + + if [[ $cmd == "bulk-dashboards" ]]; then + bulkDashboard + elif [[ $cmd == "default-datasources" ]]; then + defaultDatasources + elif [[ $cmd == "default-dashboards" ]]; then + bulkDashboard + else + usage + fi +} + +main "$@" \ No newline at end of file From be34417b3aa85c5eddfcff044ecd3df38c56c905 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Tue, 29 May 2018 14:02:52 +0200 Subject: [PATCH 58/87] fix: refactoring PR #11996 and fixing issue #11551 16706hashkey in json editors --- public/app/features/annotations/editor_ctrl.ts | 4 ++++ public/app/features/annotations/partials/editor.html | 4 ++-- public/app/features/dashboard/save_provisioned_modal.ts | 2 +- public/app/features/dashboard/settings/settings.ts | 3 ++- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/public/app/features/annotations/editor_ctrl.ts b/public/app/features/annotations/editor_ctrl.ts index 169e2e4c2bb..34b9635ec85 100644 --- a/public/app/features/annotations/editor_ctrl.ts +++ b/public/app/features/annotations/editor_ctrl.ts @@ -70,6 +70,10 @@ export class AnnotationsEditorCtrl { this.mode = 'list'; } + move(index, dir) { + _.move(this.annotations, index, index + dir); + } + add() { this.annotations.push(this.currentAnnotation); this.reset(); diff --git a/public/app/features/annotations/partials/editor.html b/public/app/features/annotations/partials/editor.html index e1410ad0fea..65ee7e52bd0 100644 --- a/public/app/features/annotations/partials/editor.html +++ b/public/app/features/annotations/partials/editor.html @@ -33,8 +33,8 @@
{{annotation.datasource || 'Default'}} diff --git a/public/app/features/dashboard/save_provisioned_modal.ts b/public/app/features/dashboard/save_provisioned_modal.ts index ba96ce0b0b9..3f2dcd0f57b 100644 --- a/public/app/features/dashboard/save_provisioned_modal.ts +++ b/public/app/features/dashboard/save_provisioned_modal.ts @@ -48,7 +48,7 @@ export class SaveProvisionedDashboardModalCtrl { constructor(dashboardSrv) { this.dash = dashboardSrv.getCurrent().getSaveModelClone(); delete this.dash.id; - this.dashboardJson = JSON.stringify(this.dash, null, 2); + this.dashboardJson = angular.toJson(this.dash, true); } save() { diff --git a/public/app/features/dashboard/settings/settings.ts b/public/app/features/dashboard/settings/settings.ts index 5acbbcf29c5..457cac5af72 100755 --- a/public/app/features/dashboard/settings/settings.ts +++ b/public/app/features/dashboard/settings/settings.ts @@ -2,6 +2,7 @@ import { coreModule, appEvents, contextSrv } from 'app/core/core'; import { DashboardModel } from '../dashboard_model'; import $ from 'jquery'; import _ from 'lodash'; +import angular from 'angular'; import config from 'app/core/config'; export class SettingsCtrl { @@ -118,7 +119,7 @@ export class SettingsCtrl { this.viewId = this.$location.search().editview; if (this.viewId) { - this.json = JSON.stringify(this.dashboard.getSaveModelClone(), null, 2); + this.json = angular.toJson(this.dashboard.getSaveModelClone(), true); } if (this.viewId === 'settings' && this.dashboard.meta.canMakeEditable) { From 4c9b146bda91ad3a37923c3dcd478109553cd3fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Tue, 29 May 2018 14:11:05 +0200 Subject: [PATCH 59/87] PR: minor change to PR #12004 before merge --- public/sass/components/_panel_singlestat.scss | 2 -- 1 file changed, 2 deletions(-) diff --git a/public/sass/components/_panel_singlestat.scss b/public/sass/components/_panel_singlestat.scss index faaa6fc2447..af11de3b835 100644 --- a/public/sass/components/_panel_singlestat.scss +++ b/public/sass/components/_panel_singlestat.scss @@ -7,7 +7,6 @@ .singlestat-panel-value-container { line-height: 1; - display: table-cell; position: absolute; z-index: 1; font-size: 3em; @@ -16,7 +15,6 @@ top: 50%; left: 50%; transform: translate(-50%, -50%); - padding-bottom: 10px; } .singlestat-panel-prefix { From 3ba3fd9a598f73ef719f5497f658ab52dd5e909f Mon Sep 17 00:00:00 2001 From: Christophe Le Guern Date: Tue, 29 May 2018 14:26:33 +0200 Subject: [PATCH 60/87] Add new regions to handleGetRegions function (#12082) As public/app/plugins/datasource/cloudwatch/partials/config.html and this file differ between the AWS regions available, I've updated the latest so they share the same data. In that way, the regions() method in dashboards returns the same list as the frontend does. --- pkg/tsdb/cloudwatch/metric_find_query.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/tsdb/cloudwatch/metric_find_query.go b/pkg/tsdb/cloudwatch/metric_find_query.go index a7d33645b9b..136ee241c2e 100644 --- a/pkg/tsdb/cloudwatch/metric_find_query.go +++ b/pkg/tsdb/cloudwatch/metric_find_query.go @@ -230,8 +230,8 @@ func parseMultiSelectValue(input string) []string { // Please update the region list in public/app/plugins/datasource/cloudwatch/partials/config.html func (e *CloudWatchExecutor) handleGetRegions(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) ([]suggestData, error) { regions := []string{ - "ap-northeast-1", "ap-northeast-2", "ap-southeast-1", "ap-southeast-2", "ap-south-1", "ca-central-1", "cn-north-1", - "eu-central-1", "eu-west-1", "eu-west-2", "sa-east-1", "us-east-1", "us-east-2", "us-gov-west-1", "us-west-1", "us-west-2", + "ap-northeast-1", "ap-northeast-2", "ap-southeast-1", "ap-southeast-2", "ap-south-1", "ca-central-1", "cn-north-1", "cn-northwest-1", + "eu-central-1", "eu-west-1", "eu-west-2", "eu-west-3", "sa-east-1", "us-east-1", "us-east-2", "us-gov-west-1", "us-west-1", "us-west-2", } result := make([]suggestData, 0) From 79575ea124e07fcd106da646787318f8de1f29a7 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Tue, 29 May 2018 14:28:04 +0200 Subject: [PATCH 61/87] changelog: add notes about closing #11494 [skip ci] --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0d650c1aee..7c16f4f6e5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ * **Login**: Add optional option sign out url for generic oauth [#9847](https://github.com/grafana/grafana/issues/9847), thx [@roidelapluie](https://github.com/roidelapluie) * **Login**: Use proxy server from environment variable if available [#9703](https://github.com/grafana/grafana/issues/9703), thx [@iyeonok](https://github.com/iyeonok) * **Invite users**: Friendlier error message when smtp is not configured [#12087](https://github.com/grafana/grafana/issues/12087), thx [@thurt](https://github.com/thurt) +* **Graphite**: Don't send distributed tracing headers when using direct/browser access mode [#11494](https://github.com/grafana/grafana/issues/11494) # 5.1.3 (2018-05-16) From 1411709db1c8ce65fd45906fcbc43c7757256084 Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 29 May 2018 14:07:37 +0200 Subject: [PATCH 62/87] provisioning: place testfiles within testdata folder --- .../all-properties/not.yaml.txt => devenv/README.md | 0 devenv/datasources/default/default.yaml | 9 --------- .../provisioning/datasources/config_reader_test.go | 12 ++++++------ .../all-properties/all-properties.yaml | 0 .../all-properties/not.yaml.txt} | 0 .../all-properties/sample.yaml | 0 .../all-properties/second.yaml | 0 .../broken-yaml/broken.yaml | 0 .../broken-yaml/commented.yaml | 0 .../double-default/default-1.yaml | 0 .../double-default/default-2.yaml | 0 .../insert-two-delete-two/one-datasources.yaml | 0 .../insert-two-delete-two/two-datasources.yml | 0 .../two-datasources/two-datasources.yaml | 0 .../version-0/version-0.yaml | 0 .../testdata/zero-datasources/placeholder-for-git | 0 16 files changed, 6 insertions(+), 15 deletions(-) rename pkg/services/provisioning/datasources/test-configs/all-properties/not.yaml.txt => devenv/README.md (100%) rename pkg/services/provisioning/datasources/{test-configs => testdata}/all-properties/all-properties.yaml (100%) rename pkg/services/provisioning/datasources/{test-configs/zero-datasources/placeholder-for-git => testdata/all-properties/not.yaml.txt} (100%) rename pkg/services/provisioning/datasources/{test-configs => testdata}/all-properties/sample.yaml (100%) rename pkg/services/provisioning/datasources/{test-configs => testdata}/all-properties/second.yaml (100%) rename pkg/services/provisioning/datasources/{test-configs => testdata}/broken-yaml/broken.yaml (100%) rename pkg/services/provisioning/datasources/{test-configs => testdata}/broken-yaml/commented.yaml (100%) rename pkg/services/provisioning/datasources/{test-configs => testdata}/double-default/default-1.yaml (100%) rename pkg/services/provisioning/datasources/{test-configs => testdata}/double-default/default-2.yaml (100%) rename pkg/services/provisioning/datasources/{test-configs => testdata}/insert-two-delete-two/one-datasources.yaml (100%) rename pkg/services/provisioning/datasources/{test-configs => testdata}/insert-two-delete-two/two-datasources.yml (100%) rename pkg/services/provisioning/datasources/{test-configs => testdata}/two-datasources/two-datasources.yaml (100%) rename pkg/services/provisioning/datasources/{test-configs => testdata}/version-0/version-0.yaml (100%) create mode 100644 pkg/services/provisioning/datasources/testdata/zero-datasources/placeholder-for-git diff --git a/pkg/services/provisioning/datasources/test-configs/all-properties/not.yaml.txt b/devenv/README.md similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/all-properties/not.yaml.txt rename to devenv/README.md diff --git a/devenv/datasources/default/default.yaml b/devenv/datasources/default/default.yaml index b721c093f3a..dc2310f15aa 100644 --- a/devenv/datasources/default/default.yaml +++ b/devenv/datasources/default/default.yaml @@ -71,12 +71,3 @@ datasources: authType: credentials defaultRegion: eu-west-2 - - name: Cloudwatch keys - type: cloudwatch - editable: true - jsonData: - authType: keys - defaultRegion: eu-west-2 - secureJsonData: - accessKey: AKIAJL347VWN6MK63N2A - secretKey: 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 diff --git a/pkg/services/provisioning/datasources/config_reader_test.go b/pkg/services/provisioning/datasources/config_reader_test.go index 89ecc5a0b68..2e407dbe4de 100644 --- a/pkg/services/provisioning/datasources/config_reader_test.go +++ b/pkg/services/provisioning/datasources/config_reader_test.go @@ -13,12 +13,12 @@ import ( var ( logger log.Logger = log.New("fake.log") - twoDatasourcesConfig = "./test-configs/two-datasources" - twoDatasourcesConfigPurgeOthers = "./test-configs/insert-two-delete-two" - doubleDatasourcesConfig = "./test-configs/double-default" - allProperties = "./test-configs/all-properties" - versionZero = "./test-configs/version-0" - brokenYaml = "./test-configs/broken-yaml" + twoDatasourcesConfig = "testdata/two-datasources" + twoDatasourcesConfigPurgeOthers = "testdata/insert-two-delete-two" + doubleDatasourcesConfig = "testdata/double-default" + allProperties = "testdata/all-properties" + versionZero = "testdata/version-0" + brokenYaml = "testdata/broken-yaml" fakeRepo *fakeRepository ) diff --git a/pkg/services/provisioning/datasources/test-configs/all-properties/all-properties.yaml b/pkg/services/provisioning/datasources/testdata/all-properties/all-properties.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/all-properties/all-properties.yaml rename to pkg/services/provisioning/datasources/testdata/all-properties/all-properties.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/zero-datasources/placeholder-for-git b/pkg/services/provisioning/datasources/testdata/all-properties/not.yaml.txt similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/zero-datasources/placeholder-for-git rename to pkg/services/provisioning/datasources/testdata/all-properties/not.yaml.txt diff --git a/pkg/services/provisioning/datasources/test-configs/all-properties/sample.yaml b/pkg/services/provisioning/datasources/testdata/all-properties/sample.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/all-properties/sample.yaml rename to pkg/services/provisioning/datasources/testdata/all-properties/sample.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/all-properties/second.yaml b/pkg/services/provisioning/datasources/testdata/all-properties/second.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/all-properties/second.yaml rename to pkg/services/provisioning/datasources/testdata/all-properties/second.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/broken-yaml/broken.yaml b/pkg/services/provisioning/datasources/testdata/broken-yaml/broken.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/broken-yaml/broken.yaml rename to pkg/services/provisioning/datasources/testdata/broken-yaml/broken.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/broken-yaml/commented.yaml b/pkg/services/provisioning/datasources/testdata/broken-yaml/commented.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/broken-yaml/commented.yaml rename to pkg/services/provisioning/datasources/testdata/broken-yaml/commented.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/double-default/default-1.yaml b/pkg/services/provisioning/datasources/testdata/double-default/default-1.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/double-default/default-1.yaml rename to pkg/services/provisioning/datasources/testdata/double-default/default-1.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/double-default/default-2.yaml b/pkg/services/provisioning/datasources/testdata/double-default/default-2.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/double-default/default-2.yaml rename to pkg/services/provisioning/datasources/testdata/double-default/default-2.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/insert-two-delete-two/one-datasources.yaml b/pkg/services/provisioning/datasources/testdata/insert-two-delete-two/one-datasources.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/insert-two-delete-two/one-datasources.yaml rename to pkg/services/provisioning/datasources/testdata/insert-two-delete-two/one-datasources.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/insert-two-delete-two/two-datasources.yml b/pkg/services/provisioning/datasources/testdata/insert-two-delete-two/two-datasources.yml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/insert-two-delete-two/two-datasources.yml rename to pkg/services/provisioning/datasources/testdata/insert-two-delete-two/two-datasources.yml diff --git a/pkg/services/provisioning/datasources/test-configs/two-datasources/two-datasources.yaml b/pkg/services/provisioning/datasources/testdata/two-datasources/two-datasources.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/two-datasources/two-datasources.yaml rename to pkg/services/provisioning/datasources/testdata/two-datasources/two-datasources.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/version-0/version-0.yaml b/pkg/services/provisioning/datasources/testdata/version-0/version-0.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/version-0/version-0.yaml rename to pkg/services/provisioning/datasources/testdata/version-0/version-0.yaml diff --git a/pkg/services/provisioning/datasources/testdata/zero-datasources/placeholder-for-git b/pkg/services/provisioning/datasources/testdata/zero-datasources/placeholder-for-git new file mode 100644 index 00000000000..e69de29bb2d From b253284accef14e4ad5fa0d89ee55c8837cb5047 Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 29 May 2018 16:52:02 +0200 Subject: [PATCH 63/87] devenv: improve readme --- devenv/README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/devenv/README.md b/devenv/README.md index e69de29bb2d..4ec6f672f25 100644 --- a/devenv/README.md +++ b/devenv/README.md @@ -0,0 +1,11 @@ +This folder contains useful scripts and configuration for... + +* Configuring datasources in Grafana +* Provision example dashboards in Grafana +* Run preconfiured datasources as docker containers + +want to know more? run setup! + +```bash +./setup.sh +``` From f32e3a29609ad311595ec7e6b87b6c740d3ec270 Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 29 May 2018 17:22:52 +0200 Subject: [PATCH 64/87] changelog: note about closing #11858 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c16f4f6e5b..9eb6125492d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ * **Login**: Use proxy server from environment variable if available [#9703](https://github.com/grafana/grafana/issues/9703), thx [@iyeonok](https://github.com/iyeonok) * **Invite users**: Friendlier error message when smtp is not configured [#12087](https://github.com/grafana/grafana/issues/12087), thx [@thurt](https://github.com/thurt) * **Graphite**: Don't send distributed tracing headers when using direct/browser access mode [#11494](https://github.com/grafana/grafana/issues/11494) +* **Sidenav**: Show create dashboard link for viewers if at least editor in one folder [#11858](https://github.com/grafana/grafana/issues/11858) # 5.1.3 (2018-05-16) From c7acbcdaf5e28092a2be9d44c348d2a767bc7e3b Mon Sep 17 00:00:00 2001 From: bergquist Date: Wed, 30 May 2018 08:46:44 +0200 Subject: [PATCH 65/87] provisioning: enable relative path's this commit enable relatives path for provisioning dashboards. this enables easier dev setups --- .../bulk-testing/bulk-dashboards.yaml | 2 +- .../provisioning/dashboards/file_reader.go | 8 +- .../dashboards/file_reader_test.go | 75 ++++++++++++------- 3 files changed, 55 insertions(+), 30 deletions(-) diff --git a/devenv/dashboards/bulk-testing/bulk-dashboards.yaml b/devenv/dashboards/bulk-testing/bulk-dashboards.yaml index 7838e4bc342..e0ba8a88e68 100644 --- a/devenv/dashboards/bulk-testing/bulk-dashboards.yaml +++ b/devenv/dashboards/bulk-testing/bulk-dashboards.yaml @@ -5,5 +5,5 @@ providers: folder: 'Bulk dashboards' type: file options: - path: /home/carl/go/src/github.com/grafana/grafana/devenv/dashboards/bulk-testing + path: devenv/dashboards/bulk-testing diff --git a/pkg/services/provisioning/dashboards/file_reader.go b/pkg/services/provisioning/dashboards/file_reader.go index e5186e12f06..93846f5c474 100644 --- a/pkg/services/provisioning/dashboards/file_reader.go +++ b/pkg/services/provisioning/dashboards/file_reader.go @@ -47,9 +47,15 @@ func NewDashboardFileReader(cfg *DashboardsAsConfig, log log.Logger) (*fileReade log.Error("Cannot read directory", "error", err) } + absPath, err := filepath.Abs(path) + if err != nil { + log.Error("Could not create absolute path ", "path", path) + absPath = path //if .Abs return an error we fallback to path + } + return &fileReader{ Cfg: cfg, - Path: path, + Path: absPath, log: log, dashboardService: dashboards.NewProvisioningService(), }, nil diff --git a/pkg/services/provisioning/dashboards/file_reader_test.go b/pkg/services/provisioning/dashboards/file_reader_test.go index 084fae1310a..a04fbb23f82 100644 --- a/pkg/services/provisioning/dashboards/file_reader_test.go +++ b/pkg/services/provisioning/dashboards/file_reader_test.go @@ -15,14 +15,57 @@ import ( ) var ( - defaultDashboards = "./testdata/test-dashboards/folder-one" - brokenDashboards = "./testdata/test-dashboards/broken-dashboards" - oneDashboard = "./testdata/test-dashboards/one-dashboard" - containingId = "./testdata/test-dashboards/containing-id" + defaultDashboards = "testdata/test-dashboards/folder-one" + brokenDashboards = "testdata/test-dashboards/broken-dashboards" + oneDashboard = "testdata/test-dashboards/one-dashboard" + containingId = "testdata/test-dashboards/containing-id" fakeService *fakeDashboardProvisioningService ) +func TestCreatingNewDashboardFileReader(t *testing.T) { + Convey("creating new dashboard file reader", t, func() { + cfg := &DashboardsAsConfig{ + Name: "Default", + Type: "file", + OrgId: 1, + Folder: "", + Options: map[string]interface{}{}, + } + + Convey("using path parameter", func() { + cfg.Options["path"] = defaultDashboards + reader, err := NewDashboardFileReader(cfg, log.New("test-logger")) + So(err, ShouldBeNil) + So(reader.Path, ShouldNotEqual, "") + }) + + Convey("using folder as options", func() { + cfg.Options["folder"] = defaultDashboards + reader, err := NewDashboardFileReader(cfg, log.New("test-logger")) + So(err, ShouldBeNil) + So(reader.Path, ShouldNotEqual, "") + }) + + Convey("using full path", func() { + cfg.Options["folder"] = "/var/lib/grafana/dashboards" + reader, err := NewDashboardFileReader(cfg, log.New("test-logger")) + So(err, ShouldBeNil) + + So(reader.Path, ShouldEqual, "/var/lib/grafana/dashboards") + So(filepath.IsAbs(reader.Path), ShouldBeTrue) + }) + + Convey("using relative path", func() { + cfg.Options["folder"] = defaultDashboards + reader, err := NewDashboardFileReader(cfg, log.New("test-logger")) + So(err, ShouldBeNil) + + So(filepath.IsAbs(reader.Path), ShouldBeTrue) + }) + }) +} + func TestDashboardFileReader(t *testing.T) { Convey("Dashboard file reader", t, func() { bus.ClearBusHandlers() @@ -170,30 +213,6 @@ func TestDashboardFileReader(t *testing.T) { }) }) - Convey("Can use bpth path and folder as dashboard path", func() { - cfg := &DashboardsAsConfig{ - Name: "Default", - Type: "file", - OrgId: 1, - Folder: "", - Options: map[string]interface{}{}, - } - - Convey("using path parameter", func() { - cfg.Options["path"] = defaultDashboards - reader, err := NewDashboardFileReader(cfg, log.New("test-logger")) - So(err, ShouldBeNil) - So(reader.Path, ShouldEqual, defaultDashboards) - }) - - Convey("using folder as options", func() { - cfg.Options["folder"] = defaultDashboards - reader, err := NewDashboardFileReader(cfg, log.New("test-logger")) - So(err, ShouldBeNil) - So(reader.Path, ShouldEqual, defaultDashboards) - }) - }) - Reset(func() { dashboards.NewProvisioningService = origNewDashboardProvisioningService }) From 48fc5edda19a7c426d70a494786a46378722a6b2 Mon Sep 17 00:00:00 2001 From: Kim Christensen Date: Wed, 30 May 2018 09:22:16 +0200 Subject: [PATCH 66/87] Support InfluxDB count distinct aggregation (#11658) influxdb: support count distinct aggregation --- pkg/tsdb/influxdb/query_part_test.go | 8 + .../plugins/datasource/influxdb/query_part.ts | 23 +++ .../influxdb/specs/query_part.jest.ts | 144 ++++++++++++++++++ 3 files changed, 175 insertions(+) diff --git a/pkg/tsdb/influxdb/query_part_test.go b/pkg/tsdb/influxdb/query_part_test.go index d23865174c8..cd0863cee9b 100644 --- a/pkg/tsdb/influxdb/query_part_test.go +++ b/pkg/tsdb/influxdb/query_part_test.go @@ -76,5 +76,13 @@ func TestInfluxdbQueryPart(t *testing.T) { res := part.Render(query, queryContext, "mean(value)") So(res, ShouldEqual, `mean(value) AS "test"`) }) + + Convey("render count distinct", func() { + part, err := NewQueryPart("count", []string{}) + So(err, ShouldBeNil) + + res := part.Render(query, queryContext, "distinct(value)") + So(res, ShouldEqual, `count(distinct(value))`) + }) }) } diff --git a/public/app/plugins/datasource/influxdb/query_part.ts b/public/app/plugins/datasource/influxdb/query_part.ts index ce5588abe53..2a2f9f2a4ef 100644 --- a/public/app/plugins/datasource/influxdb/query_part.ts +++ b/public/app/plugins/datasource/influxdb/query_part.ts @@ -44,6 +44,28 @@ function replaceAggregationAddStrategy(selectParts, partModel) { for (var i = 0; i < selectParts.length; i++) { var part = selectParts[i]; if (part.def.category === categories.Aggregations) { + if (part.def.type === partModel.def.type) { + return; + } + // count distinct is allowed + if (part.def.type === 'count' && partModel.def.type === 'distinct') { + break; + } + // remove next aggregation if distinct was replaced + if (part.def.type === 'distinct') { + var morePartsAvailable = selectParts.length >= i + 2; + if (partModel.def.type !== 'count' && morePartsAvailable) { + var nextPart = selectParts[i + 1]; + if (nextPart.def.category === categories.Aggregations) { + selectParts.splice(i + 1, 1); + } + } else if (partModel.def.type === 'count') { + if (!morePartsAvailable || selectParts[i + 1].def.type !== 'count') { + selectParts.splice(i + 1, 0, partModel); + } + return; + } + } selectParts[i] = partModel; return; } @@ -434,4 +456,5 @@ export default { getCategories: function() { return categories; }, + replaceAggregationAdd: replaceAggregationAddStrategy, }; diff --git a/public/app/plugins/datasource/influxdb/specs/query_part.jest.ts b/public/app/plugins/datasource/influxdb/specs/query_part.jest.ts index cabe8bc9b6f..e9e6d216c1e 100644 --- a/public/app/plugins/datasource/influxdb/specs/query_part.jest.ts +++ b/public/app/plugins/datasource/influxdb/specs/query_part.jest.ts @@ -40,5 +40,149 @@ describe('InfluxQueryPart', () => { expect(part.text).toBe('alias(test)'); expect(part.render('mean(value)')).toBe('mean(value) AS "test"'); }); + + it('should nest distinct when count is selected', () => { + var selectParts = [ + queryPart.create({ + type: 'field', + category: queryPart.getCategories().Fields, + }), + queryPart.create({ + type: 'count', + category: queryPart.getCategories().Aggregations, + }), + ]; + var partModel = queryPart.create({ + type: 'distinct', + category: queryPart.getCategories().Aggregations, + }); + + queryPart.replaceAggregationAdd(selectParts, partModel); + + expect(selectParts[1].text).toBe('distinct()'); + expect(selectParts[2].text).toBe('count()'); + }); + + it('should convert to count distinct when distinct is selected and count added', () => { + var selectParts = [ + queryPart.create({ + type: 'field', + category: queryPart.getCategories().Fields, + }), + queryPart.create({ + type: 'distinct', + category: queryPart.getCategories().Aggregations, + }), + ]; + var partModel = queryPart.create({ + type: 'count', + category: queryPart.getCategories().Aggregations, + }); + + queryPart.replaceAggregationAdd(selectParts, partModel); + + expect(selectParts[1].text).toBe('distinct()'); + expect(selectParts[2].text).toBe('count()'); + }); + + it('should replace count distinct if an aggregation is selected', () => { + var selectParts = [ + queryPart.create({ + type: 'field', + category: queryPart.getCategories().Fields, + }), + queryPart.create({ + type: 'distinct', + category: queryPart.getCategories().Aggregations, + }), + queryPart.create({ + type: 'count', + category: queryPart.getCategories().Aggregations, + }), + ]; + var partModel = queryPart.create({ + type: 'mean', + category: queryPart.getCategories().Selectors, + }); + + queryPart.replaceAggregationAdd(selectParts, partModel); + + expect(selectParts[1].text).toBe('mean()'); + expect(selectParts).toHaveLength(2); + }); + + it('should not allowed nested counts when count distinct is selected', () => { + var selectParts = [ + queryPart.create({ + type: 'field', + category: queryPart.getCategories().Fields, + }), + queryPart.create({ + type: 'distinct', + category: queryPart.getCategories().Aggregations, + }), + queryPart.create({ + type: 'count', + category: queryPart.getCategories().Aggregations, + }), + ]; + var partModel = queryPart.create({ + type: 'count', + category: queryPart.getCategories().Aggregations, + }); + + queryPart.replaceAggregationAdd(selectParts, partModel); + + expect(selectParts[1].text).toBe('distinct()'); + expect(selectParts[2].text).toBe('count()'); + expect(selectParts).toHaveLength(3); + }); + + it('should not remove count distinct when distinct is added', () => { + var selectParts = [ + queryPart.create({ + type: 'field', + category: queryPart.getCategories().Fields, + }), + queryPart.create({ + type: 'distinct', + category: queryPart.getCategories().Aggregations, + }), + queryPart.create({ + type: 'count', + category: queryPart.getCategories().Aggregations, + }), + ]; + var partModel = queryPart.create({ + type: 'distinct', + category: queryPart.getCategories().Aggregations, + }); + + queryPart.replaceAggregationAdd(selectParts, partModel); + + expect(selectParts[1].text).toBe('distinct()'); + expect(selectParts[2].text).toBe('count()'); + expect(selectParts).toHaveLength(3); + }); + + it('should remove distinct when sum aggregation is selected', () => { + var selectParts = [ + queryPart.create({ + type: 'field', + category: queryPart.getCategories().Fields, + }), + queryPart.create({ + type: 'distinct', + category: queryPart.getCategories().Aggregations, + }), + ]; + var partModel = queryPart.create({ + type: 'sum', + category: queryPart.getCategories().Aggregations, + }); + queryPart.replaceAggregationAdd(selectParts, partModel); + + expect(selectParts[1].text).toBe('sum()'); + }); }); }); From f2942d94a5b3c8d48616e2ee77f53e20f50420ff Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 30 May 2018 09:26:15 +0200 Subject: [PATCH 67/87] changelog: add notes about closing #11645 [skip ci] --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9eb6125492d..3597b1b6a1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ * **Dashboard**: Fix date selector styling for dark/light theme in time picker control [#11616](https://github.com/grafana/grafana/issues/11616) * **Discord**: Alert notification channel type for Discord, [#7964](https://github.com/grafana/grafana/issues/7964) thx [@jereksel](https://github.com/jereksel), * **InfluxDB**: Support SELECT queries in templating query, [#5013](https://github.com/grafana/grafana/issues/5013) +* **InfluxDB**: Support count distinct aggregation [#11645](https://github.com/grafana/grafana/issues/11645), thx [@kichristensen](https://github.com/kichristensen) * **Dashboard**: JSON Model under dashboard settings can now be updated & changes saved, [#1429](https://github.com/grafana/grafana/issues/1429), thx [@jereksel](https://github.com/jereksel) * **Security**: Fix XSS vulnerabilities in dashboard links [#11813](https://github.com/grafana/grafana/pull/11813) * **Singlestat**: Fix "time of last point" shows local time when dashboard timezone set to UTC [#10338](https://github.com/grafana/grafana/issues/10338) From ac1dda3b3a522d3174fd2035c4e562312994e92d Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Wed, 30 May 2018 12:07:51 +0200 Subject: [PATCH 68/87] Fix CSS to hide grid controls in fullscreen/low-activity views * there was a comma missing to hide the handles, fixed now * added new styles to hide header interaction in full screen panels --- public/sass/components/_dashboard_grid.scss | 14 ++++++++++++++ public/sass/components/_view_states.scss | 3 ++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/public/sass/components/_dashboard_grid.scss b/public/sass/components/_dashboard_grid.scss index 0a27df75164..f1908ca8786 100644 --- a/public/sass/components/_dashboard_grid.scss +++ b/public/sass/components/_dashboard_grid.scss @@ -18,6 +18,20 @@ height: 100% !important; transform: translate(0px, 0px) !important; } + + // Disable grid interaction indicators in fullscreen panels + + .panel-header:hover { + background-color: inherit; + } + + .panel-title-container { + cursor: pointer; + } + + .react-resizable-handle { + display: none; + } } @include media-breakpoint-down(sm) { diff --git a/public/sass/components/_view_states.scss b/public/sass/components/_view_states.scss index b1fa47d0c0a..c14590b4ec9 100644 --- a/public/sass/components/_view_states.scss +++ b/public/sass/components/_view_states.scss @@ -10,7 +10,8 @@ .playlist-active, .user-activity-low { - .react-resizable-handle .add-row-panel-hint, + .react-resizable-handle, + .add-row-panel-hint, .dash-row-menu-container, .navbar-button--refresh, .navbar-buttons--zoom, From 21ecaae6ff2f91b5b58e008f81a32d30bd06d74d Mon Sep 17 00:00:00 2001 From: Leonard Gram Date: Wed, 30 May 2018 14:30:01 +0200 Subject: [PATCH 69/87] changelog: Second epochs are now correctly converted to ms. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3597b1b6a1c..3a86eeba75e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ * **Invite users**: Friendlier error message when smtp is not configured [#12087](https://github.com/grafana/grafana/issues/12087), thx [@thurt](https://github.com/thurt) * **Graphite**: Don't send distributed tracing headers when using direct/browser access mode [#11494](https://github.com/grafana/grafana/issues/11494) * **Sidenav**: Show create dashboard link for viewers if at least editor in one folder [#11858](https://github.com/grafana/grafana/issues/11858) +* **SQL**: Second epochs are now correctly converted to ms. [#12085](https://github.com/grafana/grafana/pull/12085) # 5.1.3 (2018-05-16) From 50d1519a916a5526d02e7cb3621b97b5db8505e2 Mon Sep 17 00:00:00 2001 From: Leonard Gram Date: Wed, 30 May 2018 13:55:30 +0200 Subject: [PATCH 70/87] build: mysql integration testing on ci. --- .circleci/config.yml | 26 +++++++++++++++++++++++++ docker/blocks/mysql/docker-compose.yaml | 2 +- docker/blocks/mysql_tests/Dockerfile | 4 ++-- pkg/tsdb/mysql/mysql_test.go | 6 +++--- 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c92a68bf99d..d9cc03b9527 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,6 +12,26 @@ aliases: version: 2 jobs: + mysql-integration-test: + docker: + - image: circleci/golang:1.10 + - image: circleci/mysql:5.6-ram + environment: + MYSQL_ROOT_PASSWORD: rootpass + MYSQL_DATABASE: grafana_tests + MYSQL_USER: grafana + MYSQL_PASSWORD: password + working_directory: /go/src/github.com/grafana/grafana + steps: + - checkout + - run: sudo apt update + - run: sudo apt install -y mysql-client + - run: dockerize -wait tcp://127.0.0.1:3306 -timeout 120s + - run: cat docker/blocks/mysql_tests/setup.sql | mysql -h 127.0.0.1 -P 3306 -u root -prootpass + - run: + name: mysql integration tests + command: 'GRAFANA_TEST_DB=mysql go test ./pkg/...' + codespell: docker: - image: circleci/python @@ -188,6 +208,8 @@ workflows: filters: *filter-not-release - test-backend: filters: *filter-not-release + - mysql-integration-test: + filters: *filter-not-release - deploy-master: requires: - build-all @@ -195,6 +217,7 @@ workflows: - test-frontend - codespell - gometalinter + - mysql-integration-test filters: branches: only: master @@ -210,6 +233,8 @@ workflows: filters: *filter-only-release - test-backend: filters: *filter-only-release + - mysql-integration-test: + filters: *filter-only-release - deploy-release: requires: - build-all @@ -217,4 +242,5 @@ workflows: - test-frontend - codespell - gometalinter + - mysql-integration-test filters: *filter-only-release diff --git a/docker/blocks/mysql/docker-compose.yaml b/docker/blocks/mysql/docker-compose.yaml index 53ff9da62a7..381b04a53c8 100644 --- a/docker/blocks/mysql/docker-compose.yaml +++ b/docker/blocks/mysql/docker-compose.yaml @@ -1,5 +1,5 @@ mysql: - image: mysql:latest + image: mysql:5.6 environment: MYSQL_ROOT_PASSWORD: rootpass MYSQL_DATABASE: grafana diff --git a/docker/blocks/mysql_tests/Dockerfile b/docker/blocks/mysql_tests/Dockerfile index fa91fa3c023..89e16bc2ed6 100644 --- a/docker/blocks/mysql_tests/Dockerfile +++ b/docker/blocks/mysql_tests/Dockerfile @@ -1,3 +1,3 @@ -FROM mysql:latest +FROM mysql:5.6 ADD setup.sql /docker-entrypoint-initdb.d -CMD ["mysqld"] \ No newline at end of file +CMD ["mysqld"] diff --git a/pkg/tsdb/mysql/mysql_test.go b/pkg/tsdb/mysql/mysql_test.go index 29c5b72b408..5650de237c5 100644 --- a/pkg/tsdb/mysql/mysql_test.go +++ b/pkg/tsdb/mysql/mysql_test.go @@ -601,7 +601,7 @@ func TestMySQL(t *testing.T) { Queries: []*tsdb.Query{ { Model: simplejson.NewFromAny(map[string]interface{}{ - "rawSql": `SELECT $__time(time), CONCAT(measurement, ' - value one') as metric, valueOne FROM metric_values ORDER BY 1`, + "rawSql": `SELECT $__time(time), CONCAT(measurement, ' - value one') as metric, valueOne FROM metric_values ORDER BY 1,2`, "format": "time_series", }), RefId: "A", @@ -615,8 +615,8 @@ func TestMySQL(t *testing.T) { So(queryResult.Error, ShouldBeNil) So(len(queryResult.Series), ShouldEqual, 2) - So(queryResult.Series[0].Name, ShouldEqual, "Metric B - value one") - So(queryResult.Series[1].Name, ShouldEqual, "Metric A - value one") + So(queryResult.Series[0].Name, ShouldEqual, "Metric A - value one") + So(queryResult.Series[1].Name, ShouldEqual, "Metric B - value one") }) Convey("When doing a metric query grouping by time should return correct series", func() { From e33b17fac666e03135fdeb1c5b9a0227e85e1ff2 Mon Sep 17 00:00:00 2001 From: Leonard Gram Date: Wed, 30 May 2018 09:40:45 +0200 Subject: [PATCH 71/87] build: integration testing postegres on ci. --- .circleci/config.yml | 25 ++++++++++++++++++++++ docker/blocks/postgres/docker-compose.yaml | 4 ++-- docker/blocks/postgres_tests/Dockerfile | 4 ++-- docker/blocks/postgres_tests/setup.sql | 2 +- 4 files changed, 30 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d9cc03b9527..46404e4e650 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -32,6 +32,25 @@ jobs: name: mysql integration tests command: 'GRAFANA_TEST_DB=mysql go test ./pkg/...' + postgres-integration-test: + docker: + - image: circleci/golang:1.10 + - image: circleci/postgres:9.3-ram + environment: + POSTGRES_USER: grafanatest + POSTGRES_PASSWORD: grafanatest + POSTGRES_DB: grafanatest + working_directory: /go/src/github.com/grafana/grafana + steps: + - checkout + - run: sudo apt update + - run: sudo apt install -y postgresql-client + - run: dockerize -wait tcp://127.0.0.1:5432 -timeout 120s + - run: 'PGPASSWORD=grafanatest psql -p 5432 -h 127.0.0.1 -U grafanatest -d grafanatest -f docker/blocks/postgres_tests/setup.sql' + - run: + name: postgres integration tests + command: 'GRAFANA_TEST_DB=postgres go test ./pkg/...' + codespell: docker: - image: circleci/python @@ -210,6 +229,8 @@ workflows: filters: *filter-not-release - mysql-integration-test: filters: *filter-not-release + - postgres-integration-test: + filters: *filter-not-release - deploy-master: requires: - build-all @@ -218,6 +239,7 @@ workflows: - codespell - gometalinter - mysql-integration-test + - postgres-integration-test filters: branches: only: master @@ -235,6 +257,8 @@ workflows: filters: *filter-only-release - mysql-integration-test: filters: *filter-only-release + - postgres-integration-test: + filters: *filter-only-release - deploy-release: requires: - build-all @@ -243,4 +267,5 @@ workflows: - codespell - gometalinter - mysql-integration-test + - postgres-integration-test filters: *filter-only-release diff --git a/docker/blocks/postgres/docker-compose.yaml b/docker/blocks/postgres/docker-compose.yaml index 566df7b8877..27736042f7b 100644 --- a/docker/blocks/postgres/docker-compose.yaml +++ b/docker/blocks/postgres/docker-compose.yaml @@ -1,5 +1,5 @@ postgrestest: - image: postgres:latest + image: postgres:9.3 environment: POSTGRES_USER: grafana POSTGRES_PASSWORD: password @@ -13,4 +13,4 @@ network_mode: bridge environment: FD_DATASOURCE: postgres - FD_PORT: 5432 \ No newline at end of file + FD_PORT: 5432 diff --git a/docker/blocks/postgres_tests/Dockerfile b/docker/blocks/postgres_tests/Dockerfile index afe4d199651..df188e1094d 100644 --- a/docker/blocks/postgres_tests/Dockerfile +++ b/docker/blocks/postgres_tests/Dockerfile @@ -1,3 +1,3 @@ -FROM postgres:latest +FROM postgres:9.3 ADD setup.sql /docker-entrypoint-initdb.d -CMD ["postgres"] \ No newline at end of file +CMD ["postgres"] diff --git a/docker/blocks/postgres_tests/setup.sql b/docker/blocks/postgres_tests/setup.sql index b182b7c292d..3b8a48f938d 100644 --- a/docker/blocks/postgres_tests/setup.sql +++ b/docker/blocks/postgres_tests/setup.sql @@ -1,3 +1,3 @@ CREATE DATABASE grafanadstest; REVOKE CONNECT ON DATABASE grafanadstest FROM PUBLIC; -GRANT CONNECT ON DATABASE grafanadstest TO grafanatest; \ No newline at end of file +GRANT CONNECT ON DATABASE grafanadstest TO grafanatest; From b379b2833760a24a5f4221f178255dfcbb6f1254 Mon Sep 17 00:00:00 2001 From: Leonard Gram Date: Wed, 30 May 2018 15:16:31 +0200 Subject: [PATCH 72/87] build: only runs db related tests on db. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 46404e4e650..e898ad9e214 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,7 +30,7 @@ jobs: - run: cat docker/blocks/mysql_tests/setup.sql | mysql -h 127.0.0.1 -P 3306 -u root -prootpass - run: name: mysql integration tests - command: 'GRAFANA_TEST_DB=mysql go test ./pkg/...' + command: 'GRAFANA_TEST_DB=mysql go test ./pkg/services/sqlstore/... ./pkg/tsdb/mysql/... ' postgres-integration-test: docker: @@ -49,7 +49,7 @@ jobs: - run: 'PGPASSWORD=grafanatest psql -p 5432 -h 127.0.0.1 -U grafanatest -d grafanatest -f docker/blocks/postgres_tests/setup.sql' - run: name: postgres integration tests - command: 'GRAFANA_TEST_DB=postgres go test ./pkg/...' + command: 'GRAFANA_TEST_DB=postgres go test ./pkg/services/sqlstore/... ./pkg/tsdb/postgres/...' codespell: docker: From b894b5e669f94424b83b364238d2e7b254954989 Mon Sep 17 00:00:00 2001 From: Patrick O'Carroll Date: Wed, 30 May 2018 18:09:57 +0200 Subject: [PATCH 73/87] Fix singlestat threshold tooltip (#12109) fix singlestat threshold tooltip --- public/app/plugins/panel/singlestat/editor.html | 2 +- public/app/plugins/panel/singlestat/module.ts | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/public/app/plugins/panel/singlestat/editor.html b/public/app/plugins/panel/singlestat/editor.html index f444cd0170c..15f4e6a9efa 100644 --- a/public/app/plugins/panel/singlestat/editor.html +++ b/public/app/plugins/panel/singlestat/editor.html @@ -61,7 +61,7 @@
diff --git a/public/app/plugins/panel/singlestat/module.ts b/public/app/plugins/panel/singlestat/module.ts index b73a3bb32bd..20c4dcfeb70 100644 --- a/public/app/plugins/panel/singlestat/module.ts +++ b/public/app/plugins/panel/singlestat/module.ts @@ -714,11 +714,13 @@ function getColorForValue(data, value) { if (!_.isFinite(value)) { return null; } + for (var i = data.thresholds.length; i > 0; i--) { if (value >= data.thresholds[i - 1]) { return data.colorMap[i]; } } + return _.first(data.colorMap); } From a4b1dd036d04cd372a7475be425b9c53467f15c7 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 30 May 2018 18:11:47 +0200 Subject: [PATCH 74/87] changelog: add notes about closing #11971 [skip ci] --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a86eeba75e..3d77986b290 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ * **Graphite**: Don't send distributed tracing headers when using direct/browser access mode [#11494](https://github.com/grafana/grafana/issues/11494) * **Sidenav**: Show create dashboard link for viewers if at least editor in one folder [#11858](https://github.com/grafana/grafana/issues/11858) * **SQL**: Second epochs are now correctly converted to ms. [#12085](https://github.com/grafana/grafana/pull/12085) +* **Singlestat**: Fix singlestat threshold tooltip [#11971](https://github.com/grafana/grafana/issues/11971) # 5.1.3 (2018-05-16) From 82ba27b5f22c60153f620430937392151c3d312f Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 30 May 2018 21:31:31 +0200 Subject: [PATCH 75/87] changelog: add notes about closing #11771 [skip ci] --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d77986b290..5b756ea0102 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ * **Sidenav**: Show create dashboard link for viewers if at least editor in one folder [#11858](https://github.com/grafana/grafana/issues/11858) * **SQL**: Second epochs are now correctly converted to ms. [#12085](https://github.com/grafana/grafana/pull/12085) * **Singlestat**: Fix singlestat threshold tooltip [#11971](https://github.com/grafana/grafana/issues/11971) +* **Dashboard**: Hide grid controls in fullscreen/low-activity views [#11771](https://github.com/grafana/grafana/issues/11771) # 5.1.3 (2018-05-16) From d5aeae3a90e2cd7b1318b2d62a7e4516aabff9a0 Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 31 May 2018 08:27:29 +0200 Subject: [PATCH 76/87] test: fixes broken test on windows --- pkg/services/provisioning/dashboards/file_reader_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/services/provisioning/dashboards/file_reader_test.go b/pkg/services/provisioning/dashboards/file_reader_test.go index a04fbb23f82..87e9ec6d226 100644 --- a/pkg/services/provisioning/dashboards/file_reader_test.go +++ b/pkg/services/provisioning/dashboards/file_reader_test.go @@ -3,6 +3,7 @@ package dashboards import ( "os" "path/filepath" + "runtime" "testing" "time" @@ -52,7 +53,9 @@ func TestCreatingNewDashboardFileReader(t *testing.T) { reader, err := NewDashboardFileReader(cfg, log.New("test-logger")) So(err, ShouldBeNil) - So(reader.Path, ShouldEqual, "/var/lib/grafana/dashboards") + if runtime.GOOS != "windows" { + So(reader.Path, ShouldEqual, "/var/lib/grafana/dashboards") + } So(filepath.IsAbs(reader.Path), ShouldBeTrue) }) From 938deae4b467c2fcf4f35304dba7968e514f49a8 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Thu, 31 May 2018 15:24:01 +0200 Subject: [PATCH 77/87] changelog: add notes about closing #11515 [skip ci] --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b756ea0102..280d4429778 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ * **SQL**: Second epochs are now correctly converted to ms. [#12085](https://github.com/grafana/grafana/pull/12085) * **Singlestat**: Fix singlestat threshold tooltip [#11971](https://github.com/grafana/grafana/issues/11971) * **Dashboard**: Hide grid controls in fullscreen/low-activity views [#11771](https://github.com/grafana/grafana/issues/11771) +* **Dashboard**: Validate uid when importing dashboards [#11515](https://github.com/grafana/grafana/issues/11515) # 5.1.3 (2018-05-16) From 37f9bdfc8ce15f061d30c613e9c849a8907a6a54 Mon Sep 17 00:00:00 2001 From: Patrick O'Carroll Date: Thu, 31 May 2018 15:40:57 +0200 Subject: [PATCH 78/87] save modal ux improvements (#11822) changes to save modal when saving an updated dashboard Changed time range and variables are now not saved by default, you'll need to actively choose if you want to save updated time range and or variables. --- .../app/features/dashboard/dashboard_model.ts | 26 +++++- public/app/features/dashboard/save_modal.ts | 65 ++++++++++++-- .../dashboard/specs/dashboard_model.jest.ts | 59 ++++++++++++ .../dashboard/specs/save_modal.jest.ts | 90 +++++++++++++++++++ 4 files changed, 233 insertions(+), 7 deletions(-) create mode 100644 public/app/features/dashboard/specs/save_modal.jest.ts diff --git a/public/app/features/dashboard/dashboard_model.ts b/public/app/features/dashboard/dashboard_model.ts index 8a300a80341..a37e753bd89 100644 --- a/public/app/features/dashboard/dashboard_model.ts +++ b/public/app/features/dashboard/dashboard_model.ts @@ -22,8 +22,10 @@ export class DashboardModel { editable: any; graphTooltip: any; time: any; + originalTime: any; timepicker: any; templating: any; + originalTemplating: any; annotations: any; refresh: any; snapshot: any; @@ -68,8 +70,12 @@ export class DashboardModel { this.editable = data.editable !== false; this.graphTooltip = data.graphTooltip || 0; this.time = data.time || { from: 'now-6h', to: 'now' }; + this.originalTime = _.cloneDeep(this.time); this.timepicker = data.timepicker || {}; this.templating = this.ensureListExist(data.templating); + this.originalTemplating = _.map(this.templating.list, variable => { + return { name: variable.name, current: _.clone(variable.current) }; + }); this.annotations = this.ensureListExist(data.annotations); this.refresh = data.refresh; this.snapshot = data.snapshot; @@ -130,7 +136,12 @@ export class DashboardModel { } // cleans meta data and other non persistent state - getSaveModelClone() { + getSaveModelClone(options?) { + let defaults = _.defaults(options || {}, { + saveVariables: false, + saveTimerange: false, + }); + // make clone var copy: any = {}; for (var property in this) { @@ -142,10 +153,23 @@ export class DashboardModel { } // get variable save models + //console.log(this.templating.list); copy.templating = { list: _.map(this.templating.list, variable => (variable.getSaveModel ? variable.getSaveModel() : variable)), }; + if (!defaults.saveVariables && copy.templating.list.length === this.originalTemplating.length) { + for (let i = 0; i < copy.templating.list.length; i++) { + if (copy.templating.list[i].name === this.originalTemplating[i].name) { + copy.templating.list[i].current = this.originalTemplating[i].current; + } + } + } + + if (!defaults.saveTimerange) { + copy.time = this.originalTime; + } + // get panel save models copy.panels = _.chain(this.panels) .filter(panel => panel.type !== 'add-panel') diff --git a/public/app/features/dashboard/save_modal.ts b/public/app/features/dashboard/save_modal.ts index 33165758555..1c364fbc55f 100644 --- a/public/app/features/dashboard/save_modal.ts +++ b/public/app/features/dashboard/save_modal.ts @@ -1,4 +1,5 @@ import coreModule from 'app/core/core_module'; +import _ from 'lodash'; const template = `
@@ -14,19 +15,29 @@ const template = `
-
Add a note to describe your changes
-
+
+
+ + + + +
- + Cancel
@@ -48,14 +59,51 @@ const template = ` export class SaveDashboardModalCtrl { message: string; + saveVariables = false; + saveTimerange = false; + templating: any; + time: any; + originalTime: any; + current = []; + originalCurrent = []; max: number; saveForm: any; dismiss: () => void; + timeChange = false; + variableChange = false; /** @ngInject */ constructor(private dashboardSrv) { this.message = ''; this.max = 64; + this.templating = dashboardSrv.dash.templating.list; + + this.compareTemplating(); + this.compareTime(); + } + + compareTime() { + if (_.isEqual(this.dashboardSrv.dash.time, this.dashboardSrv.dash.originalTime)) { + this.timeChange = false; + } else { + this.timeChange = true; + } + } + + compareTemplating() { + if (this.dashboardSrv.dash.templating.list.length > 0) { + for (let i = 0; i < this.dashboardSrv.dash.templating.list.length; i++) { + if ( + this.dashboardSrv.dash.templating.list[i].current.text !== + this.dashboardSrv.dash.originalTemplating[i].current.text + ) { + return (this.variableChange = true); + } + } + return (this.variableChange = false); + } else { + return (this.variableChange = false); + } } save() { @@ -63,9 +111,14 @@ export class SaveDashboardModalCtrl { return; } + var options = { + saveVariables: this.saveVariables, + saveTimerange: this.saveTimerange, + message: this.message, + }; + var dashboard = this.dashboardSrv.getCurrent(); - var saveModel = dashboard.getSaveModelClone(); - var options = { message: this.message }; + var saveModel = dashboard.getSaveModelClone(options); return this.dashboardSrv.save(saveModel, options).then(this.dismiss); } diff --git a/public/app/features/dashboard/specs/dashboard_model.jest.ts b/public/app/features/dashboard/specs/dashboard_model.jest.ts index 6f0b45c9ba8..3fb5b3c3b73 100644 --- a/public/app/features/dashboard/specs/dashboard_model.jest.ts +++ b/public/app/features/dashboard/specs/dashboard_model.jest.ts @@ -434,4 +434,63 @@ describe('DashboardModel', function() { }); }); }); + + describe('save variables and timeline', () => { + let model; + + beforeEach(() => { + model = new DashboardModel({ + templating: { + list: [ + { + name: 'Server', + current: { + selected: true, + text: 'server_001', + value: 'server_001', + }, + }, + ], + }, + time: { + from: 'now-6h', + to: 'now', + }, + }); + model.templating.list[0] = { + name: 'Server', + current: { + selected: true, + text: 'server_002', + value: 'server_002', + }, + }; + model.time = { + from: 'now-3h', + to: 'now', + }; + }); + + it('should not save variables and timeline', () => { + let options = { + saveVariables: false, + saveTimerange: false, + }; + let saveModel = model.getSaveModelClone(options); + + expect(saveModel.templating.list[0].current.text).toBe('server_001'); + expect(saveModel.time.from).toBe('now-6h'); + }); + + it('should save variables and timeline', () => { + let options = { + saveVariables: true, + saveTimerange: true, + }; + let saveModel = model.getSaveModelClone(options); + + expect(saveModel.templating.list[0].current.text).toBe('server_002'); + expect(saveModel.time.from).toBe('now-3h'); + }); + }); }); diff --git a/public/app/features/dashboard/specs/save_modal.jest.ts b/public/app/features/dashboard/specs/save_modal.jest.ts new file mode 100644 index 00000000000..05281cc6097 --- /dev/null +++ b/public/app/features/dashboard/specs/save_modal.jest.ts @@ -0,0 +1,90 @@ +import { SaveDashboardModalCtrl } from '../save_modal'; + +jest.mock('app/core/services/context_srv', () => ({})); + +describe('SaveDashboardModal', () => { + describe('save modal checkboxes', () => { + it('should show checkboxes', () => { + let fakeDashboardSrv = { + dash: { + templating: { + list: [ + { + current: { + selected: true, + tags: Array(0), + text: 'server_001', + value: 'server_001', + }, + name: 'Server', + }, + ], + }, + originalTemplating: [ + { + current: { + selected: true, + text: 'server_002', + value: 'server_002', + }, + name: 'Server', + }, + ], + time: { + from: 'now-3h', + to: 'now', + }, + originalTime: { + from: 'now-6h', + to: 'now', + }, + }, + }; + let modal = new SaveDashboardModalCtrl(fakeDashboardSrv); + + expect(modal.timeChange).toBe(true); + expect(modal.variableChange).toBe(true); + }); + + it('should hide checkboxes', () => { + let fakeDashboardSrv = { + dash: { + templating: { + list: [ + { + current: { + selected: true, + //tags: Array(0), + text: 'server_002', + value: 'server_002', + }, + name: 'Server', + }, + ], + }, + originalTemplating: [ + { + current: { + selected: true, + text: 'server_002', + value: 'server_002', + }, + name: 'Server', + }, + ], + time: { + from: 'now-3h', + to: 'now', + }, + originalTime: { + from: 'now-3h', + to: 'now', + }, + }, + }; + let modal = new SaveDashboardModalCtrl(fakeDashboardSrv); + expect(modal.timeChange).toBe(false); + expect(modal.variableChange).toBe(false); + }); + }); +}); From 67410f7a4d13be8a491f4deb5192b967faf63235 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Thu, 31 May 2018 15:48:19 +0200 Subject: [PATCH 79/87] changelog: add notes about closing #10748, #8805 [skip ci] --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 280d4429778..1fa0d9173f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # 5.2.0 (unreleased) +### Breaking change + +* **Dashboard**: Modified time range and variables are now not saved by default [#10748](https://github.com/grafana/grafana/issues/10748), [#8805](https://github.com/grafana/grafana/issues/8805) + ### Minor * **Graph**: Show invisible highest value bucket in histogram [#11498](https://github.com/grafana/grafana/issues/11498) From a2cd05f6dbecb9b15455ad57aa8d97c24bdc53f4 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Thu, 31 May 2018 16:31:53 +0200 Subject: [PATCH 80/87] changelog: update [skip ci] --- CHANGELOG.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1fa0d9173f7..7ad20e5e5bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,8 @@ # 5.2.0 (unreleased) -### Breaking change - -* **Dashboard**: Modified time range and variables are now not saved by default [#10748](https://github.com/grafana/grafana/issues/10748), [#8805](https://github.com/grafana/grafana/issues/8805) - ### Minor +* **Dashboard**: Modified time range and variables are now not saved by default [#10748](https://github.com/grafana/grafana/issues/10748), [#8805](https://github.com/grafana/grafana/issues/8805) * **Graph**: Show invisible highest value bucket in histogram [#11498](https://github.com/grafana/grafana/issues/11498) * **Dashboard**: Enable "Save As..." if user has edit permission [#11625](https://github.com/grafana/grafana/issues/11625) * **Prometheus**: Table columns order now changes when rearrange queries [#11690](https://github.com/grafana/grafana/issues/11690), thx [@mtanda](https://github.com/mtanda) From dcac63936bbff64fb0e9277b216a0e1e27955d51 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Thu, 31 May 2018 19:02:20 +0200 Subject: [PATCH 81/87] elasticsearch: minor refactor Handle all replacements if interval template variables in the client. Fix issue with client and different versions. Adds better tests of the client --- pkg/tsdb/elasticsearch/client/client.go | 128 +++----- pkg/tsdb/elasticsearch/client/client_test.go | 279 ++++++++++++------ pkg/tsdb/elasticsearch/client/models.go | 3 + .../elasticsearch/client/search_request.go | 11 +- .../client/search_request_test.go | 12 +- pkg/tsdb/elasticsearch/time_series_query.go | 12 +- .../elasticsearch/time_series_query_test.go | 2 +- 7 files changed, 252 insertions(+), 195 deletions(-) diff --git a/pkg/tsdb/elasticsearch/client/client.go b/pkg/tsdb/elasticsearch/client/client.go index 3a2d31b42c6..8ae45599484 100644 --- a/pkg/tsdb/elasticsearch/client/client.go +++ b/pkg/tsdb/elasticsearch/client/client.go @@ -8,6 +8,7 @@ import ( "net/http" "net/url" "path" + "strconv" "strings" "time" @@ -25,6 +26,10 @@ var ( clientLog = log.New(loggerName) ) +var newDatasourceHttpClient = func(ds *models.DataSource) (*http.Client, error) { + return ds.GetHttpClient() +} + // Client represents a client which can interact with elasticsearch api type Client interface { GetVersion() int @@ -57,23 +62,18 @@ var NewClient = func(ctx context.Context, ds *models.DataSource, timeRange *tsdb return nil, err } - bc := &baseClientImpl{ - ctx: ctx, - ds: ds, - version: version, - timeField: timeField, - indices: indices, - } - clientLog.Debug("Creating new client", "version", version, "timeField", timeField, "indices", strings.Join(indices, ", ")) switch version { - case 2: - return newV2Client(bc) - case 5: - return newV5Client(bc) - case 56: - return newV56Client(bc) + case 2, 5, 56: + return &baseClientImpl{ + ctx: ctx, + ds: ds, + version: version, + timeField: timeField, + indices: indices, + timeRange: timeRange, + }, nil } return nil, fmt.Errorf("elasticsearch version=%d is not supported", version) @@ -93,6 +93,7 @@ type baseClientImpl struct { version int timeField string indices []string + timeRange *tsdb.TimeRange } func (c *baseClientImpl) GetVersion() int { @@ -114,11 +115,20 @@ func (c *baseClientImpl) getSettings() *simplejson.Json { } type multiRequest struct { - header map[string]interface{} - body interface{} + header map[string]interface{} + body interface{} + interval tsdb.Interval } func (c *baseClientImpl) executeBatchRequest(uriPath string, requests []*multiRequest) (*http.Response, error) { + bytes, err := c.encodeBatchRequests(requests) + if err != nil { + return nil, err + } + return c.executeRequest(http.MethodPost, uriPath, bytes) +} + +func (c *baseClientImpl) encodeBatchRequests(requests []*multiRequest) ([]byte, error) { clientLog.Debug("Encoding batch requests to json", "batch requests", len(requests)) start := time.Now() @@ -134,13 +144,18 @@ func (c *baseClientImpl) executeBatchRequest(uriPath string, requests []*multiRe if err != nil { return nil, err } - payload.WriteString(string(reqBody) + "\n") + + body := string(reqBody) + body = strings.Replace(body, "$__interval_ms", strconv.FormatInt(r.interval.Value.Nanoseconds()/int64(time.Millisecond), 10), -1) + body = strings.Replace(body, "$__interval", r.interval.Text, -1) + + payload.WriteString(body + "\n") } elapsed := time.Now().Sub(start) clientLog.Debug("Encoded batch requests to json", "took", elapsed) - return c.executeRequest(http.MethodPost, uriPath, payload.Bytes()) + return payload.Bytes(), nil } func (c *baseClientImpl) executeRequest(method, uriPath string, body []byte) (*http.Response, error) { @@ -173,7 +188,7 @@ func (c *baseClientImpl) executeRequest(method, uriPath string, body []byte) (*h req.SetBasicAuth(c.ds.User, c.ds.Password) } - httpClient, err := c.ds.GetHttpClient() + httpClient, err := newDatasourceHttpClient(c.ds) if err != nil { return nil, err } @@ -220,77 +235,26 @@ func (c *baseClientImpl) createMultiSearchRequests(searchRequests []*SearchReque multiRequests := []*multiRequest{} for _, searchReq := range searchRequests { - multiRequests = append(multiRequests, &multiRequest{ + mr := multiRequest{ header: map[string]interface{}{ "search_type": "query_then_fetch", "ignore_unavailable": true, "index": strings.Join(c.indices, ","), }, - body: searchReq, - }) - } + body: searchReq, + interval: searchReq.Interval, + } - return multiRequests -} + if c.version == 2 { + mr.header["search_type"] = "count" + } -type v2Client struct { - baseClient -} + if c.version >= 56 { + maxConcurrentShardRequests := c.getSettings().Get("maxConcurrentShardRequests").MustInt(256) + mr.header["max_concurrent_shard_requests"] = maxConcurrentShardRequests + } -func newV2Client(bc baseClient) (*v2Client, error) { - c := v2Client{ - baseClient: bc, - } - - return &c, nil -} - -func (c *v2Client) createMultiSearchRequests(searchRequests []*SearchRequest) []*multiRequest { - multiRequests := c.baseClient.createMultiSearchRequests(searchRequests) - - for _, mr := range multiRequests { - mr.header["search_type"] = "count" - } - - return multiRequests -} - -type v5Client struct { - baseClient -} - -func newV5Client(bc baseClient) (*v5Client, error) { - c := v5Client{ - baseClient: bc, - } - - return &c, nil -} - -type v56Client struct { - *v5Client - maxConcurrentShardRequests int -} - -func newV56Client(bc baseClient) (*v56Client, error) { - v5Client := v5Client{ - baseClient: bc, - } - maxConcurrentShardRequests := bc.getSettings().Get("maxConcurrentShardRequests").MustInt(256) - - c := v56Client{ - v5Client: &v5Client, - maxConcurrentShardRequests: maxConcurrentShardRequests, - } - - return &c, nil -} - -func (c *v56Client) createMultiSearchRequests(searchRequests []*SearchRequest) []*multiRequest { - multiRequests := c.v5Client.createMultiSearchRequests(searchRequests) - - for _, mr := range multiRequests { - mr.header["max_concurrent_shard_requests"] = c.maxConcurrentShardRequests + multiRequests = append(multiRequests, &mr) } return multiRequests diff --git a/pkg/tsdb/elasticsearch/client/client_test.go b/pkg/tsdb/elasticsearch/client/client_test.go index d557ceb28b1..11d1cdb1d71 100644 --- a/pkg/tsdb/elasticsearch/client/client_test.go +++ b/pkg/tsdb/elasticsearch/client/client_test.go @@ -1,10 +1,17 @@ package es import ( + "bytes" + "context" + "fmt" + "io/ioutil" "net/http" + "net/http/httptest" "testing" + "time" "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/tsdb" "github.com/grafana/grafana/pkg/models" . "github.com/smartystreets/goconvey/convey" @@ -85,131 +92,213 @@ func TestClient(t *testing.T) { }) }) - Convey("v2", func() { - ds := &models.DataSource{ - JsonData: simplejson.NewFromAny(map[string]interface{}{ - "esVersion": 2, - }), + Convey("Given a fake http client", func() { + var responseBuffer *bytes.Buffer + var req *http.Request + ts := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + req = r + buf, err := ioutil.ReadAll(r.Body) + if err != nil { + t.Fatalf("Failed to read response body, err=%v", err) + } + responseBuffer = bytes.NewBuffer(buf) + })) + + currentNewDatasourceHttpClient := newDatasourceHttpClient + + newDatasourceHttpClient = func(ds *models.DataSource) (*http.Client, error) { + return ts.Client(), nil } - c, err := newV2Client(newFakeBaseClient(ds, []string{"test-*"})) - So(err, ShouldBeNil) - So(c, ShouldNotBeNil) + from := time.Date(2018, 5, 15, 17, 50, 0, 0, time.UTC) + to := time.Date(2018, 5, 15, 17, 55, 0, 0, time.UTC) + fromStr := fmt.Sprintf("%d", from.UnixNano()/int64(time.Millisecond)) + toStr := fmt.Sprintf("%d", to.UnixNano()/int64(time.Millisecond)) + timeRange := tsdb.NewTimeRange(fromStr, toStr) - Convey("When creating multisearch requests should have correct headers", func() { - multiRequests := c.createMultiSearchRequests([]*SearchRequest{ - {Index: "test-*"}, - }) - So(multiRequests, ShouldHaveLength, 1) - header := multiRequests[0].header - So(header, ShouldHaveLength, 3) - So(header["index"], ShouldEqual, "test-*") - So(header["ignore_unavailable"], ShouldEqual, true) - So(header["search_type"], ShouldEqual, "count") - }) - }) - - Convey("v5", func() { - ds := &models.DataSource{ - JsonData: simplejson.NewFromAny(map[string]interface{}{ - "esVersion": 5, - }), - } - - c, err := newV5Client(newFakeBaseClient(ds, []string{"test-*"})) - So(err, ShouldBeNil) - So(c, ShouldNotBeNil) - - Convey("When creating multisearch requests should have correct headers", func() { - multiRequests := c.createMultiSearchRequests([]*SearchRequest{ - {Index: "test-*"}, - }) - So(multiRequests, ShouldHaveLength, 1) - header := multiRequests[0].header - So(header, ShouldHaveLength, 3) - So(header["index"], ShouldEqual, "test-*") - So(header["ignore_unavailable"], ShouldEqual, true) - So(header["search_type"], ShouldEqual, "query_then_fetch") - }) - }) - - Convey("v5.6", func() { - Convey("With default settings", func() { + Convey("and a v2.x client", func() { ds := models.DataSource{ + Database: "[metrics-]YYYY.MM.DD", + Url: ts.URL, JsonData: simplejson.NewFromAny(map[string]interface{}{ - "esVersion": 56, + "esVersion": 2, + "timeField": "@timestamp", + "interval": "Daily", }), } - c, err := newV56Client(newFakeBaseClient(&ds, []string{"test-*"})) + c, err := NewClient(context.Background(), &ds, timeRange) So(err, ShouldBeNil) So(c, ShouldNotBeNil) - Convey("When creating multisearch requests should have correct headers", func() { - multiRequests := c.createMultiSearchRequests([]*SearchRequest{ - {Index: "test-*"}, + Convey("When executing multi search", func() { + ms, err := createMultisearchForTest(c) + So(err, ShouldBeNil) + c.ExecuteMultisearch(ms) + + Convey("Should send correct request and payload", func() { + So(req, ShouldNotBeNil) + So(req.Method, ShouldEqual, http.MethodPost) + So(req.URL.Path, ShouldEqual, "/_msearch") + + So(responseBuffer, ShouldNotBeNil) + + headerBytes, err := responseBuffer.ReadBytes('\n') + So(err, ShouldBeNil) + bodyBytes := responseBuffer.Bytes() + + jHeader, err := simplejson.NewJson(headerBytes) + So(err, ShouldBeNil) + + jBody, err := simplejson.NewJson(bodyBytes) + So(err, ShouldBeNil) + + fmt.Println("body", string(headerBytes)) + + So(jHeader.Get("index").MustString(), ShouldEqual, "metrics-2018.05.15") + So(jHeader.Get("ignore_unavailable").MustBool(false), ShouldEqual, true) + So(jHeader.Get("search_type").MustString(), ShouldEqual, "count") + So(jHeader.Get("max_concurrent_shard_requests").MustInt(10), ShouldEqual, 10) + + Convey("and replace $__interval variable", func() { + So(jBody.GetPath("aggs", "2", "aggs", "1", "avg", "script").MustString(), ShouldEqual, "15000*@hostname") + }) + + Convey("and replace $__interval_ms variable", func() { + So(jBody.GetPath("aggs", "2", "date_histogram", "interval").MustString(), ShouldEqual, "15s") + }) }) - So(multiRequests, ShouldHaveLength, 1) - header := multiRequests[0].header - So(header, ShouldHaveLength, 4) - So(header["index"], ShouldEqual, "test-*") - So(header["ignore_unavailable"], ShouldEqual, true) - So(header["search_type"], ShouldEqual, "query_then_fetch") - So(header["max_concurrent_shard_requests"], ShouldEqual, 256) }) }) - Convey("With custom settings", func() { + Convey("and a v5.x client", func() { ds := models.DataSource{ + Database: "[metrics-]YYYY.MM.DD", + Url: ts.URL, + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 5, + "maxConcurrentShardRequests": 100, + "timeField": "@timestamp", + "interval": "Daily", + }), + } + + c, err := NewClient(context.Background(), &ds, timeRange) + So(err, ShouldBeNil) + So(c, ShouldNotBeNil) + + Convey("When executing multi search", func() { + ms, err := createMultisearchForTest(c) + So(err, ShouldBeNil) + c.ExecuteMultisearch(ms) + + Convey("Should send correct request and payload", func() { + So(req, ShouldNotBeNil) + So(req.Method, ShouldEqual, http.MethodPost) + So(req.URL.Path, ShouldEqual, "/_msearch") + + So(responseBuffer, ShouldNotBeNil) + + headerBytes, err := responseBuffer.ReadBytes('\n') + So(err, ShouldBeNil) + bodyBytes := responseBuffer.Bytes() + + jHeader, err := simplejson.NewJson(headerBytes) + So(err, ShouldBeNil) + + jBody, err := simplejson.NewJson(bodyBytes) + So(err, ShouldBeNil) + + fmt.Println("body", string(headerBytes)) + + So(jHeader.Get("index").MustString(), ShouldEqual, "metrics-2018.05.15") + So(jHeader.Get("ignore_unavailable").MustBool(false), ShouldEqual, true) + So(jHeader.Get("search_type").MustString(), ShouldEqual, "query_then_fetch") + So(jHeader.Get("max_concurrent_shard_requests").MustInt(10), ShouldEqual, 10) + + Convey("and replace $__interval variable", func() { + So(jBody.GetPath("aggs", "2", "aggs", "1", "avg", "script").MustString(), ShouldEqual, "15000*@hostname") + }) + + Convey("and replace $__interval_ms variable", func() { + So(jBody.GetPath("aggs", "2", "date_histogram", "interval").MustString(), ShouldEqual, "15s") + }) + }) + }) + }) + + Convey("and a v5.6 client", func() { + ds := models.DataSource{ + Database: "[metrics-]YYYY.MM.DD", + Url: ts.URL, JsonData: simplejson.NewFromAny(map[string]interface{}{ "esVersion": 56, "maxConcurrentShardRequests": 100, + "timeField": "@timestamp", + "interval": "Daily", }), } - c, err := newV56Client(newFakeBaseClient(&ds, []string{"test-*"})) + c, err := NewClient(context.Background(), &ds, timeRange) So(err, ShouldBeNil) So(c, ShouldNotBeNil) - Convey("When creating multisearch requests should have correct headers", func() { - multiRequests := c.createMultiSearchRequests([]*SearchRequest{ - {Index: "test-*"}, + + Convey("When executing multi search", func() { + ms, err := createMultisearchForTest(c) + So(err, ShouldBeNil) + c.ExecuteMultisearch(ms) + + Convey("Should send correct request and payload", func() { + So(req, ShouldNotBeNil) + So(req.Method, ShouldEqual, http.MethodPost) + So(req.URL.Path, ShouldEqual, "/_msearch") + + So(responseBuffer, ShouldNotBeNil) + + headerBytes, err := responseBuffer.ReadBytes('\n') + So(err, ShouldBeNil) + bodyBytes := responseBuffer.Bytes() + + jHeader, err := simplejson.NewJson(headerBytes) + So(err, ShouldBeNil) + + jBody, err := simplejson.NewJson(bodyBytes) + So(err, ShouldBeNil) + + fmt.Println("body", string(headerBytes)) + + So(jHeader.Get("index").MustString(), ShouldEqual, "metrics-2018.05.15") + So(jHeader.Get("ignore_unavailable").MustBool(false), ShouldEqual, true) + So(jHeader.Get("search_type").MustString(), ShouldEqual, "query_then_fetch") + So(jHeader.Get("max_concurrent_shard_requests").MustInt(), ShouldEqual, 100) + + Convey("and replace $__interval variable", func() { + So(jBody.GetPath("aggs", "2", "aggs", "1", "avg", "script").MustString(), ShouldEqual, "15000*@hostname") + }) + + Convey("and replace $__interval_ms variable", func() { + So(jBody.GetPath("aggs", "2", "date_histogram", "interval").MustString(), ShouldEqual, "15s") + }) }) - So(multiRequests, ShouldHaveLength, 1) - header := multiRequests[0].header - So(header, ShouldHaveLength, 4) - So(header["index"], ShouldEqual, "test-*") - So(header["ignore_unavailable"], ShouldEqual, true) - So(header["search_type"], ShouldEqual, "query_then_fetch") - So(header["max_concurrent_shard_requests"], ShouldEqual, 100) }) }) + + Reset(func() { + newDatasourceHttpClient = currentNewDatasourceHttpClient + }) }) }) } -type fakeBaseClient struct { - *baseClientImpl - ds *models.DataSource -} +func createMultisearchForTest(c Client) (*MultiSearchRequest, error) { + msb := c.MultiSearch() + s := msb.Search(tsdb.Interval{Value: 15 * time.Second, Text: "15s"}) + s.Agg().DateHistogram("2", "@timestamp", func(a *DateHistogramAgg, ab AggBuilder) { + a.Interval = "$__interval" -func newFakeBaseClient(ds *models.DataSource, indices []string) baseClient { - return &fakeBaseClient{ - baseClientImpl: &baseClientImpl{ - ds: ds, - indices: indices, - }, - ds: ds, - } -} - -func (c *fakeBaseClient) executeBatchRequest(uriPath string, requests []*multiRequest) (*http.Response, error) { - return nil, nil -} - -func (c *fakeBaseClient) executeRequest(method, uriPath string, body []byte) (*http.Response, error) { - return nil, nil -} - -func (c *fakeBaseClient) executeMultisearch(searchRequests []*SearchRequest) ([]*SearchResponse, error) { - return nil, nil + ab.Metric("1", "avg", "@hostname", func(a *MetricAggregation) { + a.Settings["script"] = "$__interval_ms*@hostname" + }) + }) + return msb.Build() } diff --git a/pkg/tsdb/elasticsearch/client/models.go b/pkg/tsdb/elasticsearch/client/models.go index 2f4f5dcd162..a5810a9b109 100644 --- a/pkg/tsdb/elasticsearch/client/models.go +++ b/pkg/tsdb/elasticsearch/client/models.go @@ -2,11 +2,14 @@ package es import ( "encoding/json" + + "github.com/grafana/grafana/pkg/tsdb" ) // SearchRequest represents a search request type SearchRequest struct { Index string + Interval tsdb.Interval Size int Sort map[string]interface{} Query *Query diff --git a/pkg/tsdb/elasticsearch/client/search_request.go b/pkg/tsdb/elasticsearch/client/search_request.go index a582d8ec247..2b833ce78d3 100644 --- a/pkg/tsdb/elasticsearch/client/search_request.go +++ b/pkg/tsdb/elasticsearch/client/search_request.go @@ -2,11 +2,14 @@ package es import ( "strings" + + "github.com/grafana/grafana/pkg/tsdb" ) // SearchRequestBuilder represents a builder which can build a search request type SearchRequestBuilder struct { version int + interval tsdb.Interval index string size int sort map[string]interface{} @@ -16,9 +19,10 @@ type SearchRequestBuilder struct { } // NewSearchRequestBuilder create a new search request builder -func NewSearchRequestBuilder(version int) *SearchRequestBuilder { +func NewSearchRequestBuilder(version int, interval tsdb.Interval) *SearchRequestBuilder { builder := &SearchRequestBuilder{ version: version, + interval: interval, sort: make(map[string]interface{}), customProps: make(map[string]interface{}), aggBuilders: make([]AggBuilder, 0), @@ -30,6 +34,7 @@ func NewSearchRequestBuilder(version int) *SearchRequestBuilder { func (b *SearchRequestBuilder) Build() (*SearchRequest, error) { sr := SearchRequest{ Index: b.index, + Interval: b.interval, Size: b.size, Sort: b.sort, CustomProps: b.customProps, @@ -128,8 +133,8 @@ func NewMultiSearchRequestBuilder(version int) *MultiSearchRequestBuilder { } // Search initiates and returns a new search request builder -func (m *MultiSearchRequestBuilder) Search() *SearchRequestBuilder { - b := NewSearchRequestBuilder(m.version) +func (m *MultiSearchRequestBuilder) Search(interval tsdb.Interval) *SearchRequestBuilder { + b := NewSearchRequestBuilder(m.version, interval) m.requestBuilders = append(m.requestBuilders, b) return b } diff --git a/pkg/tsdb/elasticsearch/client/search_request_test.go b/pkg/tsdb/elasticsearch/client/search_request_test.go index d93f8826442..b026578d64f 100644 --- a/pkg/tsdb/elasticsearch/client/search_request_test.go +++ b/pkg/tsdb/elasticsearch/client/search_request_test.go @@ -3,8 +3,10 @@ package es import ( "encoding/json" "testing" + "time" "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/tsdb" . "github.com/smartystreets/goconvey/convey" ) @@ -13,7 +15,7 @@ func TestSearchRequest(t *testing.T) { Convey("Test elasticsearch search request", t, func() { timeField := "@timestamp" Convey("Given new search request builder for es version 5", func() { - b := NewSearchRequestBuilder(5) + b := NewSearchRequestBuilder(5, tsdb.Interval{Value: 15 * time.Second, Text: "15s"}) Convey("When building search request", func() { sr, err := b.Build() @@ -388,7 +390,7 @@ func TestSearchRequest(t *testing.T) { }) Convey("Given new search request builder for es version 2", func() { - b := NewSearchRequestBuilder(2) + b := NewSearchRequestBuilder(2, tsdb.Interval{Value: 15 * time.Second, Text: "15s"}) Convey("When adding doc value field", func() { b.AddDocValueField(timeField) @@ -447,7 +449,7 @@ func TestMultiSearchRequest(t *testing.T) { b := NewMultiSearchRequestBuilder(0) Convey("When adding one search request", func() { - b.Search() + b.Search(tsdb.Interval{Value: 15 * time.Second, Text: "15s"}) Convey("When building search request should contain one search request", func() { mr, err := b.Build() @@ -457,8 +459,8 @@ func TestMultiSearchRequest(t *testing.T) { }) Convey("When adding two search requests", func() { - b.Search() - b.Search() + b.Search(tsdb.Interval{Value: 15 * time.Second, Text: "15s"}) + b.Search(tsdb.Interval{Value: 15 * time.Second, Text: "15s"}) Convey("When building search request should contain two search requests", func() { mr, err := b.Build() diff --git a/pkg/tsdb/elasticsearch/time_series_query.go b/pkg/tsdb/elasticsearch/time_series_query.go index ef59c62c1dc..c9bb05dd09a 100644 --- a/pkg/tsdb/elasticsearch/time_series_query.go +++ b/pkg/tsdb/elasticsearch/time_series_query.go @@ -3,8 +3,6 @@ package elasticsearch import ( "fmt" "strconv" - "strings" - "time" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/tsdb" @@ -47,7 +45,7 @@ func (e *timeSeriesQuery) execute() (*tsdb.Response, error) { } interval := e.intervalCalculator.Calculate(e.tsdbQuery.TimeRange, minInterval) - b := ms.Search() + b := ms.Search(interval) b.Size(0) filters := b.Query().Bool().Filter() filters.AddDateRangeFilter(e.client.GetTimeField(), to, from, es.DateFormatEpochMS) @@ -78,7 +76,7 @@ func (e *timeSeriesQuery) execute() (*tsdb.Response, error) { for _, bucketAgg := range q.BucketAggs { switch bucketAgg.Type { case "date_histogram": - aggBuilder = addDateHistogramAgg(aggBuilder, bucketAgg, from, to, interval) + aggBuilder = addDateHistogramAgg(aggBuilder, bucketAgg, from, to) case "histogram": aggBuilder = addHistogramAgg(aggBuilder, bucketAgg) case "filters": @@ -125,7 +123,7 @@ func (e *timeSeriesQuery) execute() (*tsdb.Response, error) { return rp.getTimeSeries() } -func addDateHistogramAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg, timeFrom, timeTo string, interval tsdb.Interval) es.AggBuilder { +func addDateHistogramAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg, timeFrom, timeTo string) es.AggBuilder { aggBuilder.DateHistogram(bucketAgg.ID, bucketAgg.Field, func(a *es.DateHistogramAgg, b es.AggBuilder) { a.Interval = bucketAgg.Settings.Get("interval").MustString("auto") a.MinDocCount = bucketAgg.Settings.Get("min_doc_count").MustInt(0) @@ -136,10 +134,6 @@ func addDateHistogramAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg, timeFro a.Interval = "$__interval" } - a.Interval = strings.Replace(a.Interval, "$interval", interval.Text, -1) - a.Interval = strings.Replace(a.Interval, "$__interval_ms", strconv.FormatInt(interval.Value.Nanoseconds()/int64(time.Millisecond), 10), -1) - a.Interval = strings.Replace(a.Interval, "$__interval", interval.Text, -1) - if missing, err := bucketAgg.Settings.Get("missing").String(); err == nil { a.Missing = &missing } diff --git a/pkg/tsdb/elasticsearch/time_series_query_test.go b/pkg/tsdb/elasticsearch/time_series_query_test.go index e2af4de749a..49bf5f5bc75 100644 --- a/pkg/tsdb/elasticsearch/time_series_query_test.go +++ b/pkg/tsdb/elasticsearch/time_series_query_test.go @@ -268,7 +268,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { So(firstLevel.Aggregation.Type, ShouldEqual, "date_histogram") hAgg := firstLevel.Aggregation.Aggregation.(*es.DateHistogramAgg) So(hAgg.Field, ShouldEqual, "@timestamp") - So(hAgg.Interval, ShouldEqual, "15s") + So(hAgg.Interval, ShouldEqual, "$__interval") So(hAgg.MinDocCount, ShouldEqual, 2) }) From b8ff3b1e3fa2fa061aaad41f2e44edb15272150b Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Thu, 31 May 2018 19:05:32 +0200 Subject: [PATCH 82/87] remove dead code --- pkg/tsdb/elasticsearch/client/client.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pkg/tsdb/elasticsearch/client/client.go b/pkg/tsdb/elasticsearch/client/client.go index 8ae45599484..0cd3eb0d21b 100644 --- a/pkg/tsdb/elasticsearch/client/client.go +++ b/pkg/tsdb/elasticsearch/client/client.go @@ -79,14 +79,6 @@ var NewClient = func(ctx context.Context, ds *models.DataSource, timeRange *tsdb return nil, fmt.Errorf("elasticsearch version=%d is not supported", version) } -type baseClient interface { - Client - getSettings() *simplejson.Json - executeBatchRequest(uriPath string, requests []*multiRequest) (*http.Response, error) - executeRequest(method, uriPath string, body []byte) (*http.Response, error) - createMultiSearchRequests(searchRequests []*SearchRequest) []*multiRequest -} - type baseClientImpl struct { ctx context.Context ds *models.DataSource From 13a9701581a38e6c18ee3bc39709498bd4672be3 Mon Sep 17 00:00:00 2001 From: David Date: Fri, 1 Jun 2018 10:13:48 +0200 Subject: [PATCH 83/87] Update CHANGELOG.md - added Prometheus query date alignment --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ad20e5e5bd..6d7e46d6cf4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ * **Dashboard**: Modified time range and variables are now not saved by default [#10748](https://github.com/grafana/grafana/issues/10748), [#8805](https://github.com/grafana/grafana/issues/8805) * **Graph**: Show invisible highest value bucket in histogram [#11498](https://github.com/grafana/grafana/issues/11498) * **Dashboard**: Enable "Save As..." if user has edit permission [#11625](https://github.com/grafana/grafana/issues/11625) +* **Prometheus**: Query dates are now step-aligned [#10434](https://github.com/grafana/grafana/pull/10434) * **Prometheus**: Table columns order now changes when rearrange queries [#11690](https://github.com/grafana/grafana/issues/11690), thx [@mtanda](https://github.com/mtanda) * **Variables**: Fix variable interpolation when using multiple formatting types [#11800](https://github.com/grafana/grafana/issues/11800), thx [@svenklemm](https://github.com/svenklemm) * **Dashboard**: Fix date selector styling for dark/light theme in time picker control [#11616](https://github.com/grafana/grafana/issues/11616) From ce75afa413defc682fab6a22f02dad45c644f7ce Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Fri, 1 Jun 2018 12:26:20 +0200 Subject: [PATCH 84/87] docs: update alerting docs with alerting support for elasticsearch --- docs/sources/alerting/rules.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/sources/alerting/rules.md b/docs/sources/alerting/rules.md index bcca3c6b2fb..fa7332e7145 100644 --- a/docs/sources/alerting/rules.md +++ b/docs/sources/alerting/rules.md @@ -27,7 +27,9 @@ and the conditions that need to be met for the alert to change state and trigger ## Execution The alert rules are evaluated in the Grafana backend in a scheduler and query execution engine that is part -of core Grafana. Only some data sources are supported right now. They include `Graphite`, `Prometheus`, `InfluxDB`, `OpenTSDB`, `MySQL`, `Postgres` and `Cloudwatch`. +of core Grafana. Only some data sources are supported right now. They include `Graphite`, `Prometheus`, `Elasticsearch`, `InfluxDB`, `OpenTSDB`, `MySQL`, `Postgres` and `Cloudwatch`. + +> Alerting support for Elasticsearch is only available in Grafana v5.2 and above. ### Clustering @@ -152,6 +154,8 @@ filters = alerting.scheduler:debug \ tsdb.prometheus:debug \ tsdb.opentsdb:debug \ tsdb.influxdb:debug \ + tsdb.elasticsearch:debug \ + tsdb.elasticsearch.client:debug \ ``` If you want to log raw query sent to your TSDB and raw response in log you also have to set grafana.ini option `app_mode` to From 0c269d64d02c80c0d1a58b2cc12be186dd16eddc Mon Sep 17 00:00:00 2001 From: Patrick O'Carroll Date: Fri, 1 Jun 2018 14:36:40 +0200 Subject: [PATCH 85/87] Alert panel filters (#11712) alert list panel: filter alerts by name, dashboard, folder, tags --- docs/sources/http_api/alerting.md | 11 +++- pkg/api/alerting.go | 64 +++++++++++++++++-- pkg/api/alerting_test.go | 55 ++++++++++++++++ pkg/models/alert.go | 13 ++-- pkg/services/sqlstore/alert.go | 12 +++- pkg/services/sqlstore/alert_test.go | 15 +++-- .../dashboard/folder_picker/folder_picker.ts | 12 +++- .../app/plugins/panel/alertlist/editor.html | 24 +++++++ public/app/plugins/panel/alertlist/module.ts | 29 +++++++++ 9 files changed, 210 insertions(+), 25 deletions(-) diff --git a/docs/sources/http_api/alerting.md b/docs/sources/http_api/alerting.md index 4d52105cf3c..e4fe0dad3ff 100644 --- a/docs/sources/http_api/alerting.md +++ b/docs/sources/http_api/alerting.md @@ -35,10 +35,15 @@ Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk `/api/alerts?dashboardId=1` - - **dashboardId** – Return alerts for a specified dashboard. - - **panelId** – Return alerts for a specified panel on a dashboard. - - **limit** - Limit response to x number of alerts. + - **dashboardId** – Limit response to alerts in specified dashboard(s). You can specify multiple dashboards, e.g. dashboardId=23&dashboardId=35. + - **panelId** – Limit response to alert for a specified panel on a dashboard. + - **query** - Limit response to alerts having a name like this value. - **state** - Return alerts with one or more of the following alert states: `ALL`,`no_data`, `paused`, `alerting`, `ok`, `pending`. To specify multiple states use the following format: `?state=paused&state=alerting` + - **limit** - Limit response to *X* number of alerts. + - **folderId** – Limit response to alerts of dashboards in specified folder(s). You can specify multiple folders, e.g. folderId=23&folderId=35. + - **dashboardQuery** - Limit response to alerts having a dashboard name like this value. + - **dashboardTag** - Limit response to alerts of dashboards with specified tags. To do an "AND" filtering with multiple tags, specify the tags parameter multiple times e.g. dashboardTag=tag1&dashboardTag=tag2. + **Example Response**: diff --git a/pkg/api/alerting.go b/pkg/api/alerting.go index a9a3773ceb1..961fc11b2dc 100644 --- a/pkg/api/alerting.go +++ b/pkg/api/alerting.go @@ -2,12 +2,14 @@ package api import ( "fmt" + "strconv" "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/bus" m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/alerting" "github.com/grafana/grafana/pkg/services/guardian" + "github.com/grafana/grafana/pkg/services/search" ) func ValidateOrgAlert(c *m.ReqContext) { @@ -46,12 +48,64 @@ func GetAlertStatesForDashboard(c *m.ReqContext) Response { // GET /api/alerts func GetAlerts(c *m.ReqContext) Response { + dashboardQuery := c.Query("dashboardQuery") + dashboardTags := c.QueryStrings("dashboardTag") + stringDashboardIDs := c.QueryStrings("dashboardId") + stringFolderIDs := c.QueryStrings("folderId") + + dashboardIDs := make([]int64, 0) + for _, id := range stringDashboardIDs { + dashboardID, err := strconv.ParseInt(id, 10, 64) + if err == nil { + dashboardIDs = append(dashboardIDs, dashboardID) + } + } + + if dashboardQuery != "" || len(dashboardTags) > 0 || len(stringFolderIDs) > 0 { + folderIDs := make([]int64, 0) + for _, id := range stringFolderIDs { + folderID, err := strconv.ParseInt(id, 10, 64) + if err == nil { + folderIDs = append(folderIDs, folderID) + } + } + + searchQuery := search.Query{ + Title: dashboardQuery, + Tags: dashboardTags, + SignedInUser: c.SignedInUser, + Limit: 1000, + OrgId: c.OrgId, + DashboardIds: dashboardIDs, + Type: string(search.DashHitDB), + FolderIds: folderIDs, + Permission: m.PERMISSION_EDIT, + } + + err := bus.Dispatch(&searchQuery) + if err != nil { + return Error(500, "List alerts failed", err) + } + + for _, d := range searchQuery.Result { + if d.Type == search.DashHitDB && d.Id > 0 { + dashboardIDs = append(dashboardIDs, d.Id) + } + } + + // if we didn't find any dashboards, return empty result + if len(dashboardIDs) == 0 { + return JSON(200, []*m.AlertListItemDTO{}) + } + } + query := m.GetAlertsQuery{ - OrgId: c.OrgId, - DashboardId: c.QueryInt64("dashboardId"), - PanelId: c.QueryInt64("panelId"), - Limit: c.QueryInt64("limit"), - User: c.SignedInUser, + OrgId: c.OrgId, + DashboardIDs: dashboardIDs, + PanelId: c.QueryInt64("panelId"), + Limit: c.QueryInt64("limit"), + User: c.SignedInUser, + Query: c.Query("query"), } states := c.QueryStrings("state") diff --git a/pkg/api/alerting_test.go b/pkg/api/alerting_test.go index 9302ef7beca..abfdfb66322 100644 --- a/pkg/api/alerting_test.go +++ b/pkg/api/alerting_test.go @@ -6,6 +6,7 @@ import ( "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/bus" m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/search" . "github.com/smartystreets/goconvey/convey" ) @@ -64,6 +65,60 @@ func TestAlertingApiEndpoint(t *testing.T) { }) }) }) + + loggedInUserScenarioWithRole("When calling GET on", "GET", "/api/alerts?dashboardId=1", "/api/alerts", m.ROLE_EDITOR, func(sc *scenarioContext) { + var searchQuery *search.Query + bus.AddHandler("test", func(query *search.Query) error { + searchQuery = query + return nil + }) + + var getAlertsQuery *m.GetAlertsQuery + bus.AddHandler("test", func(query *m.GetAlertsQuery) error { + getAlertsQuery = query + return nil + }) + + sc.handlerFunc = GetAlerts + sc.fakeReqWithParams("GET", sc.url, map[string]string{}).exec() + + So(searchQuery, ShouldBeNil) + So(getAlertsQuery, ShouldNotBeNil) + }) + + loggedInUserScenarioWithRole("When calling GET on", "GET", "/api/alerts?dashboardId=1&dashboardId=2&folderId=3&dashboardTag=abc&dashboardQuery=dbQuery&limit=5&query=alertQuery", "/api/alerts", m.ROLE_EDITOR, func(sc *scenarioContext) { + var searchQuery *search.Query + bus.AddHandler("test", func(query *search.Query) error { + searchQuery = query + query.Result = search.HitList{ + &search.Hit{Id: 1}, + &search.Hit{Id: 2}, + } + return nil + }) + + var getAlertsQuery *m.GetAlertsQuery + bus.AddHandler("test", func(query *m.GetAlertsQuery) error { + getAlertsQuery = query + return nil + }) + + sc.handlerFunc = GetAlerts + sc.fakeReqWithParams("GET", sc.url, map[string]string{}).exec() + + So(searchQuery, ShouldNotBeNil) + So(searchQuery.DashboardIds[0], ShouldEqual, 1) + So(searchQuery.DashboardIds[1], ShouldEqual, 2) + So(searchQuery.FolderIds[0], ShouldEqual, 3) + So(searchQuery.Tags[0], ShouldEqual, "abc") + So(searchQuery.Title, ShouldEqual, "dbQuery") + + So(getAlertsQuery, ShouldNotBeNil) + So(getAlertsQuery.DashboardIDs[0], ShouldEqual, 1) + So(getAlertsQuery.DashboardIDs[1], ShouldEqual, 2) + So(getAlertsQuery.Limit, ShouldEqual, 5) + So(getAlertsQuery.Query, ShouldEqual, "alertQuery") + }) }) } diff --git a/pkg/models/alert.go b/pkg/models/alert.go index b72d87e94b2..fba2aa63df9 100644 --- a/pkg/models/alert.go +++ b/pkg/models/alert.go @@ -161,12 +161,13 @@ type SetAlertStateCommand struct { //Queries type GetAlertsQuery struct { - OrgId int64 - State []string - DashboardId int64 - PanelId int64 - Limit int64 - User *SignedInUser + OrgId int64 + State []string + DashboardIDs []int64 + PanelId int64 + Limit int64 + Query string + User *SignedInUser Result []*AlertListItemDTO } diff --git a/pkg/services/sqlstore/alert.go b/pkg/services/sqlstore/alert.go index b0ca50eb67d..58ec7e2857a 100644 --- a/pkg/services/sqlstore/alert.go +++ b/pkg/services/sqlstore/alert.go @@ -82,8 +82,16 @@ func HandleAlertsQuery(query *m.GetAlertsQuery) error { builder.Write(`WHERE alert.org_id = ?`, query.OrgId) - if query.DashboardId != 0 { - builder.Write(` AND alert.dashboard_id = ?`, query.DashboardId) + if len(strings.TrimSpace(query.Query)) > 0 { + builder.Write(" AND alert.name "+dialect.LikeStr()+" ?", "%"+query.Query+"%") + } + + if len(query.DashboardIDs) > 0 { + builder.sql.WriteString(` AND alert.dashboard_id IN (?` + strings.Repeat(",?", len(query.DashboardIDs)-1) + `) `) + + for _, dbID := range query.DashboardIDs { + builder.AddParams(dbID) + } } if query.PanelId != 0 { diff --git a/pkg/services/sqlstore/alert_test.go b/pkg/services/sqlstore/alert_test.go index 296d16c2f45..be48c7b2f52 100644 --- a/pkg/services/sqlstore/alert_test.go +++ b/pkg/services/sqlstore/alert_test.go @@ -3,10 +3,11 @@ package sqlstore import ( "testing" + "time" + "github.com/grafana/grafana/pkg/components/simplejson" m "github.com/grafana/grafana/pkg/models" . "github.com/smartystreets/goconvey/convey" - "time" ) func mockTimeNow() { @@ -99,7 +100,7 @@ func TestAlertingDataAccess(t *testing.T) { }) Convey("Can read properties", func() { - alertQuery := m.GetAlertsQuery{DashboardId: testDash.Id, PanelId: 1, OrgId: 1, User: &m.SignedInUser{OrgRole: m.ROLE_ADMIN}} + alertQuery := m.GetAlertsQuery{DashboardIDs: []int64{testDash.Id}, PanelId: 1, OrgId: 1, User: &m.SignedInUser{OrgRole: m.ROLE_ADMIN}} err2 := HandleAlertsQuery(&alertQuery) alert := alertQuery.Result[0] @@ -109,7 +110,7 @@ func TestAlertingDataAccess(t *testing.T) { }) Convey("Viewer cannot read alerts", func() { - alertQuery := m.GetAlertsQuery{DashboardId: testDash.Id, PanelId: 1, OrgId: 1, User: &m.SignedInUser{OrgRole: m.ROLE_VIEWER}} + alertQuery := m.GetAlertsQuery{DashboardIDs: []int64{testDash.Id}, PanelId: 1, OrgId: 1, User: &m.SignedInUser{OrgRole: m.ROLE_VIEWER}} err2 := HandleAlertsQuery(&alertQuery) So(err2, ShouldBeNil) @@ -134,7 +135,7 @@ func TestAlertingDataAccess(t *testing.T) { }) Convey("Alerts should be updated", func() { - query := m.GetAlertsQuery{DashboardId: testDash.Id, OrgId: 1, User: &m.SignedInUser{OrgRole: m.ROLE_ADMIN}} + query := m.GetAlertsQuery{DashboardIDs: []int64{testDash.Id}, OrgId: 1, User: &m.SignedInUser{OrgRole: m.ROLE_ADMIN}} err2 := HandleAlertsQuery(&query) So(err2, ShouldBeNil) @@ -183,7 +184,7 @@ func TestAlertingDataAccess(t *testing.T) { Convey("Should save 3 dashboards", func() { So(err, ShouldBeNil) - queryForDashboard := m.GetAlertsQuery{DashboardId: testDash.Id, OrgId: 1, User: &m.SignedInUser{OrgRole: m.ROLE_ADMIN}} + queryForDashboard := m.GetAlertsQuery{DashboardIDs: []int64{testDash.Id}, OrgId: 1, User: &m.SignedInUser{OrgRole: m.ROLE_ADMIN}} err2 := HandleAlertsQuery(&queryForDashboard) So(err2, ShouldBeNil) @@ -197,7 +198,7 @@ func TestAlertingDataAccess(t *testing.T) { err = SaveAlerts(&cmd) Convey("should delete the missing alert", func() { - query := m.GetAlertsQuery{DashboardId: testDash.Id, OrgId: 1, User: &m.SignedInUser{OrgRole: m.ROLE_ADMIN}} + query := m.GetAlertsQuery{DashboardIDs: []int64{testDash.Id}, OrgId: 1, User: &m.SignedInUser{OrgRole: m.ROLE_ADMIN}} err2 := HandleAlertsQuery(&query) So(err2, ShouldBeNil) So(len(query.Result), ShouldEqual, 2) @@ -232,7 +233,7 @@ func TestAlertingDataAccess(t *testing.T) { So(err, ShouldBeNil) Convey("Alerts should be removed", func() { - query := m.GetAlertsQuery{DashboardId: testDash.Id, OrgId: 1, User: &m.SignedInUser{OrgRole: m.ROLE_ADMIN}} + query := m.GetAlertsQuery{DashboardIDs: []int64{testDash.Id}, OrgId: 1, User: &m.SignedInUser{OrgRole: m.ROLE_ADMIN}} err2 := HandleAlertsQuery(&query) So(testDash.Id, ShouldEqual, 1) diff --git a/public/app/features/dashboard/folder_picker/folder_picker.ts b/public/app/features/dashboard/folder_picker/folder_picker.ts index b8ae18b14d3..69a09455c4d 100644 --- a/public/app/features/dashboard/folder_picker/folder_picker.ts +++ b/public/app/features/dashboard/folder_picker/folder_picker.ts @@ -12,6 +12,7 @@ export class FolderPickerCtrl { enterFolderCreation: any; exitFolderCreation: any; enableCreateNew: boolean; + enableReset: boolean; rootName = 'General'; folder: any; createNewFolder: boolean; @@ -58,6 +59,10 @@ export class FolderPickerCtrl { result.unshift({ title: '-- New Folder --', id: -1 }); } + if (this.enableReset && query === '' && this.initialTitle !== '') { + result.unshift({ title: this.initialTitle, id: null }); + } + return _.map(result, item => { return { text: item.title, value: item.id }; }); @@ -65,7 +70,9 @@ export class FolderPickerCtrl { } onFolderChange(option) { - if (option.value === -1) { + if (!option) { + option = { value: 0, text: this.rootName }; + } else if (option.value === -1) { this.createNewFolder = true; this.enterFolderCreation(); return; @@ -134,7 +141,7 @@ export class FolderPickerCtrl { this.onFolderLoad(); }); } else { - if (this.initialTitle) { + if (this.initialTitle && this.initialFolderId === null) { this.folder = { text: this.initialTitle, value: null }; } else { this.folder = { text: this.rootName, value: 0 }; @@ -171,6 +178,7 @@ export function folderPicker() { enterFolderCreation: '&', exitFolderCreation: '&', enableCreateNew: '@', + enableReset: '@', }, }; } diff --git a/public/app/plugins/panel/alertlist/editor.html b/public/app/plugins/panel/alertlist/editor.html index 36c989dd72c..c48b70e02c0 100644 --- a/public/app/plugins/panel/alertlist/editor.html +++ b/public/app/plugins/panel/alertlist/editor.html @@ -19,6 +19,30 @@
+
+
Filter
+
+ Alert name + +
+
+ Dashboard title + +
+
+ + +
+
+ Dashboard tags + + +
+
State filter
diff --git a/public/app/plugins/panel/alertlist/module.ts b/public/app/plugins/panel/alertlist/module.ts index 35fbaead3b1..55869ce626d 100644 --- a/public/app/plugins/panel/alertlist/module.ts +++ b/public/app/plugins/panel/alertlist/module.ts @@ -21,6 +21,7 @@ class AlertListPanel extends PanelCtrl { currentAlerts: any = []; alertHistory: any = []; noAlertsMessage: string; + // Set and populate defaults panelDefaults = { show: 'current', @@ -28,6 +29,9 @@ class AlertListPanel extends PanelCtrl { stateFilter: [], onlyAlertsOnDashboard: false, sortOrder: 1, + dashboardFilter: '', + nameFilter: '', + folderId: null, }; /** @ngInject */ @@ -89,6 +93,11 @@ class AlertListPanel extends PanelCtrl { }); } + onFolderChange(folder: any) { + this.panel.folderId = folder.id; + this.refresh(); + } + getStateChanges() { var params: any = { limit: this.panel.limit, @@ -110,6 +119,7 @@ class AlertListPanel extends PanelCtrl { al.info = alertDef.getAlertAnnotationInfo(al); return al; }); + this.noAlertsMessage = this.alertHistory.length === 0 ? 'No alerts in current time range' : ''; return this.alertHistory; @@ -121,10 +131,26 @@ class AlertListPanel extends PanelCtrl { state: this.panel.stateFilter, }; + if (this.panel.nameFilter) { + params.query = this.panel.nameFilter; + } + + if (this.panel.folderId >= 0) { + params.folderId = this.panel.folderId; + } + + if (this.panel.dashboardFilter) { + params.dashboardQuery = this.panel.dashboardFilter; + } + if (this.panel.onlyAlertsOnDashboard) { params.dashboardId = this.dashboard.id; } + if (this.panel.dashboardTags) { + params.dashboardTag = this.panel.dashboardTags; + } + return this.backendSrv.get(`/api/alerts`, params).then(res => { this.currentAlerts = this.sortResult( _.map(res, al => { @@ -135,6 +161,9 @@ class AlertListPanel extends PanelCtrl { return al; }) ); + if (this.currentAlerts.length > this.panel.limit) { + this.currentAlerts = this.currentAlerts.slice(0, this.panel.limit); + } this.noAlertsMessage = this.currentAlerts.length === 0 ? 'No alerts' : ''; return this.currentAlerts; From b67872bc35c63eb6debf2ac121673442d0a3f948 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Fri, 1 Jun 2018 14:49:14 +0200 Subject: [PATCH 86/87] changelog: add notes about closing #11500, #8168, #6541 [skip ci] --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d7e46d6cf4..7ef36a8796f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # 5.2.0 (unreleased) +### New Features + +* **Alert list panel**: Updated to support filtering alerts by name, dashboard title, folder, tags [#11500](https://github.com/grafana/grafana/issues/11500), [#8168](https://github.com/grafana/grafana/issues/8168), [#6541](https://github.com/grafana/grafana/issues/6541) + ### Minor * **Dashboard**: Modified time range and variables are now not saved by default [#10748](https://github.com/grafana/grafana/issues/10748), [#8805](https://github.com/grafana/grafana/issues/8805) From f5cf92636451ef2bb80f86606e6e8b03cb28c962 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Fri, 1 Jun 2018 15:23:26 +0200 Subject: [PATCH 87/87] changelog: add notes about closing #5893 [skip ci] --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ef36a8796f..76e538a8e32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### New Features +* **Elasticsearch**: Alerting support [#5893](https://github.com/grafana/grafana/issues/5893), thx [@WPH95](https://github.com/WPH95) * **Alert list panel**: Updated to support filtering alerts by name, dashboard title, folder, tags [#11500](https://github.com/grafana/grafana/issues/11500), [#8168](https://github.com/grafana/grafana/issues/8168), [#6541](https://github.com/grafana/grafana/issues/6541) ### Minor