From 6f83a6b2fdfc6d3198c5c53be1480d08df5b13db Mon Sep 17 00:00:00 2001 From: Andreas Christou Date: Thu, 11 Sep 2025 13:31:11 +0200 Subject: [PATCH] Graphite: Backend health-check (#110518) * Add lint rules * Backend decoupling - Add standalone files - Add graphite query type - Add logger to Service - Create logger in the ProvideService method - Use a pointer for the HTTP client provider - Update logger usage everywhere - Update tracer type - Replace simplejson with json - Add dummy CallResource and CheckHealth methods - Update tests * Update ConfigEditor imports * Update types imports * Update datasource - Switch to using semver package - Update imports * Update store imports * Update helper imports and notification creation * Update context import * Update version numbers and logic * Copy array_move from core * Test updates * Add required files and update plugin.json * Update core references and packages * Remove commented code * Update wire * Lint * Fix import * Copy null type * More lint * Update snapshot * Refactor backend - Split query logic into separate file - Move utils to separate file * Add health-check logic - Support backend healthcheck if the FF is enabled * Remove query import support as unneeded * Add test * Add tests * Review * Review * Fix packages * Fix merge issues --- pkg/tsdb/graphite/graphite.go | 299 +---------------- pkg/tsdb/graphite/healthcheck.go | 101 ++++++ pkg/tsdb/graphite/healthcheck_test.go | 133 ++++++++ pkg/tsdb/graphite/query.go | 303 ++++++++++++++++++ .../{graphite_test.go => query_test.go} | 72 ++--- pkg/tsdb/graphite/utils.go | 1 + pkg/tsdb/graphite/utils_test.go | 1 + .../plugins/datasource/graphite/datasource.ts | 16 +- 8 files changed, 589 insertions(+), 337 deletions(-) create mode 100644 pkg/tsdb/graphite/healthcheck.go create mode 100644 pkg/tsdb/graphite/healthcheck_test.go create mode 100644 pkg/tsdb/graphite/query.go rename pkg/tsdb/graphite/{graphite_test.go => query_test.go} (100%) create mode 100644 pkg/tsdb/graphite/utils.go create mode 100644 pkg/tsdb/graphite/utils_test.go diff --git a/pkg/tsdb/graphite/graphite.go b/pkg/tsdb/graphite/graphite.go index d948a7bc1d5..ad37275406c 100644 --- a/pkg/tsdb/graphite/graphite.go +++ b/pkg/tsdb/graphite/graphite.go @@ -2,26 +2,14 @@ package graphite import ( "context" - "encoding/json" - "errors" "fmt" - "io" "net/http" - "net/url" - "path" - "regexp" - "strconv" - "strings" - "time" "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/backend/datasource" "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" "github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt" "github.com/grafana/grafana-plugin-sdk-go/backend/log" - "github.com/grafana/grafana-plugin-sdk-go/data" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/trace" ) @@ -93,294 +81,9 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest) return nil, err } - emptyQueries := []string{} - graphiteQueries := map[string]struct { - req *http.Request - formData url.Values - }{} - for _, query := range req.Queries { - graphiteReq, formData, emptyQuery, err := s.createGraphiteRequest(ctx, query, dsInfo) - if err != nil { - return nil, err - } - - if emptyQuery != nil { - emptyQueries = append(emptyQueries, fmt.Sprintf("Query: %v has no target", emptyQuery)) - continue - } - - graphiteQueries[query.RefID] = struct { - req *http.Request - formData url.Values - }{ - req: graphiteReq, - formData: formData, - } - } - - var result = backend.QueryDataResponse{} - if len(emptyQueries) != 0 { - s.logger.Warn("Found query models without targets", "models without targets", strings.Join(emptyQueries, "\n")) - // If no queries had a valid target, return an error; otherwise, attempt with the targets we have - if len(emptyQueries) == len(req.Queries) { - if result.Responses == nil { - result.Responses = make(map[string]backend.DataResponse) - } - // marking this downstream error as it is a user error, but arguably this is a plugin error - // since the plugin should have frontend validation that prevents us from getting into this state - missingQueryResponse := backend.ErrDataResponseWithSource(400, backend.ErrorSourceDownstream, "no query target found for the alert rule") - result.Responses["A"] = missingQueryResponse - return &result, nil - } - } - - frames := data.Frames{} - - for refId, graphiteReq := range graphiteQueries { - _, span := s.tracer.Start(ctx, "graphite query") - defer span.End() - targetStr := strings.Join(graphiteReq.formData["target"], ",") - span.SetAttributes( - attribute.String("refId", refId), - attribute.String("target", targetStr), - attribute.String("from", graphiteReq.formData["from"][0]), - attribute.String("until", graphiteReq.formData["until"][0]), - attribute.Int64("datasource_id", dsInfo.Id), - attribute.Int64("org_id", req.PluginContext.OrgID), - ) - res, err := dsInfo.HTTPClient.Do(graphiteReq.req) - if res != nil { - span.SetAttributes(attribute.Int("graphite.response.code", res.StatusCode)) - } - if err != nil { - span.RecordError(err) - span.SetStatus(codes.Error, err.Error()) - return &result, err - } - - defer func() { - err := res.Body.Close() - if err != nil { - s.logger.Warn("Failed to close response body", "error", err) - } - }() - - queryFrames, err := s.toDataFrames(res, refId) - if err != nil { - span.RecordError(err) - span.SetStatus(codes.Error, err.Error()) - return &result, err - } - - frames = append(frames, queryFrames...) - } - - result = backend.QueryDataResponse{ - Responses: make(backend.Responses), - } - - for _, f := range frames { - if resp, ok := result.Responses[f.Name]; ok { - resp.Frames = append(resp.Frames, f) - result.Responses[f.Name] = resp - } else { - result.Responses[f.Name] = backend.DataResponse{ - Frames: data.Frames{f}, - } - } - } - - return &result, nil -} - -// processQuery converts a Graphite data source query to a Graphite query target. It returns the target, -// and the model if the target is invalid -func (s *Service) processQuery(query backend.DataQuery) (string, *GraphiteQuery, error) { - queryJSON := GraphiteQuery{} - err := json.Unmarshal(query.JSON, &queryJSON) - if err != nil { - return "", &queryJSON, fmt.Errorf("failed to decode the Graphite query: %w", err) - } - s.logger.Debug("Graphite", "query", queryJSON) - currTarget := queryJSON.TargetFull - - if currTarget == "" { - currTarget = queryJSON.Target - } - - if currTarget == "" { - s.logger.Debug("Graphite", "empty query target", queryJSON) - return "", &queryJSON, nil - } - target := fixIntervalFormat(currTarget) - - return target, nil, nil -} - -func (s *Service) createRequest(ctx context.Context, dsInfo *datasourceInfo, data url.Values) (*http.Request, error) { - u, err := url.Parse(dsInfo.URL) - if err != nil { - return nil, err - } - u.Path = path.Join(u.Path, "render") - - req, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), strings.NewReader(data.Encode())) - if err != nil { - s.logger.Info("Failed to create request", "error", err) - return nil, fmt.Errorf("failed to create request: %w", err) - } - - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - return req, err -} - -func (s *Service) createGraphiteRequest(ctx context.Context, query backend.DataQuery, dsInfo *datasourceInfo) (*http.Request, url.Values, *GraphiteQuery, error) { - /* - graphite doc about from and until, with sdk we are getting absolute instead of relative time - https://graphite-api.readthedocs.io/en/latest/api.html#from-until - */ - from, until := epochMStoGraphiteTime(query.TimeRange) - formData := url.Values{ - "from": []string{from}, - "until": []string{until}, - "format": []string{"json"}, - "maxDataPoints": []string{fmt.Sprintf("%d", query.MaxDataPoints)}, - "target": []string{}, - } - - target, emptyQuery, err := s.processQuery(query) - if err != nil { - return nil, formData, nil, err - } - - if emptyQuery != nil { - s.logger.Debug("Graphite", "empty query target", emptyQuery) - return nil, formData, emptyQuery, nil - } - - formData["target"] = []string{target} - - s.logger.Debug("Graphite request", "params", formData) - - graphiteReq, err := s.createRequest(ctx, dsInfo, formData) - if err != nil { - return nil, formData, nil, err - } - - return graphiteReq, formData, emptyQuery, nil -} - -func (s *Service) parseResponse(res *http.Response) ([]TargetResponseDTO, error) { - body, err := io.ReadAll(res.Body) - if err != nil { - return nil, err - } - defer func() { - if err := res.Body.Close(); err != nil { - s.logger.Warn("Failed to close response body", "err", err) - } - }() - - if res.StatusCode/100 != 2 { - s.logger.Info("Request failed", "status", res.Status, "body", string(body)) - return nil, fmt.Errorf("request failed, status: %s", res.Status) - } - - var data []TargetResponseDTO - err = json.Unmarshal(body, &data) - if err != nil { - s.logger.Info("Failed to unmarshal graphite response", "error", err, "status", res.Status, "body", string(body)) - return nil, err - } - - return data, nil -} - -func (s *Service) toDataFrames(response *http.Response, refId string) (frames data.Frames, error error) { - responseData, err := s.parseResponse(response) - if err != nil { - return nil, err - } - - frames = data.Frames{} - for _, series := range responseData { - timeVector := make([]time.Time, 0, len(series.DataPoints)) - values := make([]*float64, 0, len(series.DataPoints)) - - for _, dataPoint := range series.DataPoints { - var timestamp, value, err = parseDataTimePoint(dataPoint) - if err != nil { - return nil, err - } - timeVector = append(timeVector, timestamp) - values = append(values, value) - } - - tags := make(map[string]string) - for name, value := range series.Tags { - if name == "name" { - value = series.Target - } - switch value := value.(type) { - case string: - tags[name] = value - case float64: - tags[name] = strconv.FormatFloat(value, 'f', -1, 64) - } - } - - frames = append(frames, data.NewFrame(refId, - data.NewField("time", nil, timeVector), - data.NewField("value", tags, values).SetConfig(&data.FieldConfig{DisplayNameFromDS: series.Target})).SetMeta( - &data.FrameMeta{Type: data.FrameTypeTimeSeriesMulti})) - - s.logger.Debug("Graphite response", "target", series.Target, "datapoints", len(series.DataPoints)) - } - return frames, nil -} - -func fixIntervalFormat(target string) string { - rMinute := regexp.MustCompile(`'(\d+)m'`) - target = rMinute.ReplaceAllStringFunc(target, func(m string) string { - return strings.ReplaceAll(m, "m", "min") - }) - rMonth := regexp.MustCompile(`'(\d+)M'`) - target = rMonth.ReplaceAllStringFunc(target, func(M string) string { - return strings.ReplaceAll(M, "M", "mon") - }) - return target -} - -func epochMStoGraphiteTime(tr backend.TimeRange) (string, string) { - return fmt.Sprintf("%d", tr.From.UTC().Unix()), fmt.Sprintf("%d", tr.To.UTC().Unix()) -} - -/** - * Graphite should always return timestamp as a number but values might be nil when data is missing - */ -func parseDataTimePoint(dataTimePoint DataTimePoint) (time.Time, *float64, error) { - if !dataTimePoint[1].Valid { - return time.Time{}, nil, errors.New("failed to parse data point timestamp") - } - - timestamp := time.Unix(int64(dataTimePoint[1].Float64), 0).UTC() - - if dataTimePoint[0].Valid { - var value = new(float64) - *value = dataTimePoint[0].Float64 - return timestamp, value, nil - } else { - return timestamp, nil, nil - } + return s.RunQuery(ctx, req, dsInfo) } func (s *Service) CallResource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error { return nil } - -func (s *Service) CheckHealth(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) { - return &backend.CheckHealthResult{ - Status: backend.HealthStatusOk, - Message: "Successfully connected to Graphite.", - }, nil -} diff --git a/pkg/tsdb/graphite/healthcheck.go b/pkg/tsdb/graphite/healthcheck.go new file mode 100644 index 00000000000..e6c3f005095 --- /dev/null +++ b/pkg/tsdb/graphite/healthcheck.go @@ -0,0 +1,101 @@ +package graphite + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana-plugin-sdk-go/backend/tracing" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" +) + +func (s *Service) CheckHealth(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) { + dsInfo, err := s.getDSInfo(ctx, req.PluginContext) + if err != nil { + s.logger.Error("failed to get data source info", "error", err) + return &backend.CheckHealthResult{ + Status: backend.HealthStatusError, + Message: "Graphite health check failed. See details below", + JSONDetails: []byte( + fmt.Sprintf(`{"verboseMessage": %s }`, strconv.Quote(err.Error())), + ), + }, nil + } + + healthCheckQuery := backend.DataQuery{ + Interval: 10 * time.Millisecond, + RefID: "graphite-healthcheck", + TimeRange: backend.TimeRange{ + From: time.Now().Add(-time.Hour), + To: time.Now(), + }, + MaxDataPoints: 100, + JSON: []byte(`{"target": "constantLine(100)"}`), + } + + _, span := tracing.DefaultTracer().Start(ctx, "graphite healthcheck") + defer span.End() + graphiteReq, formData, _, err := s.createGraphiteRequest(ctx, healthCheckQuery, dsInfo) + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + return &backend.CheckHealthResult{ + Status: backend.HealthStatusError, + Message: "Graphite health check failed. See details below", + JSONDetails: []byte( + fmt.Sprintf(`{"verboseMessage": %s }`, strconv.Quote(err.Error())), + ), + }, nil + } + targetStr := strings.Join(formData["target"], ",") + span.SetAttributes( + attribute.String("target", targetStr), + attribute.String("from", formData["from"][0]), + attribute.String("until", formData["until"][0]), + attribute.Int64("datasource_id", dsInfo.Id), + ) + res, err := dsInfo.HTTPClient.Do(graphiteReq) + if res != nil { + span.SetAttributes(attribute.Int("graphite.response.code", res.StatusCode)) + } + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + return &backend.CheckHealthResult{ + Status: backend.HealthStatusError, + Message: "Graphite health check failed. See details below", + JSONDetails: []byte( + fmt.Sprintf(`{"verboseMessage": %s }`, strconv.Quote(err.Error())), + ), + }, nil + } + + defer func() { + err := res.Body.Close() + if err != nil { + s.logger.Warn("Failed to close response body", "error", err) + } + }() + + _, err = s.toDataFrames(res, healthCheckQuery.RefID) + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + return &backend.CheckHealthResult{ + Status: backend.HealthStatusError, + Message: "Graphite health check failed. See details below", + JSONDetails: []byte( + fmt.Sprintf(`{"verboseMessage": %s }`, strconv.Quote(err.Error())), + ), + }, nil + } + + return &backend.CheckHealthResult{ + Status: backend.HealthStatusOk, + Message: "Successfully connected to Graphite", + }, nil +} diff --git a/pkg/tsdb/graphite/healthcheck_test.go b/pkg/tsdb/graphite/healthcheck_test.go new file mode 100644 index 00000000000..8c19d809ea0 --- /dev/null +++ b/pkg/tsdb/graphite/healthcheck_test.go @@ -0,0 +1,133 @@ +package graphite + +import ( + "context" + "io" + "net/http" + "strings" + "testing" + "time" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana-plugin-sdk-go/backend/datasource" + "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" + + "github.com/grafana/grafana-plugin-sdk-go/backend/tracing" + "github.com/stretchr/testify/assert" +) + +type healthCheckProvider[T http.RoundTripper] struct { + httpclient.Provider + RoundTripper *T +} + +type healthCheckSuccessRoundTripper struct { +} +type healthCheckFailRoundTripper struct { +} + +func (rt *healthCheckSuccessRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + return &http.Response{ + Status: "200", + StatusCode: 200, + Header: nil, + Body: io.NopCloser(strings.NewReader(`[{"target": "100.0", "tags": {"name": "100.0"}, "datapoints": [[100.0, 10000], [100.0, 10001], [100.0, 10002]]}]`)), + ContentLength: 0, + Request: req, + }, nil +} + +func (rt *healthCheckFailRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + return &http.Response{ + Status: "400", + StatusCode: 400, + Header: nil, + Body: nil, + ContentLength: 0, + Request: req, + }, nil +} + +func (provider *healthCheckProvider[T]) New(opts ...httpclient.Options) (*http.Client, error) { + client := &http.Client{} + provider.RoundTripper = new(T) + client.Transport = *provider.RoundTripper + return client, nil +} + +func (provider *healthCheckProvider[T]) GetTransport(opts ...httpclient.Options) (http.RoundTripper, error) { + return *new(T), nil +} + +func getMockProvider[T http.RoundTripper]() *httpclient.Provider { + p := &healthCheckProvider[T]{ + RoundTripper: new(T), + } + rtFunction := func(o httpclient.Options, next http.RoundTripper) http.RoundTripper { + return *p.RoundTripper + } + fn := httpclient.MiddlewareFunc(rtFunction) + mid := httpclient.NamedMiddlewareFunc("mock", fn) + return httpclient.NewProvider(httpclient.ProviderOptions{Middlewares: []httpclient.Middleware{mid}}) +} + +func Test_CheckHealth(t *testing.T) { + t.Run("should return a successful health check", func(t *testing.T) { + httpProvider := getMockProvider[*healthCheckSuccessRoundTripper]() + s := &Service{ + im: datasource.NewInstanceManager(newInstanceSettings(httpProvider)), + tracer: tracing.DefaultTracer(), + logger: backend.NewLoggerWith("logger", "graphite test"), + } + + req := &backend.CheckHealthRequest{ + PluginContext: getPluginContext(), + Headers: nil, + } + + res, err := s.CheckHealth(context.Background(), req) + assert.NoError(t, err) + assert.Equal(t, backend.HealthStatusOk, res.Status) + }) + + t.Run("should return an error for an unsuccessful health check", func(t *testing.T) { + httpProvider := getMockProvider[*healthCheckFailRoundTripper]() + s := &Service{ + im: datasource.NewInstanceManager(newInstanceSettings(httpProvider)), + tracer: tracing.DefaultTracer(), + logger: backend.NewLoggerWith("logger", "graphite test"), + } + + req := &backend.CheckHealthRequest{ + PluginContext: getPluginContext(), + Headers: nil, + } + + res, err := s.CheckHealth(context.Background(), req) + assert.NoError(t, err) + assert.Equal(t, backend.HealthStatusError, res.Status) + assert.Equal(t, "Graphite health check failed. See details below", res.Message) + assert.Equal(t, []byte("{\"verboseMessage\": \"request failed, status: 400\" }"), res.JSONDetails) + }) +} + +func getPluginContext() backend.PluginContext { + return backend.PluginContext{ + OrgID: 0, + PluginID: "graphite", + User: nil, + AppInstanceSettings: nil, + DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{ + ID: 0, + UID: "", + Type: "graphite", + Name: "test-graphite", + URL: "http://graphite", + User: "", + Database: "", + JSONData: []byte("{}"), + DecryptedSecureJSONData: map[string]string{}, + Updated: time.Time{}, + }, + } +} diff --git a/pkg/tsdb/graphite/query.go b/pkg/tsdb/graphite/query.go new file mode 100644 index 00000000000..4889328aeee --- /dev/null +++ b/pkg/tsdb/graphite/query.go @@ -0,0 +1,303 @@ +package graphite + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "path" + "regexp" + "strconv" + "strings" + "time" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana-plugin-sdk-go/backend/tracing" + "github.com/grafana/grafana-plugin-sdk-go/data" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" +) + +func (s *Service) RunQuery(ctx context.Context, req *backend.QueryDataRequest, dsInfo *datasourceInfo) (*backend.QueryDataResponse, error) { + emptyQueries := []string{} + graphiteQueries := map[string]struct { + req *http.Request + formData url.Values + }{} + for _, query := range req.Queries { + graphiteReq, formData, emptyQuery, err := s.createGraphiteRequest(ctx, query, dsInfo) + if err != nil { + return nil, err + } + + if emptyQuery != nil { + emptyQueries = append(emptyQueries, fmt.Sprintf("Query: %v has no target", emptyQuery)) + continue + } + + graphiteQueries[query.RefID] = struct { + req *http.Request + formData url.Values + }{ + req: graphiteReq, + formData: formData, + } + } + + var result = backend.QueryDataResponse{} + if len(emptyQueries) != 0 { + s.logger.Warn("Found query models without targets", "models without targets", strings.Join(emptyQueries, "\n")) + // If no queries had a valid target, return an error; otherwise, attempt with the targets we have + if len(emptyQueries) == len(req.Queries) { + if result.Responses == nil { + result.Responses = make(map[string]backend.DataResponse) + } + // marking this downstream error as it is a user error, but arguably this is a plugin error + // since the plugin should have frontend validation that prevents us from getting into this state + missingQueryResponse := backend.ErrDataResponseWithSource(400, backend.ErrorSourceDownstream, "no query target found for the alert rule") + result.Responses["A"] = missingQueryResponse + return &result, nil + } + } + + frames := data.Frames{} + + for refId, graphiteReq := range graphiteQueries { + _, span := tracing.DefaultTracer().Start(ctx, "graphite query") + defer span.End() + targetStr := strings.Join(graphiteReq.formData["target"], ",") + span.SetAttributes( + attribute.String("refId", refId), + attribute.String("target", targetStr), + attribute.String("from", graphiteReq.formData["from"][0]), + attribute.String("until", graphiteReq.formData["until"][0]), + attribute.Int64("datasource_id", dsInfo.Id), + attribute.Int64("org_id", req.PluginContext.OrgID), + ) + res, err := dsInfo.HTTPClient.Do(graphiteReq.req) + if res != nil { + span.SetAttributes(attribute.Int("graphite.response.code", res.StatusCode)) + } + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + return &result, err + } + + defer func() { + err := res.Body.Close() + if err != nil { + s.logger.Warn("Failed to close response body", "error", err) + } + }() + + queryFrames, err := s.toDataFrames(res, refId) + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + return &result, err + } + + frames = append(frames, queryFrames...) + } + + result = backend.QueryDataResponse{ + Responses: make(backend.Responses), + } + + for _, f := range frames { + if resp, ok := result.Responses[f.Name]; ok { + resp.Frames = append(resp.Frames, f) + result.Responses[f.Name] = resp + } else { + result.Responses[f.Name] = backend.DataResponse{ + Frames: data.Frames{f}, + } + } + } + + return &result, nil +} + +// processQuery converts a Graphite data source query to a Graphite query target. It returns the target, +// and the model if the target is invalid +func (s *Service) processQuery(query backend.DataQuery) (string, *GraphiteQuery, error) { + queryJSON := GraphiteQuery{} + err := json.Unmarshal(query.JSON, &queryJSON) + if err != nil { + return "", &queryJSON, fmt.Errorf("failed to decode the Graphite query: %w", err) + } + s.logger.Debug("Graphite", "query", queryJSON) + currTarget := queryJSON.TargetFull + + if currTarget == "" { + currTarget = queryJSON.Target + } + if currTarget == "" { + s.logger.Debug("Graphite", "empty query target", queryJSON) + return "", &queryJSON, nil + } + target := fixIntervalFormat(currTarget) + + return target, nil, nil +} + +func (s *Service) createGraphiteRequest(ctx context.Context, query backend.DataQuery, dsInfo *datasourceInfo) (*http.Request, url.Values, *GraphiteQuery, error) { + /* + graphite doc about from and until, with sdk we are getting absolute instead of relative time + https://graphite-api.readthedocs.io/en/latest/api.html#from-until + */ + from, until := epochMStoGraphiteTime(query.TimeRange) + formData := url.Values{ + "from": []string{from}, + "until": []string{until}, + "format": []string{"json"}, + "maxDataPoints": []string{fmt.Sprintf("%d", query.MaxDataPoints)}, + "target": []string{}, + } + + target, emptyQuery, err := s.processQuery(query) + if err != nil { + return nil, formData, nil, err + } + + if emptyQuery != nil { + s.logger.Debug("Graphite", "empty query target", emptyQuery) + return nil, formData, emptyQuery, nil + } + + formData["target"] = []string{target} + + s.logger.Debug("Graphite request", "params", formData) + + graphiteReq, err := s.createRequest(ctx, dsInfo, formData) + if err != nil { + return nil, formData, nil, err + } + + return graphiteReq, formData, emptyQuery, nil +} + +func (s *Service) createRequest(ctx context.Context, dsInfo *datasourceInfo, data url.Values) (*http.Request, error) { + u, err := url.Parse(dsInfo.URL) + if err != nil { + return nil, err + } + u.Path = path.Join(u.Path, "render") + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), strings.NewReader(data.Encode())) + if err != nil { + s.logger.Info("Failed to create request", "error", err) + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + return req, err +} + +func (s *Service) toDataFrames(response *http.Response, refId string) (frames data.Frames, error error) { + responseData, err := s.parseResponse(response) + if err != nil { + return nil, err + } + + frames = data.Frames{} + for _, series := range responseData { + timeVector := make([]time.Time, 0, len(series.DataPoints)) + values := make([]*float64, 0, len(series.DataPoints)) + + for _, dataPoint := range series.DataPoints { + var timestamp, value, err = parseDataTimePoint(dataPoint) + if err != nil { + return nil, err + } + timeVector = append(timeVector, timestamp) + values = append(values, value) + } + + tags := make(map[string]string) + for name, value := range series.Tags { + if name == "name" { + value = series.Target + } + switch value := value.(type) { + case string: + tags[name] = value + case float64: + tags[name] = strconv.FormatFloat(value, 'f', -1, 64) + } + } + + frames = append(frames, data.NewFrame(refId, + data.NewField("time", nil, timeVector), + data.NewField("value", tags, values).SetConfig(&data.FieldConfig{DisplayNameFromDS: series.Target})).SetMeta( + &data.FrameMeta{Type: data.FrameTypeTimeSeriesMulti})) + + s.logger.Debug("Graphite response", "target", series.Target, "datapoints", len(series.DataPoints)) + } + return frames, nil +} + +func (s *Service) parseResponse(res *http.Response) ([]TargetResponseDTO, error) { + body, err := io.ReadAll(res.Body) + if err != nil { + return nil, err + } + defer func() { + if err := res.Body.Close(); err != nil { + s.logger.Warn("Failed to close response body", "err", err) + } + }() + + if res.StatusCode/100 != 2 { + s.logger.Info("Request failed", "status", res.Status, "body", string(body)) + return nil, fmt.Errorf("request failed, status: %s", res.Status) + } + + var data []TargetResponseDTO + err = json.Unmarshal(body, &data) + if err != nil { + s.logger.Info("Failed to unmarshal graphite response", "error", err, "status", res.Status, "body", string(body)) + return nil, err + } + + return data, nil +} + +func fixIntervalFormat(target string) string { + rMinute := regexp.MustCompile(`'(\d+)m'`) + target = rMinute.ReplaceAllStringFunc(target, func(m string) string { + return strings.ReplaceAll(m, "m", "min") + }) + rMonth := regexp.MustCompile(`'(\d+)M'`) + target = rMonth.ReplaceAllStringFunc(target, func(M string) string { + return strings.ReplaceAll(M, "M", "mon") + }) + return target +} + +func epochMStoGraphiteTime(tr backend.TimeRange) (string, string) { + return fmt.Sprintf("%d", tr.From.UTC().Unix()), fmt.Sprintf("%d", tr.To.UTC().Unix()) +} + +/** + * Graphite should always return timestamp as a number but values might be nil when data is missing + */ +func parseDataTimePoint(dataTimePoint DataTimePoint) (time.Time, *float64, error) { + if !dataTimePoint[1].Valid { + return time.Time{}, nil, errors.New("failed to parse data point timestamp") + } + + timestamp := time.Unix(int64(dataTimePoint[1].Float64), 0).UTC() + + if dataTimePoint[0].Valid { + var value = new(float64) + *value = dataTimePoint[0].Float64 + return timestamp, value, nil + } else { + return timestamp, nil, nil + } +} diff --git a/pkg/tsdb/graphite/graphite_test.go b/pkg/tsdb/graphite/query_test.go similarity index 100% rename from pkg/tsdb/graphite/graphite_test.go rename to pkg/tsdb/graphite/query_test.go index f6054fea47b..b49479a7c8a 100644 --- a/pkg/tsdb/graphite/graphite_test.go +++ b/pkg/tsdb/graphite/query_test.go @@ -19,42 +19,6 @@ import ( "go.opentelemetry.io/otel/trace/noop" ) -func TestFixIntervalFormat(t *testing.T) { - testCases := []struct { - name string - target string - expected string - }{ - { - name: "should transform 1m to graphite unit (1min) when used as interval string", - target: "aliasByNode(hitcount(averageSeries(app.grafana.*.dashboards.views.count), '1m'), 4)", - expected: "aliasByNode(hitcount(averageSeries(app.grafana.*.dashboards.views.count), '1min'), 4)", - }, - { - name: "should transform 1M to graphite unit (1mon) when used as interval string", - target: "aliasByNode(hitcount(averageSeries(app.grafana.*.dashboards.views.count), '1M'), 4)", - expected: "aliasByNode(hitcount(averageSeries(app.grafana.*.dashboards.views.count), '1mon'), 4)", - }, - { - name: "should not transform 1m when not used as interval string", - target: "app.grafana.*.dashboards.views.1m.count", - expected: "app.grafana.*.dashboards.views.1m.count", - }, - { - name: "should not transform 1M when not used as interval string", - target: "app.grafana.*.dashboards.views.1M.count", - expected: "app.grafana.*.dashboards.views.1M.count", - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - tr := fixIntervalFormat(tc.target) - assert.Equal(t, tc.expected, tr) - }) - } -} - func TestProcessQuery(t *testing.T) { service := &Service{ logger: backend.Logger, @@ -242,3 +206,39 @@ func TestConvertResponses(t *testing.T) { } }) } + +func TestFixIntervalFormat(t *testing.T) { + testCases := []struct { + name string + target string + expected string + }{ + { + name: "should transform 1m to graphite unit (1min) when used as interval string", + target: "aliasByNode(hitcount(averageSeries(app.grafana.*.dashboards.views.count), '1m'), 4)", + expected: "aliasByNode(hitcount(averageSeries(app.grafana.*.dashboards.views.count), '1min'), 4)", + }, + { + name: "should transform 1M to graphite unit (1mon) when used as interval string", + target: "aliasByNode(hitcount(averageSeries(app.grafana.*.dashboards.views.count), '1M'), 4)", + expected: "aliasByNode(hitcount(averageSeries(app.grafana.*.dashboards.views.count), '1mon'), 4)", + }, + { + name: "should not transform 1m when not used as interval string", + target: "app.grafana.*.dashboards.views.1m.count", + expected: "app.grafana.*.dashboards.views.1m.count", + }, + { + name: "should not transform 1M when not used as interval string", + target: "app.grafana.*.dashboards.views.1M.count", + expected: "app.grafana.*.dashboards.views.1M.count", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tr := fixIntervalFormat(tc.target) + assert.Equal(t, tc.expected, tr) + }) + } +} diff --git a/pkg/tsdb/graphite/utils.go b/pkg/tsdb/graphite/utils.go new file mode 100644 index 00000000000..7a3bba9035b --- /dev/null +++ b/pkg/tsdb/graphite/utils.go @@ -0,0 +1 @@ +package graphite diff --git a/pkg/tsdb/graphite/utils_test.go b/pkg/tsdb/graphite/utils_test.go new file mode 100644 index 00000000000..7a3bba9035b --- /dev/null +++ b/pkg/tsdb/graphite/utils_test.go @@ -0,0 +1 @@ +package graphite diff --git a/public/app/plugins/datasource/graphite/datasource.ts b/public/app/plugins/datasource/graphite/datasource.ts index ff268763758..75bc5c166ad 100644 --- a/public/app/plugins/datasource/graphite/datasource.ts +++ b/public/app/plugins/datasource/graphite/datasource.ts @@ -11,7 +11,6 @@ import { DataFrame, DataQueryRequest, DataQueryResponse, - DataSourceApi, DataSourceWithQueryExportSupport, dateMath, DateTime, @@ -23,7 +22,15 @@ import { TimeRange, toDataFrame, } from '@grafana/data'; -import { BackendSrvRequest, FetchResponse, getBackendSrv, getTemplateSrv, TemplateSrv } from '@grafana/runtime'; +import { + BackendSrvRequest, + config, + DataSourceWithBackend, + FetchResponse, + getBackendSrv, + getTemplateSrv, + TemplateSrv, +} from '@grafana/runtime'; import { TimeZone } from '@grafana/schema'; import { AnnotationEditor } from './components/AnnotationsEditor'; @@ -67,7 +74,7 @@ function convertGlobToRegEx(text: string): string { } export class GraphiteDatasource - extends DataSourceApi + extends DataSourceWithBackend implements DataSourceWithQueryExportSupport { basicAuth: string; @@ -953,6 +960,9 @@ export class GraphiteDatasource } testDatasource() { + if (config.featureToggles.graphiteBackendMode) { + return super.testDatasource(); + } const query: DataQueryRequest = { app: 'graphite', interval: '10ms',