OpenTSDB: Support v2.4 (#100673)

* Add version 2.4 to frontend * Update settings and types - Set all properties on backend for consistency * Update query logic to parse new and old format - Minor naming updates - Extract logic for initial frame creation - When parsing old api responses, ensure data is in ascending order - Update tests * Update docs and provisioning file * Fix lint * Update docs/sources/datasources/opentsdb/_index.md Co-authored-by: Larissa Wandzura <126723338+lwandz13@users.noreply.github.com> * Update docs/sources/datasources/opentsdb/_index.md Co-authored-by: Larissa Wandzura <126723338+lwandz13@users.noreply.github.com> * Review nit --------- Co-authored-by: Larissa Wandzura <126723338+lwandz13@users.noreply.github.com>
2025-02-25 16:59:58 +02:00
parent 6eb335a8ce
commit 1302ee48b9
6 changed files with 263 additions and 46 deletions
@@ -8,6 +8,8 @@ import (
 	"net/http"
 	"net/url"
 	"path"
+	"sort"
+	"strconv"
 	"strings"
 	"time"

@@ -35,12 +37,21 @@ func ProvideService(httpClientProvider httpclient.Provider) *Service {
 }

 type datasourceInfo struct {
-	HTTPClient *http.Client
-	URL        string
+	HTTPClient     *http.Client
+	URL            string
+	TSDBVersion    float32
+	TSDBResolution int32
+	LookupLimit    int32
 }

 type DsAccess string

+type JSONData struct {
+	TSDBVersion    float32 `json:"tsdbVersion"`
+	TSDBResolution int32   `json:"tsdbResolution"`
+	LookupLimit    int32   `json:"lookupLimit"`
+}
+
 func newInstanceSettings(httpClientProvider httpclient.Provider) datasource.InstanceFactoryFunc {
 	return func(ctx context.Context, settings backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
 		opts, err := settings.HTTPClientOptions(ctx)
@@ -53,9 +64,18 @@ func newInstanceSettings(httpClientProvider httpclient.Provider) datasource.Inst
 			return nil, err
 		}

+		jsonData := JSONData{}
+		err = json.Unmarshal(settings.JSONData, &jsonData)
+		if err != nil {
+			return nil, fmt.Errorf("error reading settings: %w", err)
+		}
+
 		model := &datasourceInfo{
-			HTTPClient: client,
-			URL:        settings.URL,
+			HTTPClient:     client,
+			URL:            settings.URL,
+			TSDBVersion:    jsonData.TSDBVersion,
+			TSDBResolution: jsonData.TSDBResolution,
+			LookupLimit:    jsonData.LookupLimit,
 		}

 		return model, nil
@@ -69,7 +89,7 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest)

 	q := req.Queries[0]

-	myRefID := q.RefID
+	refID := q.RefID

 	tsdbQuery.Start = q.TimeRange.From.UnixNano() / int64(time.Millisecond)
 	tsdbQuery.End = q.TimeRange.To.UnixNano() / int64(time.Millisecond)
@@ -106,7 +126,7 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest)
 		}
 	}()

-	result, err := s.parseResponse(logger, res, myRefID)
+	result, err := s.parseResponse(logger, res, refID, dsInfo.TSDBVersion)
 	if err != nil {
 		return &backend.QueryDataResponse{}, err
 	}
@@ -120,9 +140,11 @@ func (s *Service) createRequest(ctx context.Context, logger log.Logger, dsInfo *
 		return nil, err
 	}
 	u.Path = path.Join(u.Path, "api/query")
-	queryParams := u.Query()
-	queryParams.Set("arrays", "true")
-	u.RawQuery = queryParams.Encode()
+	if dsInfo.TSDBVersion == 4 {
+		queryParams := u.Query()
+		queryParams.Set("arrays", "true")
+		u.RawQuery = queryParams.Encode()
+	}

 	postData, err := json.Marshal(data)
 	if err != nil {
@@ -140,7 +162,67 @@ func (s *Service) createRequest(ctx context.Context, logger log.Logger, dsInfo *
 	return req, nil
 }

-func (s *Service) parseResponse(logger log.Logger, res *http.Response, myRefID string) (*backend.QueryDataResponse, error) {
+func createInitialFrame(val OpenTsdbCommon, length int, refID string) *data.Frame {
+	labels := data.Labels{}
+	for label, value := range val.Tags {
+		labels[label] = value
+	}
+
+	frame := data.NewFrameOfFieldTypes(val.Metric, length, data.FieldTypeTime, data.FieldTypeFloat64)
+	frame.Meta = &data.FrameMeta{Type: data.FrameTypeTimeSeriesMulti, TypeVersion: data.FrameTypeVersion{0, 1}}
+	frame.RefID = refID
+	timeField := frame.Fields[0]
+	timeField.Name = data.TimeSeriesTimeFieldName
+	dataField := frame.Fields[1]
+	dataField.Name = val.Metric
+	dataField.Labels = labels
+
+	return frame
+}
+
+// Parse response function for OpenTSDB version 2.4
+func parseResponse24(responseData []OpenTsdbResponse24, refID string, frames data.Frames) data.Frames {
+	for _, val := range responseData {
+		frame := createInitialFrame(val.OpenTsdbCommon, len(val.DataPoints), refID)
+
+		for i, point := range val.DataPoints {
+			frame.SetRow(i, time.Unix(int64(point[0]), 0).UTC(), point[1])
+		}
+
+		frames = append(frames, frame)
+	}
+
+	return frames
+}
+
+// Parse response function for OpenTSDB versions < 2.4
+func parseResponseLT24(responseData []OpenTsdbResponse, refID string, frames data.Frames) (data.Frames, error) {
+	for _, val := range responseData {
+		frame := createInitialFrame(val.OpenTsdbCommon, len(val.DataPoints), refID)
+
+		// Order the timestamps in ascending order to avoid issues like https://github.com/grafana/grafana/issues/38729
+		timestamps := make([]string, 0, len(val.DataPoints))
+		for timestamp := range val.DataPoints {
+			timestamps = append(timestamps, timestamp)
+		}
+		sort.Strings(timestamps)
+
+		for i, timeString := range timestamps {
+			timestamp, err := strconv.ParseInt(timeString, 10, 64)
+			if err != nil {
+				logger.Info("Failed to unmarshal opentsdb timestamp", "timestamp", timeString)
+				return frames, err
+			}
+			frame.SetRow(i, time.Unix(timestamp, 0).UTC(), val.DataPoints[timeString])
+		}
+
+		frames = append(frames, frame)
+	}
+
+	return frames, nil
+}
+
+func (s *Service) parseResponse(logger log.Logger, res *http.Response, refID string, tsdbVersion float32) (*backend.QueryDataResponse, error) {
 	resp := backend.NewQueryDataResponse()

 	body, err := io.ReadAll(res.Body)
@@ -158,38 +240,34 @@ func (s *Service) parseResponse(logger log.Logger, res *http.Response, myRefID s
 		return nil, fmt.Errorf("request failed, status: %s", res.Status)
 	}

-	var responseData []OpenTsdbResponse
-	err = json.Unmarshal(body, &responseData)
-	if err != nil {
-		logger.Info("Failed to unmarshal opentsdb response", "error", err, "status", res.Status, "body", string(body))
-		return nil, err
-	}
-
 	frames := data.Frames{}
-	for _, val := range responseData {
-		labels := data.Labels{}
-		for label, value := range val.Tags {
-			labels[label] = value
+
+	var responseData []OpenTsdbResponse
+	var responseData24 []OpenTsdbResponse24
+	if tsdbVersion == 4 {
+		err = json.Unmarshal(body, &responseData24)
+		if err != nil {
+			logger.Info("Failed to unmarshal opentsdb response", "error", err, "status", res.Status, "body", string(body))
+			return nil, err
 		}

-		frame := data.NewFrameOfFieldTypes(val.Metric, len(val.DataPoints), data.FieldTypeTime, data.FieldTypeFloat64)
-		frame.Meta = &data.FrameMeta{Type: data.FrameTypeTimeSeriesMulti, TypeVersion: data.FrameTypeVersion{0, 1}}
-		frame.RefID = myRefID
-		timeField := frame.Fields[0]
-		timeField.Name = data.TimeSeriesTimeFieldName
-		dataField := frame.Fields[1]
-		dataField.Name = "value"
-		dataField.Labels = labels
+		frames = parseResponse24(responseData24, refID, frames)
+	} else {
+		err = json.Unmarshal(body, &responseData)
+		if err != nil {
+			logger.Info("Failed to unmarshal opentsdb response", "error", err, "status", res.Status, "body", string(body))
+			return nil, err
+		}

-		points := val.DataPoints
-		for i, point := range points {
-			frame.SetRow(i, time.Unix(int64(point[0]), 0).UTC(), point[1])
+		frames, err = parseResponseLT24(responseData, refID, frames)
+		if err != nil {
+			return nil, err
 		}
-		frames = append(frames, frame)
 	}
-	result := resp.Responses[myRefID]
+
+	result := resp.Responses[refID]
 	result.Frames = frames
-	resp.Responses[myRefID] = result
+	resp.Responses[refID] = result
 	return resp, nil
 }

@@ -33,12 +33,13 @@ func TestOpenTsdbExecutor(t *testing.T) {
 	t.Run("Parse response should handle invalid JSON", func(t *testing.T) {
 		response := `{ invalid }`

-		result, err := service.parseResponse(logger, &http.Response{Body: io.NopCloser(strings.NewReader(response))}, "A")
+		tsdbVersion := float32(4)
+		result, err := service.parseResponse(logger, &http.Response{Body: io.NopCloser(strings.NewReader(response))}, "A", tsdbVersion)
 		require.Nil(t, result)
 		require.Error(t, err)
 	})

-	t.Run("Parse response should handle JSON", func(t *testing.T) {
+	t.Run("Parse response should handle JSON (v2.4 and above)", func(t *testing.T) {
 		response := `
 		[
 			{
@@ -57,7 +58,7 @@ func TestOpenTsdbExecutor(t *testing.T) {
 			data.NewField("Time", nil, []time.Time{
 				time.Date(2014, 7, 16, 20, 55, 46, 0, time.UTC),
 			}),
-			data.NewField("value", map[string]string{"env": "prod", "app": "grafana"}, []float64{
+			data.NewField("test", map[string]string{"env": "prod", "app": "grafana"}, []float64{
 				50}),
 		)
 		testFrame.Meta = &data.FrameMeta{
@@ -65,10 +66,124 @@ func TestOpenTsdbExecutor(t *testing.T) {
 			TypeVersion: data.FrameTypeVersion{0, 1},
 		}
 		testFrame.RefID = "A"
+		tsdbVersion := float32(4)

 		resp := http.Response{Body: io.NopCloser(strings.NewReader(response))}
 		resp.StatusCode = 200
-		result, err := service.parseResponse(logger, &resp, "A")
+		result, err := service.parseResponse(logger, &resp, "A", tsdbVersion)
+		require.NoError(t, err)
+
+		frame := result.Responses["A"]
+
+		if diff := cmp.Diff(testFrame, frame.Frames[0], data.FrameTestCompareOptions()...); diff != "" {
+			t.Errorf("Result mismatch (-want +got):\n%s", diff)
+		}
+	})
+
+	t.Run("Parse response should handle JSON (v2.3 and below)", func(t *testing.T) {
+		response := `
+		[
+			{
+				"metric": "test",
+				"dps": {
+					"1405544146": 50.0
+				},
+				"tags" : {
+					"env": "prod",
+					"app": "grafana"
+				}
+			}
+		]`
+
+		testFrame := data.NewFrame("test",
+			data.NewField("Time", nil, []time.Time{
+				time.Date(2014, 7, 16, 20, 55, 46, 0, time.UTC),
+			}),
+			data.NewField("test", map[string]string{"env": "prod", "app": "grafana"}, []float64{
+				50}),
+		)
+		testFrame.Meta = &data.FrameMeta{
+			Type:        data.FrameTypeTimeSeriesMulti,
+			TypeVersion: data.FrameTypeVersion{0, 1},
+		}
+		testFrame.RefID = "A"
+		tsdbVersion := float32(3)
+
+		resp := http.Response{Body: io.NopCloser(strings.NewReader(response))}
+		resp.StatusCode = 200
+		result, err := service.parseResponse(logger, &resp, "A", tsdbVersion)
+		require.NoError(t, err)
+
+		frame := result.Responses["A"]
+
+		if diff := cmp.Diff(testFrame, frame.Frames[0], data.FrameTestCompareOptions()...); diff != "" {
+			t.Errorf("Result mismatch (-want +got):\n%s", diff)
+		}
+	})
+
+	t.Run("Parse response should handle unordered JSON (v2.3 and below)", func(t *testing.T) {
+		response := `
+		[
+			{
+				"metric": "test",
+				"dps": {
+					"1405094109": 55.0,
+					"1405124146": 124.0,
+					"1405124212": 1284.0,
+					"1405019246": 50.0,
+					"1408352146": 812.0,
+					"1405534153": 153.0,
+					"1405124397": 9035.0,
+					"1401234774": 215.0,
+					"1409712532": 356.0,
+					"1491523811": 8953.0,
+					"1405239823": 258.0
+				},
+				"tags" : {
+					"env": "prod",
+					"app": "grafana"
+				}
+			}
+		]`
+
+		testFrame := data.NewFrame("test",
+			data.NewField("Time", nil, []time.Time{
+				time.Date(2014, 5, 27, 23, 52, 54, 0, time.UTC),
+				time.Date(2014, 7, 10, 19, 7, 26, 0, time.UTC),
+				time.Date(2014, 7, 11, 15, 55, 9, 0, time.UTC),
+				time.Date(2014, 7, 12, 0, 15, 46, 0, time.UTC),
+				time.Date(2014, 7, 12, 0, 16, 52, 0, time.UTC),
+				time.Date(2014, 7, 12, 0, 19, 57, 0, time.UTC),
+				time.Date(2014, 7, 13, 8, 23, 43, 0, time.UTC),
+				time.Date(2014, 7, 16, 18, 9, 13, 0, time.UTC),
+				time.Date(2014, 8, 18, 8, 55, 46, 0, time.UTC),
+				time.Date(2014, 9, 3, 2, 48, 52, 0, time.UTC),
+				time.Date(2017, 4, 7, 0, 10, 11, 0, time.UTC),
+			}),
+			data.NewField("test", map[string]string{"env": "prod", "app": "grafana"}, []float64{
+				215,
+				50,
+				55,
+				124,
+				1284,
+				9035,
+				258,
+				153,
+				812,
+				356,
+				8953,
+			}),
+		)
+		testFrame.Meta = &data.FrameMeta{
+			Type:        data.FrameTypeTimeSeriesMulti,
+			TypeVersion: data.FrameTypeVersion{0, 1},
+		}
+		testFrame.RefID = "A"
+		tsdbVersion := float32(3)
+
+		resp := http.Response{Body: io.NopCloser(strings.NewReader(response))}
+		resp.StatusCode = 200
+		result, err := service.parseResponse(logger, &resp, "A", tsdbVersion)
 		require.NoError(t, err)

 		frame := result.Responses["A"]
@@ -99,7 +214,7 @@ func TestOpenTsdbExecutor(t *testing.T) {
 			data.NewField("Time", nil, []time.Time{
 				time.Date(2014, 7, 16, 20, 55, 46, 0, time.UTC),
 			}),
-			data.NewField("value", map[string]string{"env": "prod", "app": "grafana"}, []float64{
+			data.NewField("test", map[string]string{"env": "prod", "app": "grafana"}, []float64{
 				50}),
 		)
 		testFrame.Meta = &data.FrameMeta{
@@ -108,9 +223,11 @@ func TestOpenTsdbExecutor(t *testing.T) {
 		}
 		testFrame.RefID = myRefid

+		tsdbVersion := float32(4)
+
 		resp := http.Response{Body: io.NopCloser(strings.NewReader(response))}
 		resp.StatusCode = 200
-		result, err := service.parseResponse(logger, &resp, myRefid)
+		result, err := service.parseResponse(logger, &resp, myRefid, tsdbVersion)
 		require.NoError(t, err)

 		if diff := cmp.Diff(testFrame, result.Responses[myRefid].Frames[0], data.FrameTestCompareOptions()...); diff != "" {
@@ -6,8 +6,17 @@ type OpenTsdbQuery struct {
 	Queries []map[string]any `json:"queries"`
 }

+type OpenTsdbCommon struct {
+	Metric string            `json:"metric"`
+	Tags   map[string]string `json:"tags"`
+}
+
 type OpenTsdbResponse struct {
-	Metric     string            `json:"metric"`
-	Tags       map[string]string `json:"tags"`
-	DataPoints [][]float64       `json:"dps"`
+	OpenTsdbCommon
+	DataPoints map[string]float64 `json:"dps"`
+}
+
+type OpenTsdbResponse24 struct {
+	OpenTsdbCommon
+	DataPoints [][]float64 `json:"dps"`
 }