Chore: Add tracing to tempo, parca and pyroscope datasource backends (#76368)

* Added spans to trace.go

* Added spans to search_stream.go

* Added spans to parca datasource

* Added spans for pyroscope

* Fix tests

* Fix another test

* Lint

* Revert "Fix another test"

This reverts commit a1639049e3.

* Use grafana-sdk-go tracing

pkg/tsdb/tempo/trace.go

@@ -8,15 +8,24 @@ import (
 	"net/http"
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
+	"github.com/grafana/grafana-plugin-sdk-go/backend/tracing"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
 	"github.com/grafana/grafana/pkg/tsdb/tempo/kinds/dataquery"
 	"go.opentelemetry.io/collector/pdata/ptrace"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
 )
 
 func (s *Service) getTrace(ctx context.Context, pCtx backend.PluginContext, query backend.DataQuery) (*backend.DataResponse, error) {
 	result := &backend.DataResponse{}
 	refID := query.RefID
 
+	ctx, span := tracing.DefaultTracer().Start(ctx, "datasource.tempo.getTrace", trace.WithAttributes(
+		attribute.String("queryType", query.QueryType),
+	))
+	defer span.End()
+
 	model := &dataquery.TempoQuery{}
 	err := json.Unmarshal(query.JSON, model)
 	if err != nil {
@@ -34,11 +43,15 @@ func (s *Service) getTrace(ctx context.Context, pCtx backend.PluginContext, quer
 
 	request, err := s.createRequest(ctx, dsInfo, *model.Query, query.TimeRange.From.Unix(), query.TimeRange.To.Unix())
 	if err != nil {
+		span.RecordError(err)
+		span.SetStatus(codes.Error, err.Error())
 		return result, err
 	}
 
 	resp, err := dsInfo.HTTPClient.Do(request)
 	if err != nil {
+		span.RecordError(err)
+		span.SetStatus(codes.Error, err.Error())
 		return result, fmt.Errorf("failed get to tempo: %w", err)
 	}
 
@@ -55,6 +68,8 @@ func (s *Service) getTrace(ctx context.Context, pCtx backend.PluginContext, quer
 
 	if resp.StatusCode != http.StatusOK {
 		result.Error = fmt.Errorf("failed to get trace with id: %v Status: %s Body: %s", model.Query, resp.Status, string(body))
+		span.RecordError(result.Error)
+		span.SetStatus(codes.Error, result.Error.Error())
 		return result, nil
 	}
 
@@ -62,11 +77,15 @@ func (s *Service) getTrace(ctx context.Context, pCtx backend.PluginContext, quer
 	otTrace, err := pbUnmarshaler.UnmarshalTraces(body)
 
 	if err != nil {
+		span.RecordError(err)
+		span.SetStatus(codes.Error, err.Error())
 		return &backend.DataResponse{}, fmt.Errorf("failed to convert tempo response to Otlp: %w", err)
 	}
 
 	frame, err := TraceToFrame(otTrace)
 	if err != nil {
+		span.RecordError(err)
+		span.SetStatus(codes.Error, err.Error())
 		return &backend.DataResponse{}, fmt.Errorf("failed to transform trace %v to data frame: %w", model.Query, err)
 	}
 	frame.RefID = refID