Pyroscope: Exemplar support for series queries (#113926)

* feat(pyroscope): Exemplar support for series queries

use enum flag, add exemplar flag to explore

disable exemplars on explore as well

tests

feature toggle

fixing tests

* resolve conflicts

* lint

pkg/tsdb/grafana-pyroscope-datasource/query.go

@@ -13,6 +13,7 @@ import (
 	"github.com/grafana/grafana-plugin-sdk-go/backend/tracing"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
 	"github.com/grafana/grafana-plugin-sdk-go/live"
+	"github.com/grafana/grafana/pkg/tsdb/grafana-pyroscope-datasource/exemplar"
 	"github.com/xlab/treeprint"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/codes"
@@ -21,6 +22,8 @@ import (
 
 	"github.com/grafana/grafana/pkg/tsdb/grafana-pyroscope-datasource/annotation"
 	"github.com/grafana/grafana/pkg/tsdb/grafana-pyroscope-datasource/kinds/dataquery"
+
+	typesv1 "github.com/grafana/pyroscope/api/gen/proto/go/types/v1"
 )
 
 type queryModel struct {
@@ -36,8 +39,12 @@ const (
 	queryTypeProfile = string(dataquery.PyroscopeQueryTypeProfile)
 	queryTypeMetrics = string(dataquery.PyroscopeQueryTypeMetrics)
 	queryTypeBoth    = string(dataquery.PyroscopeQueryTypeBoth)
+
+	exemplarsFeatureToggle = "profilesExemplars"
 )
 
+var identityTransformation = func(value float64) float64 { return value }
+
 // query processes single Pyroscope query transforming the response to data.Frame packaged in DataResponse
 func (d *PyroscopeDatasource) query(ctx context.Context, pCtx backend.PluginContext, query backend.DataQuery) backend.DataResponse {
 	ctx, span := tracing.DefaultTracer().Start(ctx, "datasource.pyroscope.query", trace.WithAttributes(attribute.String("query_type", query.QueryType)))
@@ -77,6 +84,10 @@ func (d *PyroscopeDatasource) query(ctx context.Context, pCtx backend.PluginCont
 					logger.Error("Failed to parse the MinStep using default", "MinStep", dsJson.MinStep, "function", logEntrypoint())
 				}
 			}
+			exemplarType := typesv1.ExemplarType_EXEMPLAR_TYPE_NONE
+			if qm.IncludeExemplars && backend.GrafanaConfigFromContext(ctx).FeatureToggles().IsEnabled(exemplarsFeatureToggle) {
+				exemplarType = typesv1.ExemplarType_EXEMPLAR_TYPE_INDIVIDUAL
+			}
 			seriesResp, err := d.client.GetSeries(
 				gCtx,
 				profileTypeId,
@@ -86,6 +97,7 @@ func (d *PyroscopeDatasource) query(ctx context.Context, pCtx backend.PluginCont
 				qm.GroupBy,
 				qm.Limit,
 				math.Max(query.Interval.Seconds(), parsedInterval.Seconds()),
+				exemplarType,
 			)
 			if err != nil {
 				span.RecordError(err)
@@ -475,6 +487,7 @@ func seriesToDataFrames(resp *SeriesResponse, withAnnotations bool, stepDuration
 	annotations := make([]*annotation.TimedAnnotation, 0)
 
 	for _, series := range resp.Series {
+		exemplars := make([]*exemplar.Exemplar, 0)
 		// We create separate data frames as the series may not have the same length
 		frame := data.NewFrame("series")
 		frameMeta := &data.FrameMeta{PreferredVisualization: "graph"}
@@ -516,14 +529,20 @@ func seriesToDataFrames(resp *SeriesResponse, withAnnotations bool, stepDuration
 
 			// Apply rate calculation for cumulative profiles
 			value := point.Value
+			transformation := identityTransformation
 			if isCumulativeProfile(profileTypeID) && stepDurationSec > 0 {
-				value = value / stepDurationSec
+				transformation = func(value float64) float64 {
+					return value / stepDurationSec
+				}
 
 				// Convert CPU nanoseconds to cores
 				if isCPUTimeProfile(profileTypeID) {
-					value = value / 1e9
+					transformation = func(value float64) float64 {
+						return value / stepDurationSec / 1e9
+					}
 				}
 			}
+			value = transformation(value)
 			valueField.Append(value)
 			if withAnnotations {
 				for _, a := range point.Annotations {
@@ -533,10 +552,22 @@ func seriesToDataFrames(resp *SeriesResponse, withAnnotations bool, stepDuration
 					})
 				}
 			}
+			for _, e := range point.Exemplars {
+				exemplars = append(exemplars, &exemplar.Exemplar{
+					Id:        e.Id,
+					Value:     transformation(float64(e.Value)),
+					Timestamp: e.Timestamp,
+				})
+			}
 		}
 
 		frame.Fields = fields
 		frames = append(frames, frame)
+
+		if len(exemplars) > 0 {
+			frame := exemplar.CreateExemplarFrame(labels, exemplars)
+			frames = append(frames, frame)
+		}
 	}
 
 	if len(annotations) > 0 {