Prometheus: Add Exemplar sampling for streaming parser (#56049)

2022-10-04 08:40:01 -04:00
parent 4eea5d5190
commit 152c7f149a
30 changed files with 76206 additions and 41 deletions
@@ -0,0 +1,42 @@
+GO = go
+SHELL = /bin/zsh
+
+ITERATIONS=10
+BENCH=repeat $(ITERATIONS) $(LEFT_BRACKET) $(GO) test -benchmem -run=^$$ -bench
+PROFILE=$(GO) test -benchmem -run=^$$ -benchtime 1x -memprofile memprofile.out -memprofilerate 1 -cpuprofile cpuprofile.out  -bench
+
+LEFT_BRACKET = {
+RIGHT_BRACKET = }
+
+memprofile-exemplar memprofile-range: %: --%
+	$(GO) tool pprof -http=localhost:6061 memprofile.out
+
+cpuprofile-exemplar cpuprofile-range: %: --%
+	$(GO) tool pprof -http=localhost:6061 cpuprofile.out
+
+benchmark-exemplar benchmark-range: %: --%
+	sed -i 's/buffered/querydata/g' old.txt
+	benchstat old.txt new.txt
+	rm old.txt new.txt
+
+--benchmark-range:
+	$(BENCH) ^BenchmarkRangeJson ./buffered >> old.txt $(RIGHT_BRACKET)
+	$(BENCH) ^BenchmarkRangeJson ./querydata >> new.txt $(RIGHT_BRACKET)
+
+--memprofile-range:
+	$(PROFILE) ^BenchmarkRangeJson ./querydata
+
+--cpuprofile-range:
+	$(PROFILE) ^BenchmarkRangeJson ./querydata
+
+--benchmark-exemplar:
+	$(BENCH) ^BenchmarkExemplarJson ./buffered >> old.txt $(RIGHT_BRACKET)
+	$(BENCH) ^BenchmarkExemplarJson ./querydata >> new.txt $(RIGHT_BRACKET)
+
+--memprofile-exemplar:
+	$(PROFILE) ^BenchmarkExemplarJson ./querydata
+
+--cpuprofile-exemplar:
+	$(PROFILE) ^BenchmarkExemplarJson ./querydata
+
+.PHONY: benchmark-range benchmark-exemplar memprofile-range memprofile-exemplar cpuprofile-range cpuprofile-exemplar
@@ -26,16 +26,17 @@ import (

 var update = true

-func TestMatrixResponses(t *testing.T) {
+func TestResponses(t *testing.T) {
 	tt := []struct {
 		name     string
 		filepath string
 	}{
 		{name: "parse a simple matrix response", filepath: "range_simple"},
 		{name: "parse a simple matrix response with value missing steps", filepath: "range_missing"},
-		{name: "parse a response with Infinity", filepath: "range_infinity"},
-		{name: "parse a response with NaN", filepath: "range_nan"},
+		{name: "parse a matrix response with Infinity", filepath: "range_infinity"},
+		{name: "parse a matrix response with NaN", filepath: "range_nan"},
 		{name: "parse a response with legendFormat __auto", filepath: "range_auto"},
+		{name: "parse an exemplar response", filepath: "exemplar"},
 	}

 	for _, test := range tt {
@@ -96,13 +97,14 @@ func makeMockedApi(responseBytes []byte) (apiv1.API, error) {
 // struct here, because it has `time.time` and `time.duration` fields that
 // cannot be unmarshalled from JSON automatically.
 type storedPrometheusQuery struct {
-	RefId        string
-	RangeQuery   bool
-	Start        int64
-	End          int64
-	Step         int64
-	Expr         string
-	LegendFormat string
+	RefId         string
+	RangeQuery    bool
+	ExemplarQuery bool
+	Start         int64
+	End           int64
+	Step          int64
+	Expr          string
+	LegendFormat  string
 }

 func loadStoredPrometheusQuery(fileName string) (storedPrometheusQuery, error) {
@@ -126,11 +128,12 @@ func runQuery(response []byte, sq storedPrometheusQuery) (*backend.QueryDataResp
 	tracer := tracing.InitializeTracerForTest()

 	qm := QueryModel{
-		RangeQuery:   sq.RangeQuery,
-		Expr:         sq.Expr,
-		Interval:     fmt.Sprintf("%ds", sq.Step),
-		IntervalMS:   sq.Step * 1000,
-		LegendFormat: sq.LegendFormat,
+		RangeQuery:    sq.RangeQuery,
+		ExemplarQuery: sq.ExemplarQuery,
+		Expr:          sq.Expr,
+		Interval:      fmt.Sprintf("%ds", sq.Step),
+		IntervalMS:    sq.Step * 1000,
+		LegendFormat:  sq.LegendFormat,
 	}

 	b := Buffered{
@@ -165,6 +168,14 @@ func runQuery(response []byte, sq storedPrometheusQuery) (*backend.QueryDataResp
 		return nil, err
 	}

+	// parseTimeSeriesQuery forces range queries if the only query is an exemplar query
+	// so we need to set it back to false
+	if qm.ExemplarQuery {
+		for i := range queries {
+			queries[i].RangeQuery = false
+		}
+	}
+
 	return b.runQueries(context.Background(), queries)
 }

@@ -2,8 +2,11 @@ package buffered

 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"math/rand"
+	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 	"time"
@@ -13,9 +16,37 @@ import (
 )

 // when memory-profiling this benchmark, these commands are recommended:
-// - go test -benchmem -run=^$ -benchtime 1x -memprofile memprofile.out -memprofilerate 1 -bench ^BenchmarkJson$ github.com/grafana/grafana/pkg/tsdb/prometheus
+// - go test -benchmem -run=^$ -benchtime 1x -memprofile memprofile.out -memprofilerate 1 -bench ^BenchmarkExemplarJson$ github.com/grafana/grafana/pkg/tsdb/prometheus/buffered
 // - go tool pprof -http=localhost:6061 memprofile.out
-func BenchmarkJson(b *testing.B) {
+func BenchmarkExemplarJson(b *testing.B) {
+	queryFileName := filepath.Join("../testdata", "exemplar.query.json")
+	query, err := loadStoredQuery(queryFileName)
+	require.NoError(b, err)
+
+	responseFileName := filepath.Join("../testdata", "exemplar.result.json")
+	// This is a test, so it's safe to ignore gosec warning G304.
+	// nolint:gosec
+	responseBytes, err := os.ReadFile(responseFileName)
+	require.NoError(b, err)
+
+	api, err := makeMockedApi(responseBytes)
+	require.NoError(b, err)
+
+	tracer := tracing.InitializeTracerForTest()
+
+	s := Buffered{tracer: tracer, log: &fakeLogger{}, client: api}
+
+	b.ResetTimer()
+	for n := 0; n < b.N; n++ {
+		_, err := s.runQueries(context.Background(), []*PrometheusQuery{query})
+		require.NoError(b, err)
+	}
+}
+
+// when memory-profiling this benchmark, these commands are recommended:
+// - go test -benchmem -run=^$ -benchtime 1x -memprofile memprofile.out -memprofilerate 1 -bench ^BenchmarkRangeJson$ github.com/grafana/grafana/pkg/tsdb/prometheus/buffered
+// - go tool pprof -http=localhost:6061 memprofile.out
+func BenchmarkRangeJson(b *testing.B) {
 	resp, query := createJsonTestData(1642000000, 1, 300, 400)

 	api, err := makeMockedApi(resp)
@@ -82,3 +113,28 @@ func createJsonTestData(start int64, step int64, timestampCount int, seriesCount

 	return bytes, query
 }
+
+func loadStoredQuery(fileName string) (*PrometheusQuery, error) {
+	// This is a test, so it's safe to ignore gosec warning G304.
+	// nolint:gosec
+	bytes, err := os.ReadFile(fileName)
+	if err != nil {
+		return nil, err
+	}
+
+	var sq storedPrometheusQuery
+
+	err = json.Unmarshal(bytes, &sq)
+	if err != nil {
+		return nil, err
+	}
+
+	return &PrometheusQuery{
+		RefId:         "A",
+		ExemplarQuery: sq.ExemplarQuery,
+		Start:         time.Unix(sq.Start, 0),
+		End:           time.Unix(sq.End, 0),
+		Step:          time.Second * time.Duration(sq.Step),
+		Expr:          sq.Expr,
+	}, nil
+}
@@ -564,6 +564,10 @@ func exemplarToDataFrames(response []apiv1.ExemplarQueryResult, query *Prometheu
 		}
 	}

+	sort.SliceStable(sampleExemplars, func(i, j int) bool {
+		return sampleExemplars[i].Time.Before(sampleExemplars[j].Time)
+	})
+
 	// Create DF from sampled exemplars
 	timeField := data.NewFieldFromFieldType(data.FieldTypeTime, len(sampleExemplars))
 	timeField.Name = "Time"
@@ -133,8 +133,8 @@ func (query *Query) TimeRange() TimeRange {
 	return TimeRange{
 		Step: query.Step,
 		// Align query range to step. It rounds start and end down to a multiple of step.
-		Start: alignTimeRange(query.Start, query.Step, query.UtcOffsetSec),
-		End:   alignTimeRange(query.End, query.Step, query.UtcOffsetSec),
+		Start: AlignTimeRange(query.Start, query.Step, query.UtcOffsetSec),
+		End:   AlignTimeRange(query.End, query.Step, query.UtcOffsetSec),
 	}
 }

@@ -225,6 +225,8 @@ func isVariableInterval(interval string) bool {
 	return false
 }

-func alignTimeRange(t time.Time, step time.Duration, offset int64) time.Time {
-	return time.Unix(int64(math.Floor((float64(t.Unix()+offset)/step.Seconds()))*step.Seconds()-float64(offset)), 0)
+func AlignTimeRange(t time.Time, step time.Duration, offset int64) time.Time {
+	offsetNano := float64(offset * 1e9)
+	stepNano := float64(step.Nanoseconds())
+	return time.Unix(0, int64(math.Floor((float64(t.UnixNano())+offsetNano)/stepNano)*stepNano-offsetNano)).UTC()
 }
@@ -1,6 +1,7 @@
 package models_test

 import (
+	"reflect"
 	"testing"
 	"time"

@@ -15,7 +16,7 @@ var (
 	intervalCalculator = intervalv2.NewCalculator()
 )

-func TestPrometheus_timeSeriesQuery_parseTimeSeriesQuery(t *testing.T) {
+func TestParse(t *testing.T) {
 	t.Run("parsing query from unified alerting", func(t *testing.T) {
 		timeRange := backend.TimeRange{
 			From: now,
@@ -449,3 +450,27 @@ func queryContext(json string, timeRange backend.TimeRange) backend.DataQuery {
 		RefID:     "A",
 	}
 }
+
+func TestAlignTimeRange(t *testing.T) {
+	type args struct {
+		t      time.Time
+		step   time.Duration
+		offset int64
+	}
+	tests := []struct {
+		name string
+		args args
+		want time.Time
+	}{
+		{name: "second step", args: args{t: time.Unix(1664816826, 0), step: 10 * time.Second, offset: 0}, want: time.Unix(1664816820, 0).UTC()},
+		{name: "millisecond step", args: args{t: time.Unix(1664816825, 5*int64(time.Millisecond)), step: 10 * time.Millisecond, offset: 0}, want: time.Unix(1664816825, 0).UTC()},
+		{name: "second step with offset", args: args{t: time.Unix(1664816825, 5*int64(time.Millisecond)), step: 2 * time.Second, offset: -3}, want: time.Unix(1664816825, 0).UTC()},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := models.AlignTimeRange(tt.args.t, tt.args.step, tt.args.offset); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("AlignTimeRange() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
@@ -0,0 +1,42 @@
+package models
+
+import "github.com/grafana/grafana-plugin-sdk-go/data"
+
+type ResultType string
+
+const (
+	ResultTypeMatrix   ResultType = "matrix"
+	ResultTypeExemplar ResultType = "exemplar"
+	ResultTypeVector   ResultType = "vector"
+	ResultTypeUnknown  ResultType = ""
+)
+
+func ResultTypeFromFrame(frame *data.Frame) ResultType {
+	if frame.Meta.Custom == nil {
+		return ResultTypeUnknown
+	}
+	custom, ok := frame.Meta.Custom.(map[string]string)
+	if !ok {
+		return ResultTypeUnknown
+	}
+
+	rt, ok := custom["resultType"]
+	if !ok {
+		return ResultTypeUnknown
+	}
+
+	switch rt {
+	case ResultTypeMatrix.String():
+		return ResultTypeMatrix
+	case ResultTypeExemplar.String():
+		return ResultTypeExemplar
+	case ResultTypeVector.String():
+		return ResultTypeVector
+	}
+
+	return ResultTypeUnknown
+}
+
+func (r ResultType) String() string {
+	return string(r)
+}
@@ -0,0 +1,125 @@
+package querydata
+
+import (
+	"math"
+	"sort"
+	"time"
+
+	"github.com/grafana/grafana/pkg/tsdb/prometheus/models"
+)
+
+type exemplar struct {
+	seriesLabels map[string]string
+	labels       map[string]string
+	val          float64
+	ts           time.Time
+}
+
+type exemplarSampler struct {
+	buckets  map[time.Time][]exemplar
+	labelSet map[string]struct{}
+	count    int
+	mean     float64
+	m2       float64
+}
+
+func newExemplarSampler() *exemplarSampler {
+	return &exemplarSampler{
+		buckets:  map[time.Time][]exemplar{},
+		labelSet: map[string]struct{}{},
+	}
+}
+
+func (e *exemplarSampler) update(step time.Duration, ts time.Time, val float64, seriesLabels, labels map[string]string) {
+	bucketTs := models.AlignTimeRange(ts, step, 0)
+	e.trackNewLabels(seriesLabels, labels)
+	e.updateAggregations(val)
+
+	ex := exemplar{
+		val:          val,
+		ts:           ts,
+		labels:       labels,
+		seriesLabels: seriesLabels,
+	}
+
+	if _, exists := e.buckets[bucketTs]; !exists {
+		e.buckets[bucketTs] = []exemplar{ex}
+		return
+	}
+
+	e.buckets[bucketTs] = append(e.buckets[bucketTs], ex)
+}
+
+// updateAggregations uses Welford's online algorithm for calculating the mean and variance
+// https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm
+func (e *exemplarSampler) updateAggregations(val float64) {
+	e.count++
+	delta := val - e.mean
+	e.mean += delta / float64(e.count)
+	delta2 := val - e.mean
+	e.m2 += delta * delta2
+}
+
+// standardDeviation calculates the amount of varation in the data
+// https://en.wikipedia.org/wiki/Standard_deviation
+func (e *exemplarSampler) standardDeviation() float64 {
+	if e.count < 2 {
+		return 0
+	}
+	return math.Sqrt(e.m2 / float64(e.count-1))
+}
+
+// trackNewLabels saves label names that haven't been seen before
+// so that they can be used to build the label fields in the exemplar frame
+func (e *exemplarSampler) trackNewLabels(seriesLabels, labels map[string]string) {
+	for k := range labels {
+		if _, ok := e.labelSet[k]; !ok {
+			e.labelSet[k] = struct{}{}
+		}
+	}
+	for k := range seriesLabels {
+		if _, ok := e.labelSet[k]; !ok {
+			e.labelSet[k] = struct{}{}
+		}
+	}
+}
+
+// getLabelNames returns sorted unique label names
+func (e *exemplarSampler) getLabelNames() []string {
+	labelNames := make([]string, 0, len(e.labelSet))
+	for k := range e.labelSet {
+		labelNames = append(labelNames, k)
+	}
+	sort.SliceStable(labelNames, func(i, j int) bool {
+		return labelNames[i] < labelNames[j]
+	})
+	return labelNames
+}
+
+// getSampledExemplars returns the exemplars sorted by timestamp
+func (e *exemplarSampler) getSampledExemplars() []exemplar {
+	exemplars := make([]exemplar, 0, len(e.buckets))
+	for _, b := range e.buckets {
+		// sort by value in descending order
+		sort.SliceStable(b, func(i, j int) bool {
+			return b[i].val > b[j].val
+		})
+		sampled := []exemplar{}
+		for _, ex := range b {
+			if len(sampled) == 0 {
+				sampled = append(sampled, ex)
+				continue
+			}
+			// only sample values at least 2 standard deviation distance to previously taken value
+			prev := sampled[len(sampled)-1]
+			if e.standardDeviation() != 0.0 && prev.val-ex.val > e.standardDeviation()*2.0 {
+				sampled = append(sampled, ex)
+			}
+		}
+		exemplars = append(exemplars, sampled...)
+	}
+	sort.SliceStable(exemplars, func(i, j int) bool {
+		return exemplars[i].ts.Before(exemplars[j].ts)
+	})
+	return exemplars
+}
@@ -8,6 +8,8 @@ import (
 	"io"
 	"math/rand"
 	"net/http"
+	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 	"time"
@@ -17,24 +19,62 @@ import (
 	"github.com/stretchr/testify/require"
 )

+// when memory-profiling this benchmark, these commands are recommended:
+// - go test -benchmem -run=^$ -benchtime 1x -memprofile memprofile.out -memprofilerate 1 -bench ^BenchmarkExemplarJson$ github.com/grafana/grafana/pkg/tsdb/prometheus/buffered
+// - go tool pprof -http=localhost:6061 memprofile.out
+func BenchmarkExemplarJson(b *testing.B) {
+	queryFileName := filepath.Join("../testdata", "exemplar.query.json")
+	query, err := loadStoredQuery(queryFileName)
+	require.NoError(b, err)
+
+	responseFileName := filepath.Join("../testdata", "exemplar.result.json")
+
+	// nolint:gosec
+	// We can ignore the gosec G304 warning since this is a test file
+	responseBytes, err := os.ReadFile(responseFileName)
+	require.NoError(b, err)
+
+	tCtx, err := setup(true)
+	require.NoError(b, err)
+	b.ResetTimer()
+	for n := 0; n < b.N; n++ {
+		res := http.Response{
+			StatusCode: 200,
+			Body:       io.NopCloser(bytes.NewReader(responseBytes)),
+		}
+		tCtx.httpProvider.setResponse(&res)
+		_, err := tCtx.queryData.Execute(context.Background(), query)
+		require.NoError(b, err)
+	}
+}
+
+var resp *backend.QueryDataResponse
+
 // when memory-profiling this benchmark, these commands are recommended:
 // - go test -benchmem -run=^$ -benchtime 1x -memprofile memprofile.out -memprofilerate 1 -bench ^BenchmarkJson$ github.com/grafana/grafana/pkg/tsdb/prometheus
 // - go tool pprof -http=localhost:6061 memprofile.out
-func BenchmarkJson(b *testing.B) {
+func BenchmarkRangeJson(b *testing.B) {
+	var (
+		r   *backend.QueryDataResponse
+		err error
+	)
 	body, q := createJsonTestData(1642000000, 1, 300, 400)
 	tCtx, err := setup(true)
 	require.NoError(b, err)

 	b.ResetTimer()
+
 	for n := 0; n < b.N; n++ {
 		res := http.Response{
 			StatusCode: 200,
 			Body:       io.NopCloser(bytes.NewReader(body)),
 		}
 		tCtx.httpProvider.setResponse(&res)
-		_, err := tCtx.queryData.Execute(context.Background(), q)
+		r, err = tCtx.queryData.Execute(context.Background(), q)
 		require.NoError(b, err)
 	}
+
+	resp = r
 }

 const nanRate = 0.002
@@ -23,15 +23,15 @@ import (

 var update = true

-func TestMatrixResponses(t *testing.T) {
+func TestRangeResponses(t *testing.T) {
 	tt := []struct {
 		name     string
 		filepath string
 	}{
 		{name: "parse a simple matrix response", filepath: "range_simple"},
 		{name: "parse a simple matrix response with value missing steps", filepath: "range_missing"},
-		{name: "parse a response with Infinity", filepath: "range_infinity"},
-		{name: "parse a response with NaN", filepath: "range_nan"},
+		{name: "parse a matrix response with Infinity", filepath: "range_infinity"},
+		{name: "parse a matrix response with NaN", filepath: "range_nan"},
 		{name: "parse a response with legendFormat __auto", filepath: "range_auto"},
 	}

@@ -47,6 +47,26 @@ func TestMatrixResponses(t *testing.T) {
 	}
 }

+func TestExemplarResponses(t *testing.T) {
+	tt := []struct {
+		name     string
+		filepath string
+	}{
+		{name: "parse an exemplar response", filepath: "exemplar"},
+	}
+
+	for _, test := range tt {
+		enableWideSeries := false
+		queryFileName := filepath.Join("../testdata", test.filepath+".query.json")
+		responseFileName := filepath.Join("../testdata", test.filepath+".result.json")
+		goldenFileName := test.filepath + ".result.golden"
+		t.Run(test.name, goldenScenario(test.name, queryFileName, responseFileName, goldenFileName, enableWideSeries))
+		enableWideSeries = true
+		goldenFileName = test.filepath + ".result.streaming-wide.golden"
+		t.Run(test.name, goldenScenario(test.name, queryFileName, responseFileName, goldenFileName, enableWideSeries))
+	}
+}
+
 func goldenScenario(name, queryFileName, responseFileName, goldenFileName string, wide bool) func(t *testing.T) {
 	return func(t *testing.T) {
 		query, err := loadStoredQuery(queryFileName)
@@ -72,13 +92,14 @@ func goldenScenario(name, queryFileName, responseFileName, goldenFileName string
 // struct here, because it has `time.time` and `time.duration` fields that
 // cannot be unmarshalled from JSON automatically.
 type storedPrometheusQuery struct {
-	RefId        string
-	RangeQuery   bool
-	Start        int64
-	End          int64
-	Step         int64
-	Expr         string
-	LegendFormat string
+	RefId         string
+	RangeQuery    bool
+	ExemplarQuery bool
+	Start         int64
+	End           int64
+	Step          int64
+	Expr          string
+	LegendFormat  string
 }

 func loadStoredQuery(fileName string) (*backend.QueryDataRequest, error) {
@@ -96,11 +117,12 @@ func loadStoredQuery(fileName string) (*backend.QueryDataRequest, error) {
 	}

 	qm := models.QueryModel{
-		RangeQuery:   sq.RangeQuery,
-		Expr:         sq.Expr,
-		Interval:     fmt.Sprintf("%ds", sq.Step),
-		IntervalMS:   sq.Step * 1000,
-		LegendFormat: sq.LegendFormat,
+		RangeQuery:    sq.RangeQuery,
+		ExemplarQuery: sq.ExemplarQuery,
+		Expr:          sq.Expr,
+		Interval:      fmt.Sprintf("%ds", sq.Step),
+		IntervalMS:    sq.Step * 1000,
+		LegendFormat:  sq.LegendFormat,
 	}

 	data, err := json.Marshal(&qm)
@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"sort"
 	"strings"
+	"time"

 	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
@@ -39,6 +40,7 @@ func (s *QueryData) parseResponse(ctx context.Context, q *models.Query, res *htt
 		}
 	}

+	r = processExemplars(q, r)
 	return r, nil
 }

@@ -67,7 +69,7 @@ func addMetadataToWideFrame(q *models.Query, frame *data.Frame) {
 	}
 	frame.Fields[0].Config = &data.FieldConfig{Interval: float64(q.Step.Milliseconds())}
 	for _, f := range frame.Fields {
-		if f.Name != data.TimeSeriesTimeFieldName {
+		if f.Type() == data.FieldTypeFloat64 || f.Type() == data.FieldTypeNullableFloat64 {
 			f.Name = getName(q, f)
 		}
 	}
@@ -132,3 +134,95 @@ func getName(q *models.Query, field *data.Field) string {

 	return legend
 }
+
+func processExemplars(q *models.Query, dr *backend.DataResponse) *backend.DataResponse {
+	sampler := newExemplarSampler()
+
+	// we are moving from a multi-frame response returned
+	// by the converter to a single exemplar frame,
+	// so we need to build a new frame array with the
+	// old exemplar frames filtered out
+	frames := []*data.Frame{}
+
+	// the new exemplar frame will be a single frame in long format
+	// with a timestamp, metric value, and one or more label fields
+	exemplarFrame := data.NewFrame("exemplar")
+
+	for _, frame := range dr.Frames {
+		// we don't need to process non-exemplar frames
+		// so they can be added to the response
+		if !isExemplarFrame(frame) {
+			frames = append(frames, frame)
+			continue
+		}
+
+		// copy the frame metadata to the new exemplar frame
+		exemplarFrame.Meta = frame.Meta
+		exemplarFrame.RefID = frame.RefID
+		frame.Meta.Type = data.FrameTypeTimeSeriesMany
+
+		step := time.Duration(frame.Fields[0].Config.Interval) * time.Millisecond
+		seriesLabels := getSeriesLabels(frame)
+		for rowIdx := 0; rowIdx < frame.Fields[0].Len(); rowIdx++ {
+			row := frame.RowCopy(rowIdx)
+			ts := row[0].(time.Time)
+			val := row[1].(float64)
+			labels := getLabels(frame, row)
+			sampler.update(step, ts, val, seriesLabels, labels)
+		}
+	}
+
+	exemplars := sampler.getSampledExemplars()
+	if len(exemplars) == 0 {
+		return dr
+	}
+
+	// init the fields for the new exemplar frame
+	timeField := data.NewField(data.TimeSeriesTimeFieldName, nil, make([]time.Time, 0, len(exemplars)))
+	valueField := data.NewField(data.TimeSeriesValueFieldName, nil, make([]float64, 0, len(exemplars)))
+	exemplarFrame.Fields = append(exemplarFrame.Fields, timeField, valueField)
+	labelNames := sampler.getLabelNames()
+	for _, labelName := range labelNames {
+		exemplarFrame.Fields = append(exemplarFrame.Fields, data.NewField(labelName, nil, make([]string, 0, len(exemplars))))
+	}
+
+	// add the sampled exemplars to the new exemplar frame
+	for _, b := range exemplars {
+		timeField.Append(b.ts)
+		valueField.Append(b.val)
+		for i, labelName := range labelNames {
+			labelValue, ok := b.labels[labelName]
+			if !ok {
+				// if the label is not present in the exemplar labels, then use the series label
+				labelValue = b.seriesLabels[labelName]
+			}
+			colIdx := i + 2 // +2 to skip time and value fields
+			exemplarFrame.Fields[colIdx].Append(labelValue)
+		}
+	}
+
+	frames = append(frames, exemplarFrame)
+
+	return &backend.DataResponse{
+		Frames: frames,
+		Error:  dr.Error,
+	}
+}
+
+func isExemplarFrame(frame *data.Frame) bool {
+	rt := models.ResultTypeFromFrame(frame)
+	return rt == models.ResultTypeExemplar
+}
+
+func getSeriesLabels(frame *data.Frame) data.Labels {
+	// series labels are stored on the value field (index 1)
+	return frame.Fields[1].Labels.Copy()
+}
+
+func getLabels(frame *data.Frame, row []interface{}) map[string]string {
+	labels := make(map[string]string)
+	for i := 2; i < len(row); i++ {
+		labels[frame.Fields[i].Name] = row[i].(string)
+	}
+	return labels
+}
@@ -0,0 +1,8 @@
+{
+  "RefId": "A",
+  "ExemplarQuery": true,
+  "Start": 1654086510,
+  "End": 1654086810,
+  "Step": 15,
+  "Expr": "histogram_quantile(0.99, sum(rate(traces_spanmetrics_duration_seconds_bucket[15s])) by (le))"
+}
@@ -32,7 +32,8 @@ func TestReadPromFrames(t *testing.T) {
 		"prom-series",
 		"prom-warnings",
 		"prom-error",
-		"prom-exemplars",
+		"prom-exemplars-a",
+		"prom-exemplars-b",
 		"loki-streams-a",
 		"loki-streams-b",
 		"loki-streams-c",
@@ -0,0 +1,116 @@
+{
+  "frames": [
+    {
+      "schema": {
+        "meta": {
+          "custom": {
+            "resultType": "exemplar"
+          }
+        },
+        "fields": [
+          {
+            "name": "Time",
+            "type": "time",
+            "typeInfo": {
+              "frame": "time.Time"
+            }
+          },
+          {
+            "name": "Value",
+            "type": "number",
+            "typeInfo": {
+              "frame": "float64"
+            },
+            "labels": {
+              "__name__": "test_exemplar_metric_total",
+              "instance": "localhost:8090",
+              "job": "prometheus",
+              "service": "bar"
+            }
+          },
+          {
+            "name": "traceID",
+            "type": "string",
+            "typeInfo": {
+              "frame": "string"
+            }
+          },
+          {
+            "name": "a",
+            "type": "string",
+            "typeInfo": {
+              "frame": "string"
+            }
+          }
+        ]
+      },
+      "data": {
+        "values": [
+          [
+            1600096945479
+          ],
+          [
+            6
+          ],
+          [
+            "EpTxMJ40fUus7aGY"
+          ],
+          [
+            "not in next"
+          ]
+        ]
+      }
+    },
+    {
+      "schema": {
+        "meta": {
+          "custom": {
+            "resultType": "exemplar"
+          }
+        },
+        "fields": [
+          {
+            "name": "Time",
+            "type": "time",
+            "typeInfo": {
+              "frame": "time.Time"
+            }
+          },
+          {
+            "name": "Value",
+            "type": "number",
+            "typeInfo": {
+              "frame": "float64"
+            },
+            "labels": {
+              "__name__": "test_exemplar_metric_total",
+              "instance": "localhost:8090",
+              "job": "prometheus",
+              "service": "foo"
+            }
+          },
+          {
+            "name": "traceID",
+            "type": "string",
+            "typeInfo": {
+              "frame": "string"
+            }
+          }
+        ]
+      },
+      "data": {
+        "values": [
+          [
+            1600096955479,1600096965489
+          ],
+          [
+            19,20
+          ],
+          [
+            "Olp9XHlq763ccsfa","hCtjygkIHwAN9vs4"
+          ]
+        ]
+      }
+    }
+  ]
+}
@@ -0,0 +1,156 @@
+//  🌟 This was machine generated.  Do not edit. 🌟
+//  
+//  Frame[0] {
+//      "custom": {
+//          "resultType": "exemplar"
+//      }
+//  }
+//  Name: 
+//  Dimensions: 4 Fields by 1 Rows
+//  +-----------------------------------+---------------------------------------------------------------------------------------------------+------------------+----------------+
+//  | Name: Time                        | Name: Value                                                                                       | Name: traceID    | Name: a        |
+//  | Labels:                           | Labels: __name__=test_exemplar_metric_total, instance=localhost:8090, job=prometheus, service=bar | Labels:          | Labels:        |
+//  | Type: []time.Time                 | Type: []float64                                                                                   | Type: []string   | Type: []string |
+//  +-----------------------------------+---------------------------------------------------------------------------------------------------+------------------+----------------+
+//  | 2020-09-14 15:22:25.479 +0000 UTC | 6                                                                                                 | EpTxMJ40fUus7aGY | not in next    |
+//  +-----------------------------------+---------------------------------------------------------------------------------------------------+------------------+----------------+
+//  
+//  
+//  
+//  Frame[1] {
+//      "custom": {
+//          "resultType": "exemplar"
+//      }
+//  }
+//  Name: 
+//  Dimensions: 3 Fields by 2 Rows
+//  +-----------------------------------+---------------------------------------------------------------------------------------------------+------------------+
+//  | Name: Time                        | Name: Value                                                                                       | Name: traceID    |
+//  | Labels:                           | Labels: __name__=test_exemplar_metric_total, instance=localhost:8090, job=prometheus, service=foo | Labels:          |
+//  | Type: []time.Time                 | Type: []float64                                                                                   | Type: []string   |
+//  +-----------------------------------+---------------------------------------------------------------------------------------------------+------------------+
+//  | 2020-09-14 15:22:35.479 +0000 UTC | 19                                                                                                | Olp9XHlq763ccsfa |
+//  | 2020-09-14 15:22:45.489 +0000 UTC | 20                                                                                                | hCtjygkIHwAN9vs4 |
+//  +-----------------------------------+---------------------------------------------------------------------------------------------------+------------------+
+//  
+//  
+//  🌟 This was machine generated.  Do not edit. 🌟
+{
+  "frames": [
+    {
+      "schema": {
+        "meta": {
+          "custom": {
+            "resultType": "exemplar"
+          }
+        },
+        "fields": [
+          {
+            "name": "Time",
+            "type": "time",
+            "typeInfo": {
+              "frame": "time.Time"
+            }
+          },
+          {
+            "name": "Value",
+            "type": "number",
+            "typeInfo": {
+              "frame": "float64"
+            },
+            "labels": {
+              "__name__": "test_exemplar_metric_total",
+              "instance": "localhost:8090",
+              "job": "prometheus",
+              "service": "bar"
+            }
+          },
+          {
+            "name": "traceID",
+            "type": "string",
+            "typeInfo": {
+              "frame": "string"
+            }
+          },
+          {
+            "name": "a",
+            "type": "string",
+            "typeInfo": {
+              "frame": "string"
+            }
+          }
+        ]
+      },
+      "data": {
+        "values": [
+          [
+            1600096945479
+          ],
+          [
+            6
+          ],
+          [
+            "EpTxMJ40fUus7aGY"
+          ],
+          [
+            "not in next"
+          ]
+        ]
+      }
+    },
+    {
+      "schema": {
+        "meta": {
+          "custom": {
+            "resultType": "exemplar"
+          }
+        },
+        "fields": [
+          {
+            "name": "Time",
+            "type": "time",
+            "typeInfo": {
+              "frame": "time.Time"
+            }
+          },
+          {
+            "name": "Value",
+            "type": "number",
+            "typeInfo": {
+              "frame": "float64"
+            },
+            "labels": {
+              "__name__": "test_exemplar_metric_total",
+              "instance": "localhost:8090",
+              "job": "prometheus",
+              "service": "foo"
+            }
+          },
+          {
+            "name": "traceID",
+            "type": "string",
+            "typeInfo": {
+              "frame": "string"
+            }
+          }
+        ]
+      },
+      "data": {
+        "values": [
+          [
+            1600096955479,
+            1600096965489
+          ],
+          [
+            19,
+            20
+          ],
+          [
+            "Olp9XHlq763ccsfa",
+            "hCtjygkIHwAN9vs4"
+          ]
+        ]
+      }
+    }
+  ]
+}
@@ -0,0 +1,39 @@
+🌟 This was machine generated.  Do not edit. 🌟
+
+Frame[0] {
+    "custom": {
+        "resultType": "exemplar"
+    }
+}
+Name: 
+Dimensions: 4 Fields by 1 Rows
+-----------------------------------+---------------------------------------------------------------------------------------------------+------------------+----------------+
+| Name: Time                        | Name: Value                                                                                       | Name: traceID    | Name: a        |
+| Labels:                           | Labels: __name__=test_exemplar_metric_total, instance=localhost:8090, job=prometheus, service=bar | Labels:          | Labels:        |
+| Type: []time.Time                 | Type: []float64                                                                                   | Type: []string   | Type: []string |
+-----------------------------------+---------------------------------------------------------------------------------------------------+------------------+----------------+
+| 2020-09-14 15:22:25.479 +0000 UTC | 6                                                                                                 | EpTxMJ40fUus7aGY | not in next    |
+-----------------------------------+---------------------------------------------------------------------------------------------------+------------------+----------------+
+
+
+
+Frame[1] {
+    "custom": {
+        "resultType": "exemplar"
+    }
+}
+Name: 
+Dimensions: 3 Fields by 2 Rows
+-----------------------------------+---------------------------------------------------------------------------------------------------+------------------+
+| Name: Time                        | Name: Value                                                                                       | Name: traceID    |
+| Labels:                           | Labels: __name__=test_exemplar_metric_total, instance=localhost:8090, job=prometheus, service=foo | Labels:          |
+| Type: []time.Time                 | Type: []float64                                                                                   | Type: []string   |
+-----------------------------------+---------------------------------------------------------------------------------------------------+------------------+
+| 2020-09-14 15:22:35.479 +0000 UTC | 19                                                                                                | Olp9XHlq763ccsfa |
+| 2020-09-14 15:22:45.489 +0000 UTC | 20                                                                                                | hCtjygkIHwAN9vs4 |
+-----------------------------------+---------------------------------------------------------------------------------------------------+------------------+
+
+
+====== TEST DATA RESPONSE (arrow base64) ======
+FRAME=QVJST1cxAAD/////8AIAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAAJgAAAADAAAATAAAACgAAAAEAAAApP3//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAADE/f//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAOT9//8IAAAAMAAAACQAAAB7ImN1c3RvbSI6eyJyZXN1bHRUeXBlIjoiZXhlbXBsYXIifX0AAAAABAAAAG1ldGEAAAAABAAAALQBAAC4AAAAWAAAAAQAAABu/v//FAAAADgAAAA4AAAAAAAABTQAAAABAAAABAAAAFz+//8IAAAADAAAAAEAAABhAAAABAAAAG5hbWUAAAAAAAAAAKz///8BAAAAYQAAAL7+//8UAAAAPAAAAEAAAAAAAAAFPAAAAAEAAAAEAAAArP7//wgAAAAQAAAABwAAAHRyYWNlSUQABAAAAG5hbWUAAAAAAAAAAAQABAAEAAAABwAAAHRyYWNlSUQAGv///xQAAADIAAAAyAAAAAAAAAPIAAAAAgAAACwAAAAEAAAADP///wgAAAAQAAAABQAAAFZhbHVlAAAABAAAAG5hbWUAAAAAMP///wgAAAB0AAAAaAAAAHsiX19uYW1lX18iOiJ0ZXN0X2V4ZW1wbGFyX21ldHJpY190b3RhbCIsImluc3RhbmNlIjoibG9jYWxob3N0OjgwOTAiLCJqb2IiOiJwcm9tZXRoZXVzIiwic2VydmljZSI6ImJhciJ9AAAAAAYAAABsYWJlbHMAAAAAAACK////AAACAAUAAABWYWx1ZQASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABMAAAAAAAACkwAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAFRpbWUAAAAABAAAAG5hbWUAAAAAAAAAAAAABgAIAAYABgAAAAAAAwAEAAAAVGltZQAAAAAAAAAA/////zgBAAAUAAAAAAAAAAwAFgAUABMADAAEAAwAAABAAAAAAAAAABQAAAAAAAADBAAKABgADAAIAAQACgAAABQAAAC4AAAAAQAAAAAAAAAAAAAACgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAIAAAAAAAAABgAAAAAAAAAEAAAAAAAAAAoAAAAAAAAAAAAAAAAAAAAKAAAAAAAAAAIAAAAAAAAADAAAAAAAAAACwAAAAAAAAAAAAAABAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAADAn3a5sa80FgAAAAAAABhAAAAAABAAAABFcFR4TUo0MGZVdXM3YUdZAAAAAAsAAABub3QgaW4gbmV4dAAAAAAAEAAAAAwAFAASAAwACAAEAAwAAAAQAAAALAAAADgAAAAAAAQAAQAAAAADAAAAAAAAQAEAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAKAAwAAAAIAAQACgAAAAgAAACYAAAAAwAAAEwAAAAoAAAABAAAAKT9//8IAAAADAAAAAAAAAAAAAAABQAAAHJlZklkAAAAxP3//wgAAAAMAAAAAAAAAAAAAAAEAAAAbmFtZQAAAADk/f//CAAAADAAAAAkAAAAeyJjdXN0b20iOnsicmVzdWx0VHlwZSI6ImV4ZW1wbGFyIn19AAAAAAQAAABtZXRhAAAAAAQAAAC0AQAAuAAAAFgAAAAEAAAAbv7//xQAAAA4AAAAOAAAAAAAAAU0AAAAAQAAAAQAAABc/v//CAAAAAwAAAABAAAAYQAAAAQAAABuYW1lAAAAAAAAAACs////AQAAAGEAAAC+/v//FAAAADwAAABAAAAAAAAABTwAAAABAAAABAAAAKz+//8IAAAAEAAAAAcAAAB0cmFjZUlEAAQAAABuYW1lAAAAAAAAAAAEAAQABAAAAAcAAAB0cmFjZUlEABr///8UAAAAyAAAAMgAAAAAAAADyAAAAAIAAAAsAAAABAAAAAz///8IAAAAEAAAAAUAAABWYWx1ZQAAAAQAAABuYW1lAAAAADD///8IAAAAdAAAAGgAAAB7Il9fbmFtZV9fIjoidGVzdF9leGVtcGxhcl9tZXRyaWNfdG90YWwiLCJpbnN0YW5jZSI6ImxvY2FsaG9zdDo4MDkwIiwiam9iIjoicHJvbWV0aGV1cyIsInNlcnZpY2UiOiJiYXIifQAAAAAGAAAAbGFiZWxzAAAAAAAAiv///wAAAgAFAAAAVmFsdWUAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAATAAAAAAAAApMAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAQAAABUaW1lAAAAAAQAAABuYW1lAAAAAAAAAAAAAAYACAAGAAYAAAAAAAMABAAAAFRpbWUAAAAAGAMAAEFSUk9XMQ==
+FRAME=QVJST1cxAAD/////mAIAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAAJgAAAADAAAATAAAACgAAAAEAAAA+P3//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAAAY/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAADj+//8IAAAAMAAAACQAAAB7ImN1c3RvbSI6eyJyZXN1bHRUeXBlIjoiZXhlbXBsYXIifX0AAAAABAAAAG1ldGEAAAAAAwAAAGABAABkAAAABAAAAL7+//8UAAAAPAAAAEAAAAAAAAAFPAAAAAEAAAAEAAAArP7//wgAAAAQAAAABwAAAHRyYWNlSUQABAAAAG5hbWUAAAAAAAAAAAQABAAEAAAABwAAAHRyYWNlSUQAGv///xQAAADIAAAAyAAAAAAAAAPIAAAAAgAAACwAAAAEAAAADP///wgAAAAQAAAABQAAAFZhbHVlAAAABAAAAG5hbWUAAAAAMP///wgAAAB0AAAAaAAAAHsiX19uYW1lX18iOiJ0ZXN0X2V4ZW1wbGFyX21ldHJpY190b3RhbCIsImluc3RhbmNlIjoibG9jYWxob3N0OjgwOTAiLCJqb2IiOiJwcm9tZXRoZXVzIiwic2VydmljZSI6ImZvbyJ9AAAAAAYAAABsYWJlbHMAAAAAAACK////AAACAAUAAABWYWx1ZQASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABMAAAAAAAACkwAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAFRpbWUAAAAABAAAAG5hbWUAAAAAAAAAAAAABgAIAAYABgAAAAAAAwAEAAAAVGltZQAAAAD/////+AAAABQAAAAAAAAADAAWABQAEwAMAAQADAAAAFAAAAAAAAAAFAAAAAAAAAMEAAoAGAAMAAgABAAKAAAAFAAAAIgAAAACAAAAAAAAAAAAAAAHAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAwAAAAAAAAAMAAAAAAAAAAgAAAAAAAAAAAAAAADAAAAAgAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAwIOCDbSvNBZA/iZitq80FgAAAAAAADNAAAAAAAAANEAAAAAAEAAAACAAAAAAAAAAT2xwOVhIbHE3NjNjY3NmYWhDdGp5Z2tJSHdBTjl2czQQAAAADAAUABIADAAIAAQADAAAABAAAAAsAAAAPAAAAAAABAABAAAAqAIAAAAAAAAAAQAAAAAAAFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAwAAAAIAAQACgAAAAgAAACYAAAAAwAAAEwAAAAoAAAABAAAAPj9//8IAAAADAAAAAAAAAAAAAAABQAAAHJlZklkAAAAGP7//wgAAAAMAAAAAAAAAAAAAAAEAAAAbmFtZQAAAAA4/v//CAAAADAAAAAkAAAAeyJjdXN0b20iOnsicmVzdWx0VHlwZSI6ImV4ZW1wbGFyIn19AAAAAAQAAABtZXRhAAAAAAMAAABgAQAAZAAAAAQAAAC+/v//FAAAADwAAABAAAAAAAAABTwAAAABAAAABAAAAKz+//8IAAAAEAAAAAcAAAB0cmFjZUlEAAQAAABuYW1lAAAAAAAAAAAEAAQABAAAAAcAAAB0cmFjZUlEABr///8UAAAAyAAAAMgAAAAAAAADyAAAAAIAAAAsAAAABAAAAAz///8IAAAAEAAAAAUAAABWYWx1ZQAAAAQAAABuYW1lAAAAADD///8IAAAAdAAAAGgAAAB7Il9fbmFtZV9fIjoidGVzdF9leGVtcGxhcl9tZXRyaWNfdG90YWwiLCJpbnN0YW5jZSI6ImxvY2FsaG9zdDo4MDkwIiwiam9iIjoicHJvbWV0aGV1cyIsInNlcnZpY2UiOiJmb28ifQAAAAAGAAAAbGFiZWxzAAAAAAAAiv///wAAAgAFAAAAVmFsdWUAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAATAAAAAAAAApMAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAQAAABUaW1lAAAAAAQAAABuYW1lAAAAAAAAAAAAAAYACAAGAAYAAAAAAAMABAAAAFRpbWUAAAAAyAIAAEFSUk9XMQ==
@@ -0,0 +1,116 @@
+{
+  "frames": [
+    {
+      "schema": {
+        "meta": {
+          "custom": {
+            "resultType": "exemplar"
+          }
+        },
+        "fields": [
+          {
+            "name": "Time",
+            "type": "time",
+            "typeInfo": {
+              "frame": "time.Time"
+            }
+          },
+          {
+            "name": "Value",
+            "type": "number",
+            "typeInfo": {
+              "frame": "float64"
+            },
+            "labels": {
+              "__name__": "test_exemplar_metric_total",
+              "instance": "localhost:8090",
+              "job": "prometheus",
+              "service": "bar"
+            }
+          },
+          {
+            "name": "traceID",
+            "type": "string",
+            "typeInfo": {
+              "frame": "string"
+            }
+          },
+          {
+            "name": "a",
+            "type": "string",
+            "typeInfo": {
+              "frame": "string"
+            }
+          }
+        ]
+      },
+      "data": {
+        "values": [
+          [
+            1600096945479
+          ],
+          [
+            6
+          ],
+          [
+            "EpTxMJ40fUus7aGY"
+          ],
+          [
+            "not in next"
+          ]
+        ]
+      }
+    },
+    {
+      "schema": {
+        "meta": {
+          "custom": {
+            "resultType": "exemplar"
+          }
+        },
+        "fields": [
+          {
+            "name": "Time",
+            "type": "time",
+            "typeInfo": {
+              "frame": "time.Time"
+            }
+          },
+          {
+            "name": "Value",
+            "type": "number",
+            "typeInfo": {
+              "frame": "float64"
+            },
+            "labels": {
+              "__name__": "test_exemplar_metric_total",
+              "instance": "localhost:8090",
+              "job": "prometheus",
+              "service": "foo"
+            }
+          },
+          {
+            "name": "traceID",
+            "type": "string",
+            "typeInfo": {
+              "frame": "string"
+            }
+          }
+        ]
+      },
+      "data": {
+        "values": [
+          [
+            1600096955479,1600096965489
+          ],
+          [
+            19,20
+          ],
+          [
+            "Olp9XHlq763ccsfa","hCtjygkIHwAN9vs4"
+          ]
+        ]
+      }
+    }
+  ]
+}
@@ -0,0 +1,156 @@
+//  🌟 This was machine generated.  Do not edit. 🌟
+//  
+//  Frame[0] {
+//      "custom": {
+//          "resultType": "exemplar"
+//      }
+//  }
+//  Name: 
+//  Dimensions: 4 Fields by 1 Rows
+//  +-----------------------------------+---------------------------------------------------------------------------------------------------+------------------+----------------+
+//  | Name: Time                        | Name: Value                                                                                       | Name: traceID    | Name: a        |
+//  | Labels:                           | Labels: __name__=test_exemplar_metric_total, instance=localhost:8090, job=prometheus, service=bar | Labels:          | Labels:        |
+//  | Type: []time.Time                 | Type: []float64                                                                                   | Type: []string   | Type: []string |
+//  +-----------------------------------+---------------------------------------------------------------------------------------------------+------------------+----------------+
+//  | 2020-09-14 15:22:25.479 +0000 UTC | 6                                                                                                 | EpTxMJ40fUus7aGY | not in next    |
+//  +-----------------------------------+---------------------------------------------------------------------------------------------------+------------------+----------------+
+//  
+//  
+//  
+//  Frame[1] {
+//      "custom": {
+//          "resultType": "exemplar"
+//      }
+//  }
+//  Name: 
+//  Dimensions: 3 Fields by 2 Rows
+//  +-----------------------------------+---------------------------------------------------------------------------------------------------+------------------+
+//  | Name: Time                        | Name: Value                                                                                       | Name: traceID    |
+//  | Labels:                           | Labels: __name__=test_exemplar_metric_total, instance=localhost:8090, job=prometheus, service=foo | Labels:          |
+//  | Type: []time.Time                 | Type: []float64                                                                                   | Type: []string   |
+//  +-----------------------------------+---------------------------------------------------------------------------------------------------+------------------+
+//  | 2020-09-14 15:22:35.479 +0000 UTC | 19                                                                                                | Olp9XHlq763ccsfa |
+//  | 2020-09-14 15:22:45.489 +0000 UTC | 20                                                                                                | hCtjygkIHwAN9vs4 |
+//  +-----------------------------------+---------------------------------------------------------------------------------------------------+------------------+
+//  
+//  
+//  🌟 This was machine generated.  Do not edit. 🌟
+{
+  "frames": [
+    {
+      "schema": {
+        "meta": {
+          "custom": {
+            "resultType": "exemplar"
+          }
+        },
+        "fields": [
+          {
+            "name": "Time",
+            "type": "time",
+            "typeInfo": {
+              "frame": "time.Time"
+            }
+          },
+          {
+            "name": "Value",
+            "type": "number",
+            "typeInfo": {
+              "frame": "float64"
+            },
+            "labels": {
+              "__name__": "test_exemplar_metric_total",
+              "instance": "localhost:8090",
+              "job": "prometheus",
+              "service": "bar"
+            }
+          },
+          {
+            "name": "traceID",
+            "type": "string",
+            "typeInfo": {
+              "frame": "string"
+            }
+          },
+          {
+            "name": "a",
+            "type": "string",
+            "typeInfo": {
+              "frame": "string"
+            }
+          }
+        ]
+      },
+      "data": {
+        "values": [
+          [
+            1600096945479
+          ],
+          [
+            6
+          ],
+          [
+            "EpTxMJ40fUus7aGY"
+          ],
+          [
+            "not in next"
+          ]
+        ]
+      }
+    },
+    {
+      "schema": {
+        "meta": {
+          "custom": {
+            "resultType": "exemplar"
+          }
+        },
+        "fields": [
+          {
+            "name": "Time",
+            "type": "time",
+            "typeInfo": {
+              "frame": "time.Time"
+            }
+          },
+          {
+            "name": "Value",
+            "type": "number",
+            "typeInfo": {
+              "frame": "float64"
+            },
+            "labels": {
+              "__name__": "test_exemplar_metric_total",
+              "instance": "localhost:8090",
+              "job": "prometheus",
+              "service": "foo"
+            }
+          },
+          {
+            "name": "traceID",
+            "type": "string",
+            "typeInfo": {
+              "frame": "string"
+            }
+          }
+        ]
+      },
+      "data": {
+        "values": [
+          [
+            1600096955479,
+            1600096965489
+          ],
+          [
+            19,
+            20
+          ],
+          [
+            "Olp9XHlq763ccsfa",
+            "hCtjygkIHwAN9vs4"
+          ]
+        ]
+      }
+    }
+  ]
+}
@@ -0,0 +1,39 @@
+🌟 This was machine generated.  Do not edit. 🌟
+
+Frame[0] {
+    "custom": {
+        "resultType": "exemplar"
+    }
+}
+Name: 
+Dimensions: 4 Fields by 1 Rows
+-----------------------------------+---------------------------------------------------------------------------------------------------+------------------+----------------+
+| Name: Time                        | Name: Value                                                                                       | Name: traceID    | Name: a        |
+| Labels:                           | Labels: __name__=test_exemplar_metric_total, instance=localhost:8090, job=prometheus, service=bar | Labels:          | Labels:        |
+| Type: []time.Time                 | Type: []float64                                                                                   | Type: []string   | Type: []string |
+-----------------------------------+---------------------------------------------------------------------------------------------------+------------------+----------------+
+| 2020-09-14 15:22:25.479 +0000 UTC | 6                                                                                                 | EpTxMJ40fUus7aGY | not in next    |
+-----------------------------------+---------------------------------------------------------------------------------------------------+------------------+----------------+
+
+
+
+Frame[1] {
+    "custom": {
+        "resultType": "exemplar"
+    }
+}
+Name: 
+Dimensions: 3 Fields by 2 Rows
+-----------------------------------+---------------------------------------------------------------------------------------------------+------------------+
+| Name: Time                        | Name: Value                                                                                       | Name: traceID    |
+| Labels:                           | Labels: __name__=test_exemplar_metric_total, instance=localhost:8090, job=prometheus, service=foo | Labels:          |
+| Type: []time.Time                 | Type: []float64                                                                                   | Type: []string   |
+-----------------------------------+---------------------------------------------------------------------------------------------------+------------------+
+| 2020-09-14 15:22:35.479 +0000 UTC | 19                                                                                                | Olp9XHlq763ccsfa |
+| 2020-09-14 15:22:45.489 +0000 UTC | 20                                                                                                | hCtjygkIHwAN9vs4 |
+-----------------------------------+---------------------------------------------------------------------------------------------------+------------------+
+
+
+====== TEST DATA RESPONSE (arrow base64) ======
+FRAME=QVJST1cxAAD/////8AIAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAAJgAAAADAAAATAAAACgAAAAEAAAApP3//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAADE/f//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAOT9//8IAAAAMAAAACQAAAB7ImN1c3RvbSI6eyJyZXN1bHRUeXBlIjoiZXhlbXBsYXIifX0AAAAABAAAAG1ldGEAAAAABAAAALQBAAC4AAAAWAAAAAQAAABu/v//FAAAADgAAAA4AAAAAAAABTQAAAABAAAABAAAAFz+//8IAAAADAAAAAEAAABhAAAABAAAAG5hbWUAAAAAAAAAAKz///8BAAAAYQAAAL7+//8UAAAAPAAAAEAAAAAAAAAFPAAAAAEAAAAEAAAArP7//wgAAAAQAAAABwAAAHRyYWNlSUQABAAAAG5hbWUAAAAAAAAAAAQABAAEAAAABwAAAHRyYWNlSUQAGv///xQAAADIAAAAyAAAAAAAAAPIAAAAAgAAACwAAAAEAAAADP///wgAAAAQAAAABQAAAFZhbHVlAAAABAAAAG5hbWUAAAAAMP///wgAAAB0AAAAaAAAAHsiX19uYW1lX18iOiJ0ZXN0X2V4ZW1wbGFyX21ldHJpY190b3RhbCIsImluc3RhbmNlIjoibG9jYWxob3N0OjgwOTAiLCJqb2IiOiJwcm9tZXRoZXVzIiwic2VydmljZSI6ImJhciJ9AAAAAAYAAABsYWJlbHMAAAAAAACK////AAACAAUAAABWYWx1ZQASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABMAAAAAAAACkwAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAFRpbWUAAAAABAAAAG5hbWUAAAAAAAAAAAAABgAIAAYABgAAAAAAAwAEAAAAVGltZQAAAAAAAAAA/////zgBAAAUAAAAAAAAAAwAFgAUABMADAAEAAwAAABAAAAAAAAAABQAAAAAAAADBAAKABgADAAIAAQACgAAABQAAAC4AAAAAQAAAAAAAAAAAAAACgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAIAAAAAAAAABgAAAAAAAAAEAAAAAAAAAAoAAAAAAAAAAAAAAAAAAAAKAAAAAAAAAAIAAAAAAAAADAAAAAAAAAACwAAAAAAAAAAAAAABAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAADAn3a5sa80FgAAAAAAABhAAAAAABAAAABFcFR4TUo0MGZVdXM3YUdZAAAAAAsAAABub3QgaW4gbmV4dAAAAAAAEAAAAAwAFAASAAwACAAEAAwAAAAQAAAALAAAADgAAAAAAAQAAQAAAAADAAAAAAAAQAEAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAKAAwAAAAIAAQACgAAAAgAAACYAAAAAwAAAEwAAAAoAAAABAAAAKT9//8IAAAADAAAAAAAAAAAAAAABQAAAHJlZklkAAAAxP3//wgAAAAMAAAAAAAAAAAAAAAEAAAAbmFtZQAAAADk/f//CAAAADAAAAAkAAAAeyJjdXN0b20iOnsicmVzdWx0VHlwZSI6ImV4ZW1wbGFyIn19AAAAAAQAAABtZXRhAAAAAAQAAAC0AQAAuAAAAFgAAAAEAAAAbv7//xQAAAA4AAAAOAAAAAAAAAU0AAAAAQAAAAQAAABc/v//CAAAAAwAAAABAAAAYQAAAAQAAABuYW1lAAAAAAAAAACs////AQAAAGEAAAC+/v//FAAAADwAAABAAAAAAAAABTwAAAABAAAABAAAAKz+//8IAAAAEAAAAAcAAAB0cmFjZUlEAAQAAABuYW1lAAAAAAAAAAAEAAQABAAAAAcAAAB0cmFjZUlEABr///8UAAAAyAAAAMgAAAAAAAADyAAAAAIAAAAsAAAABAAAAAz///8IAAAAEAAAAAUAAABWYWx1ZQAAAAQAAABuYW1lAAAAADD///8IAAAAdAAAAGgAAAB7Il9fbmFtZV9fIjoidGVzdF9leGVtcGxhcl9tZXRyaWNfdG90YWwiLCJpbnN0YW5jZSI6ImxvY2FsaG9zdDo4MDkwIiwiam9iIjoicHJvbWV0aGV1cyIsInNlcnZpY2UiOiJiYXIifQAAAAAGAAAAbGFiZWxzAAAAAAAAiv///wAAAgAFAAAAVmFsdWUAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAATAAAAAAAAApMAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAQAAABUaW1lAAAAAAQAAABuYW1lAAAAAAAAAAAAAAYACAAGAAYAAAAAAAMABAAAAFRpbWUAAAAAGAMAAEFSUk9XMQ==
+FRAME=QVJST1cxAAD/////mAIAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAAJgAAAADAAAATAAAACgAAAAEAAAA+P3//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAAAY/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAADj+//8IAAAAMAAAACQAAAB7ImN1c3RvbSI6eyJyZXN1bHRUeXBlIjoiZXhlbXBsYXIifX0AAAAABAAAAG1ldGEAAAAAAwAAAGABAABkAAAABAAAAL7+//8UAAAAPAAAAEAAAAAAAAAFPAAAAAEAAAAEAAAArP7//wgAAAAQAAAABwAAAHRyYWNlSUQABAAAAG5hbWUAAAAAAAAAAAQABAAEAAAABwAAAHRyYWNlSUQAGv///xQAAADIAAAAyAAAAAAAAAPIAAAAAgAAACwAAAAEAAAADP///wgAAAAQAAAABQAAAFZhbHVlAAAABAAAAG5hbWUAAAAAMP///wgAAAB0AAAAaAAAAHsiX19uYW1lX18iOiJ0ZXN0X2V4ZW1wbGFyX21ldHJpY190b3RhbCIsImluc3RhbmNlIjoibG9jYWxob3N0OjgwOTAiLCJqb2IiOiJwcm9tZXRoZXVzIiwic2VydmljZSI6ImZvbyJ9AAAAAAYAAABsYWJlbHMAAAAAAACK////AAACAAUAAABWYWx1ZQASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABMAAAAAAAACkwAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAFRpbWUAAAAABAAAAG5hbWUAAAAAAAAAAAAABgAIAAYABgAAAAAAAwAEAAAAVGltZQAAAAD/////+AAAABQAAAAAAAAADAAWABQAEwAMAAQADAAAAFAAAAAAAAAAFAAAAAAAAAMEAAoAGAAMAAgABAAKAAAAFAAAAIgAAAACAAAAAAAAAAAAAAAHAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAwAAAAAAAAAMAAAAAAAAAAgAAAAAAAAAAAAAAADAAAAAgAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAwIOCDbSvNBZA/iZitq80FgAAAAAAADNAAAAAAAAANEAAAAAAEAAAACAAAAAAAAAAT2xwOVhIbHE3NjNjY3NmYWhDdGp5Z2tJSHdBTjl2czQQAAAADAAUABIADAAIAAQADAAAABAAAAAsAAAAPAAAAAAABAABAAAAqAIAAAAAAAAAAQAAAAAAAFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAwAAAAIAAQACgAAAAgAAACYAAAAAwAAAEwAAAAoAAAABAAAAPj9//8IAAAADAAAAAAAAAAAAAAABQAAAHJlZklkAAAAGP7//wgAAAAMAAAAAAAAAAAAAAAEAAAAbmFtZQAAAAA4/v//CAAAADAAAAAkAAAAeyJjdXN0b20iOnsicmVzdWx0VHlwZSI6ImV4ZW1wbGFyIn19AAAAAAQAAABtZXRhAAAAAAMAAABgAQAAZAAAAAQAAAC+/v//FAAAADwAAABAAAAAAAAABTwAAAABAAAABAAAAKz+//8IAAAAEAAAAAcAAAB0cmFjZUlEAAQAAABuYW1lAAAAAAAAAAAEAAQABAAAAAcAAAB0cmFjZUlEABr///8UAAAAyAAAAMgAAAAAAAADyAAAAAIAAAAsAAAABAAAAAz///8IAAAAEAAAAAUAAABWYWx1ZQAAAAQAAABuYW1lAAAAADD///8IAAAAdAAAAGgAAAB7Il9fbmFtZV9fIjoidGVzdF9leGVtcGxhcl9tZXRyaWNfdG90YWwiLCJpbnN0YW5jZSI6ImxvY2FsaG9zdDo4MDkwIiwiam9iIjoicHJvbWV0aGV1cyIsInNlcnZpY2UiOiJmb28ifQAAAAAGAAAAbGFiZWxzAAAAAAAAiv///wAAAgAFAAAAVmFsdWUAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAATAAAAAAAAApMAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAQAAABUaW1lAAAAAAQAAABuYW1lAAAAAAAAAAAAAAYACAAGAAYAAAAAAAMABAAAAFRpbWUAAAAAyAIAAEFSUk9XMQ==
@@ -0,0 +1,47 @@
+{
+    "status": "success",
+    "data": [
+        {
+            "seriesLabels": {
+                "__name__": "test_exemplar_metric_total",
+                "instance": "localhost:8090",
+                "job": "prometheus",
+                "service": "bar"
+            },
+            "exemplars": [
+                {
+                    "labels": {
+                        "traceID": "EpTxMJ40fUus7aGY",
+                        "a": "not in next"
+                    },
+                    "value": "6",
+                    "timestamp": 1600096945.479
+                }
+            ]
+        },
+        {
+            "seriesLabels": {
+                "__name__": "test_exemplar_metric_total",
+                "instance": "localhost:8090",
+                "job": "prometheus",
+                "service": "foo"
+            },
+            "exemplars": [
+                {
+                    "labels": {
+                        "traceID": "Olp9XHlq763ccsfa"
+                    },
+                    "value": "19",
+                    "timestamp": 1600096955.479
+                },
+                {
+                    "labels": {
+                        "traceID": "hCtjygkIHwAN9vs4"
+                    },
+                    "value": "20",
+                    "timestamp": 1600096965.489
+                }
+            ]
+        }
+    ]
+}