public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into encryption/use-secrets-service

Browse Source
This commit is contained in:
Joan López de la Franca Beltran
2021-10-22 16:50:33 +02:00
parent a26e36b711 0a97d0fff3
commit eb8729e622
10 changed files with 201 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/renovate.json5
+9
View File
@@ -17,6 +17,15 @@
"minor": {
"enabled": false
},
"packageRules": [
{
"matchPaths": ["grafana-toolkit/package.json"],
"ignoreDeps": [
"copy-webpack-plugin", // need to wait for Grafana 9 to upgrade toolkit to webpack 5
"css-loader", // need to wait for Grafana 9 to upgrade toolkit to webpack 5
]
}
],
"patch": {
"enabled": false
},
packaging/deb/systemd/grafana-server.service
-1
View File
@@ -33,7 +33,6 @@ MemoryDenyWriteExecute=false
NoNewPrivileges=true
PrivateDevices=true
PrivateTmp=true
PrivateUsers=true
ProtectClock=true
ProtectControlGroups=true
ProtectHome=true
packaging/rpm/systemd/grafana-server.service
-1
View File
@@ -32,7 +32,6 @@ MemoryDenyWriteExecute=false
NoNewPrivileges=true
PrivateDevices=true
PrivateTmp=true
PrivateUsers=true
ProtectClock=true
ProtectControlGroups=true
ProtectHome=true
pkg/tsdb/prometheus/prometheus.go
+85 -89
View File
@@ -161,9 +161,9 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest)
if err != nil {
plog.Error("Range query", query.Expr, "failed with", err)
result.Responses[query.RefId] = backend.DataResponse{Error: err}
continue
} else {
response[RangeQueryType] = rangeResponse
}
response[RangeQueryType] = rangeResponse
}
if query.InstantQuery {
@@ -171,9 +171,9 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest)
if err != nil {
plog.Error("Instant query", query.Expr, "failed with", err)
result.Responses[query.RefId] = backend.DataResponse{Error: err}
continue
} else {
response[InstantQueryType] = instantResponse
}
response[InstantQueryType] = instantResponse
}
if query.ExemplarQuery {
@@ -334,37 +334,32 @@ func (s *Service) parseQuery(queryContext *backend.QueryDataRequest, dsInfo *Dat
}
func parseResponse(value map[PrometheusQueryType]interface{}, query *PrometheusQuery) (data.Frames, error) {
frames := data.Frames{}
var (
frames = data.Frames{}
nextFrames = data.Frames{}
)
for _, value := range value {
matrix, ok := value.(model.Matrix)
if ok {
matrixFrames := matrixToDataFrames(matrix, query)
frames = append(frames, matrixFrames...)
// Zero out the slice to prevent data corruption.
nextFrames = nextFrames[:0]
switch v := value.(type) {
case model.Matrix:
nextFrames = matrixToDataFrames(v, query, nextFrames)
case model.Vector:
nextFrames = vectorToDataFrames(v, query, nextFrames)
case *model.Scalar:
nextFrames = scalarToDataFrames(v, query, nextFrames)
case []apiv1.ExemplarQueryResult:
nextFrames = exemplarToDataFrames(v, query, nextFrames)
default:
plog.Error("Query", query.Expr, "returned unexpected result type", v)
continue
}
vector, ok := value.(model.Vector)
if ok {
vectorFrames := vectorToDataFrames(vector, query)
frames = append(frames, vectorFrames...)
continue
}
scalar, ok := value.(*model.Scalar)
if ok {
scalarFrames := scalarToDataFrames(scalar, query)
frames = append(frames, scalarFrames...)
continue
}
exemplar, ok := value.([]apiv1.ExemplarQueryResult)
if ok {
exemplarFrames := exemplarToDataFrames(exemplar, query)
frames = append(frames, exemplarFrames...)
continue
}
frames = append(frames, nextFrames...)
}
return frames, nil
}
@@ -398,9 +393,7 @@ func calculateRateInterval(interval time.Duration, scrapeInterval string, interv
return rateInterval
}
func matrixToDataFrames(matrix model.Matrix, query *PrometheusQuery) data.Frames {
frames := data.Frames{}
func matrixToDataFrames(matrix model.Matrix, query *PrometheusQuery, frames data.Frames) data.Frames {
for _, v := range matrix {
tags := make(map[string]string, len(v.Metric))
for k, v := range v.Metric {
@@ -424,61 +417,58 @@ func matrixToDataFrames(matrix model.Matrix, query *PrometheusQuery) data.Frames
valueField.Config = &data.FieldConfig{DisplayNameFromDS: name}
valueField.Labels = tags
frame := data.NewFrame(name, timeField, valueField)
frame.Meta = &data.FrameMeta{
Custom: map[string]string{
"resultType": "matrix",
},
}
frames = append(frames, frame)
frames = append(frames, newDataFrame(name, "matrix", timeField, valueField))
}
return frames
}
func scalarToDataFrames(scalar *model.Scalar, query *PrometheusQuery) data.Frames {
func scalarToDataFrames(scalar *model.Scalar, query *PrometheusQuery, frames data.Frames) data.Frames {
timeVector := []time.Time{time.Unix(scalar.Timestamp.Unix(), 0).UTC()}
values := []float64{float64(scalar.Value)}
name := fmt.Sprintf("%g", values[0])
frame := data.NewFrame(name,
data.NewField("Time", nil, timeVector),
data.NewField("Value", nil, values).SetConfig(&data.FieldConfig{DisplayNameFromDS: name}))
frame.Meta = &data.FrameMeta{
Custom: map[string]string{
"resultType": "scalar",
},
}
frames := data.Frames{frame}
return frames
return append(
frames,
newDataFrame(
name,
"scalar",
data.NewField("Time", nil, timeVector),
data.NewField("Value", nil, values).SetConfig(&data.FieldConfig{DisplayNameFromDS: name}),
),
)
}
func vectorToDataFrames(vector model.Vector, query *PrometheusQuery) data.Frames {
frames := data.Frames{}
func vectorToDataFrames(vector model.Vector, query *PrometheusQuery, frames data.Frames) data.Frames {
for _, v := range vector {
name := formatLegend(v.Metric, query)
tags := make(map[string]string, len(v.Metric))
timeVector := []time.Time{time.Unix(v.Timestamp.Unix(), 0).UTC()}
values := []float64{float64(v.Value)}
for k, v := range v.Metric {
tags[string(k)] = string(v)
}
frame := data.NewFrame(name,
data.NewField("Time", nil, timeVector),
data.NewField("Value", tags, values).SetConfig(&data.FieldConfig{DisplayNameFromDS: name}))
frame.Meta = &data.FrameMeta{
Custom: map[string]string{
"resultType": "vector",
},
}
frames = append(frames, frame)
frames = append(
frames,
newDataFrame(
name,
"vector",
data.NewField("Time", nil, timeVector),
data.NewField("Value", tags, values).SetConfig(&data.FieldConfig{DisplayNameFromDS: name}),
),
)
}
return frames
}
func exemplarToDataFrames(response []apiv1.ExemplarQueryResult, query *PrometheusQuery) data.Frames {
frames := data.Frames{}
events := make([]ExemplarEvent, 0)
func exemplarToDataFrames(response []apiv1.ExemplarQueryResult, query *PrometheusQuery, frames data.Frames) data.Frames {
// TODO: this preallocation is very naive.
// We should figure out a better approximation here.
events := make([]ExemplarEvent, 0, len(response)*2)
for _, exemplarData := range response {
for _, exemplar := range exemplarData.Exemplars {
event := ExemplarEvent{}
@@ -486,9 +476,11 @@ func exemplarToDataFrames(response []apiv1.ExemplarQueryResult, query *Prometheu
event.Time = exemplarTime
event.Value = float64(exemplar.Value)
event.Labels = make(map[string]string)
for label, value := range exemplar.Labels {
event.Labels[string(label)] = string(value)
}
for seriesLabel, seriesValue := range exemplarData.SeriesLabels {
event.Labels[string(seriesLabel)] = string(seriesValue)
}
@@ -497,11 +489,11 @@ func exemplarToDataFrames(response []apiv1.ExemplarQueryResult, query *Prometheu
}
}
//Sampling of exemplars
// Sampling of exemplars
bucketedExemplars := make(map[string][]ExemplarEvent)
values := make([]float64, 0)
values := make([]float64, 0, len(events))
//Create bucketed exemplars based on aligned timestamp
// Create bucketed exemplars based on aligned timestamp
for _, event := range events {
alignedTs := fmt.Sprintf("%.0f", math.Floor(float64(event.Time.Unix())/query.Step.Seconds())*query.Step.Seconds())
_, ok := bucketedExemplars[alignedTs]
@@ -513,18 +505,18 @@ func exemplarToDataFrames(response []apiv1.ExemplarQueryResult, query *Prometheu
values = append(values, event.Value)
}
//Calculate standard deviation
// Calculate standard deviation
standardDeviation := deviation(values)
//Create slice with all of the bucketed exemplars
// Create slice with all of the bucketed exemplars
sampledBuckets := make([]string, len(bucketedExemplars))
for bucketTimes := range bucketedExemplars {
sampledBuckets = append(sampledBuckets, bucketTimes)
}
sort.Strings(sampledBuckets)
//Sample exemplars based ona value, so we are not showing too many of them
sampleExemplars := make([]ExemplarEvent, 0)
// Sample exemplars based ona value, so we are not showing too many of them
sampleExemplars := make([]ExemplarEvent, 0, len(sampledBuckets))
for _, bucket := range sampledBuckets {
exemplarsInBucket := bucketedExemplars[bucket]
if len(exemplarsInBucket) == 1 {
@@ -561,13 +553,15 @@ func exemplarToDataFrames(response []apiv1.ExemplarQueryResult, query *Prometheu
}
// Create DF from sampled exemplars
timeVector := make([]time.Time, 0, len(sampleExemplars))
valuesVector := make([]float64, 0, len(sampleExemplars))
timeField := data.NewFieldFromFieldType(data.FieldTypeTime, len(sampleExemplars))
timeField.Name = "Time"
valueField := data.NewFieldFromFieldType(data.FieldTypeFloat64, len(sampleExemplars))
valueField.Name = "Value"
labelsVector := make(map[string][]string, len(sampleExemplars))
for _, exemplar := range sampleExemplars {
timeVector = append(timeVector, exemplar.Time)
valuesVector = append(valuesVector, exemplar.Value)
for i, exemplar := range sampleExemplars {
timeField.Set(i, exemplar.Time)
valueField.Set(i, exemplar.Value)
for label, value := range exemplar.Labels {
if labelsVector[label] == nil {
@@ -578,22 +572,13 @@ func exemplarToDataFrames(response []apiv1.ExemplarQueryResult, query *Prometheu
}
}
frame := data.NewFrame("exemplar",
data.NewField("Time", nil, timeVector),
data.NewField("Value", nil, valuesVector))
dataFields := make([]*data.Field, 0, len(labelsVector)+2)
dataFields = append(dataFields, timeField, valueField)
for label, vector := range labelsVector {
frame.Fields = append(frame.Fields, data.NewField(label, nil, vector))
dataFields = append(dataFields, data.NewField(label, nil, vector))
}
frame.Meta = &data.FrameMeta{
Custom: map[string]PrometheusQueryType{
"resultType": "exemplar",
},
}
frames = append(frames, frame)
return frames
return append(frames, newDataFrame("exemplar", "exemplar", dataFields...))
}
func deviation(values []float64) float64 {
@@ -608,3 +593,14 @@ func deviation(values []float64) float64 {
}
return math.Sqrt(sd / (valuesLen - 1))
}
func newDataFrame(name string, typ string, fields ...*data.Field) *data.Frame {
frame := data.NewFrame(name, fields...)
frame.Meta = &data.FrameMeta{
Custom: map[string]string{
"resultType": typ,
},
}
return frame
}
public/app/features/explore/RichHistory/RichHistorySettings.tsx
-1
View File
@@ -22,7 +22,6 @@ export interface RichHistorySettingsProps {
const getStyles = stylesFactory((theme: GrafanaTheme) => {
return {
container: css`
padding-left: ${theme.spacing.sm};
font-size: ${theme.typography.size.sm};
.space-between {
margin-bottom: ${theme.spacing.lg};
public/app/plugins/datasource/prometheus/datasource.ts
+1 -1
View File
@@ -813,7 +813,7 @@ export class PrometheusDatasource extends DataSourceWithBackend<PromQuery, PromO
async areExemplarsAvailable() {
try {
const res = await this.metadataRequest('/api/v1/query_exemplars', { query: 'test' });
if (res.statusText === 'OK') {
if (res.data.status === 'success') {
return true;
}
return false;
public/app/plugins/datasource/tempo/datasource.test.ts
+14 -1
View File
@@ -209,7 +209,7 @@ const backendSrvWithPrometheus = {
if (uid === 'prom') {
return {
query() {
return of({ data: [totalsPromMetric, secondsPromMetric] });
return of({ data: [totalsPromMetric, secondsPromMetric, failedPromMetric] });
},
};
}
@@ -280,6 +280,19 @@ const secondsPromMetric = new MutableDataFrame({
],
});
const failedPromMetric = new MutableDataFrame({
refId: 'traces_service_graph_request_failed_total',
fields: [
{ name: 'Time', values: [1628169788000, 1628169788000] },
{ name: 'client', values: ['app', 'lb'] },
{ name: 'instance', values: ['127.0.0.1:12345', '127.0.0.1:12345'] },
{ name: 'job', values: ['local_scrape', 'local_scrape'] },
{ name: 'server', values: ['db', 'app'] },
{ name: 'tempo_config', values: ['default', 'default'] },
{ name: 'Value #traces_service_graph_request_failed_total', values: [2, 15] },
],
});
const mockInvalidJson = {
batches: [
{
public/app/plugins/datasource/tempo/datasource.ts
-3
View File
@@ -292,9 +292,6 @@ function serviceMapQuery(request: DataQueryRequest<TempoQuery>, datasourceUid: s
data: mapPromMetricsToServiceMap(responses, request.range),
state: LoadingState.Done,
};
}),
catchError((error) => {
return of({ error: { message: error.message }, data: [] });
})
);
}
public/app/plugins/datasource/tempo/graphTransform.test.ts
+22 -4
View File
@@ -64,16 +64,21 @@ describe('mapPromMetricsToServiceMap', () => {
from: dateTime('2000-01-01T00:00:00'),
to: dateTime('2000-01-01T00:01:00'),
};
const [nodes, edges] = mapPromMetricsToServiceMap([{ data: [totalsPromMetric, secondsPromMetric] }], {
...range,
raw: range,
});
const [nodes, edges] = mapPromMetricsToServiceMap(
[{ data: [totalsPromMetric, secondsPromMetric, failedPromMetric] }],
{
...range,
raw: range,
}
);
expect(nodes.fields).toMatchObject([
{ name: 'id', values: new ArrayVector(['db', 'app', 'lb']) },
{ name: 'title', values: new ArrayVector(['db', 'app', 'lb']) },
{ name: 'mainStat', values: new ArrayVector([1000, 2000, NaN]) },
{ name: 'secondaryStat', values: new ArrayVector([0.17, 0.33, NaN]) },
{ name: 'arc__success', values: new ArrayVector([0.8, 0.25, 1]) },
{ name: 'arc__failed', values: new ArrayVector([0.2, 0.75, 0]) },
]);
expect(edges.fields).toMatchObject([
{ name: 'id', values: new ArrayVector(['app_db', 'lb_app']) },
@@ -134,3 +139,16 @@ const secondsPromMetric = new MutableDataFrame({
{ name: 'Value #traces_service_graph_request_server_seconds_sum', values: [10, 40] },
],
});
const failedPromMetric = new MutableDataFrame({
refId: 'traces_service_graph_request_failed_total',
fields: [
{ name: 'Time', values: [1628169788000, 1628169788000] },
{ name: 'client', values: ['app', 'lb'] },
{ name: 'instance', values: ['127.0.0.1:12345', '127.0.0.1:12345'] },
{ name: 'job', values: ['local_scrape', 'local_scrape'] },
{ name: 'server', values: ['db', 'app'] },
{ name: 'tempo_config', values: ['default', 'default'] },
{ name: 'Value #traces_service_graph_request_failed_total', values: [2, 15] },
],
});
public/app/plugins/datasource/tempo/graphTransform.ts
+70 -26
View File
@@ -1,8 +1,8 @@
import { groupBy } from 'lodash';
import {
DataFrame,
DataFrameView,
DataQueryResponse,
FieldColorModeId,
FieldDTO,
MutableDataFrame,
NodeGraphDataFrameFieldNames as Fields,
@@ -10,6 +10,9 @@ import {
} from '@grafana/data';
import { getNonOverlappingDuration, getStats, makeFrames, makeSpanMap } from '../../../core/utils/tracing';
/**
* Row in a trace dataFrame
*/
interface Row {
traceID: string;
spanID: string;
@@ -129,10 +132,12 @@ function findTraceDuration(view: DataFrameView<Row>): number {
const secondsMetric = 'traces_service_graph_request_server_seconds_sum';
const totalsMetric = 'traces_service_graph_request_total';
const failedMetric = 'traces_service_graph_request_failed_total';
export const serviceMapMetrics = [
secondsMetric,
totalsMetric,
failedMetric,
// We don't show histogram in node graph at the moment but we could later add that into a node context menu.
// 'traces_service_graph_request_seconds_bucket',
// 'traces_service_graph_request_seconds_count',
@@ -147,14 +152,15 @@ export const serviceMapMetrics = [
* @param range
*/
export function mapPromMetricsToServiceMap(responses: DataQueryResponse[], range: TimeRange): [DataFrame, DataFrame] {
const [totalsDFView, secondsDFView] = getMetricFrames(responses);
const frames = getMetricFrames(responses);
// First just collect data from the metrics into a map with nodes and edges as keys
const nodesMap: Record<string, any> = {};
const edgesMap: Record<string, any> = {};
const nodesMap: Record<string, ServiceMapStatistics> = {};
const edgesMap: Record<string, EdgeObject> = {};
// At this moment we don't have any error/success or other counts so we just use these 2
collectMetricData(totalsDFView, 'total', totalsMetric, nodesMap, edgesMap);
collectMetricData(secondsDFView, 'seconds', secondsMetric, nodesMap, edgesMap);
collectMetricData(frames[totalsMetric], 'total', totalsMetric, nodesMap, edgesMap);
collectMetricData(frames[secondsMetric], 'seconds', secondsMetric, nodesMap, edgesMap);
collectMetricData(frames[failedMetric], 'failed', failedMetric, nodesMap, edgesMap);
return convertToDataFrames(nodesMap, edgesMap, range);
}
@@ -172,6 +178,14 @@ function createServiceMapDataFrames() {
name: Fields.secondaryStat,
config: { unit: 'r/sec', displayName: 'Requests per second' },
},
{
name: Fields.arc + 'success',
config: { displayName: 'Success', color: { fixedColor: 'green', mode: FieldColorModeId.Fixed } },
},
{
name: Fields.arc + 'failed',
config: { displayName: 'Failed', color: { fixedColor: 'red', mode: FieldColorModeId.Fixed } },
},
]);
const edges = createDF('Edges', [
{ name: Fields.id },
@@ -184,13 +198,29 @@ function createServiceMapDataFrames() {
return [nodes, edges];
}
function getMetricFrames(responses: DataQueryResponse[]) {
const responsesMap = groupBy(responses[0].data, (data) => data.refId);
const totalsDFView = new DataFrameView(responsesMap[totalsMetric][0]);
const secondsDFView = new DataFrameView(responsesMap[secondsMetric][0]);
return [totalsDFView, secondsDFView];
/**
* Group frames from response based on ref id which is set the same as the metric name so we know which metric is where
* and also put it into DataFrameView so it's easier to work with.
* @param responses
*/
function getMetricFrames(responses: DataQueryResponse[]): Record<string, DataFrameView> {
return responses[0].data.reduce<Record<string, DataFrameView>>((acc, frame) => {
acc[frame.refId] = new DataFrameView(frame);
return acc;
}, {});
}
type ServiceMapStatistics = {
total?: number;
seconds?: number;
failed?: number;
};
type EdgeObject = ServiceMapStatistics & {
source: string;
target: string;
};
/**
* Collect data from a metric into a map of nodes and edges. The metric data is modeled as counts of metric per edge
* which is a pair of client-server nodes. This means we convert each row of the metric 1-1 to edges and than we assign
@@ -203,12 +233,16 @@ function getMetricFrames(responses: DataQueryResponse[]) {
* @param edgesMap
*/
function collectMetricData(
frame: DataFrameView,
stat: 'total' | 'seconds',
frame: DataFrameView | undefined,
stat: keyof ServiceMapStatistics,
metric: string,
nodesMap: Record<string, any>,
edgesMap: Record<string, any>
nodesMap: Record<string, ServiceMapStatistics>,
edgesMap: Record<string, EdgeObject>
) {
if (!frame) {
return;
}
// The name of the value column is in this format
// TODO figure out if it can be changed
const valueName = `Value #${metric}`;
@@ -218,24 +252,32 @@ function collectMetricData(
const edgeId = `${row.client}_${row.server}`;
if (!edgesMap[edgeId]) {
// Create edge as it does not exist yet
edgesMap[edgeId] = {
target: row.server,
source: row.client,
[stat]: row[valueName],
};
} else {
// Add stat to edge
// We are adding the values if exists but that should not happen in general as there should be single row for
// an edge.
edgesMap[edgeId][stat] = (edgesMap[edgeId][stat] || 0) + row[valueName];
}
if (!nodesMap[row.server]) {
// Create node for server
nodesMap[row.server] = {
[stat]: row[valueName],
};
} else {
// Add stat to server node. Sum up values if there are multiple edges targeting this server node.
nodesMap[row.server][stat] = (nodesMap[row.server][stat] || 0) + row[valueName];
}
if (!nodesMap[row.client]) {
// Create the client node but don't add the stat as edge stats are attributed to the server node. This means for
// example that the number of requests in a node show how many requests it handled not how many it generated.
nodesMap[row.client] = {
[stat]: 0,
};
@@ -244,8 +286,8 @@ function collectMetricData(
}
function convertToDataFrames(
nodesMap: Record<string, any>,
edgesMap: Record<string, any>,
nodesMap: Record<string, ServiceMapStatistics>,
edgesMap: Record<string, EdgeObject>,
range: TimeRange
): [DataFrame, DataFrame] {
const rangeMs = range.to.valueOf() - range.from.valueOf();
@@ -253,22 +295,24 @@ function convertToDataFrames(
for (const nodeId of Object.keys(nodesMap)) {
const node = nodesMap[nodeId];
nodes.add({
id: nodeId,
title: nodeId,
[Fields.id]: nodeId,
[Fields.title]: nodeId,
// NaN will not be shown in the node graph. This happens for a root client node which did not process
// any requests itself.
mainStat: node.total ? (node.seconds / node.total) * 1000 : Number.NaN, // Average response time
secondaryStat: node.total ? Math.round((node.total / (rangeMs / 1000)) * 100) / 100 : Number.NaN, // Request per second (to 2 decimals)
[Fields.mainStat]: node.total ? (node.seconds! / node.total) * 1000 : Number.NaN, // Average response time
[Fields.secondaryStat]: node.total ? Math.round((node.total / (rangeMs / 1000)) * 100) / 100 : Number.NaN, // Request per second (to 2 decimals)
[Fields.arc + 'success']: node.total ? (node.total - (node.failed || 0)) / node.total : 1,
[Fields.arc + 'failed']: node.total ? (node.failed || 0) / node.total : 0,
});
}
for (const edgeId of Object.keys(edgesMap)) {
const edge = edgesMap[edgeId];
edges.add({
id: edgeId,
source: edge.source,
target: edge.target,
mainStat: edge.total, // Requests
secondaryStat: edge.total ? (edge.seconds / edge.total) * 1000 : Number.NaN, // Average response time
[Fields.id]: edgeId,
[Fields.source]: edge.source,
[Fields.target]: edge.target,
[Fields.mainStat]: edge.total, // Requests
[Fields.secondaryStat]: edge.total ? (edge.seconds! / edge.total) * 1000 : Number.NaN, // Average response time
});
}