Loki: Add support for range aggregations with by grouping (#56184) (#56369)

* Loki: Fix unwrapped range operations with grouping

* Update, fix tests

* Fix modeller

(cherry picked from commit 4469572b27)

Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com>
This commit is contained in:
Grot (@grafanabot)
2022-10-05 08:54:03 -04:00
committed by GitHub
co-authored by Ivana Huckova
parent 4d3408f4f0
commit 55186418b9
7 changed files with 428 additions and 202 deletions
@@ -155,7 +155,7 @@ export class LokiQueryModeller extends LokiAndPromQueryModellerBase {
{ id: LokiOperationId.LabelFilterNoErrors, params: [] },
{ id: LokiOperationId.Unwrap, params: ['latency'] },
{ id: LokiOperationId.LabelFilterNoErrors, params: [] },
{ id: LokiOperationId.QuantileOverTime, params: [0.5, '$__interval'] },
{ id: LokiOperationId.QuantileOverTime, params: ['$__interval', 0.5] },
{ id: LokiOperationId.Sum, params: [] },
],
},
@@ -0,0 +1,124 @@
import { createRangeOperation, createRangeOperationWithGrouping } from './operationUtils';
import { LokiVisualQueryOperationCategory } from './types';
describe('createRangeOperation', () => {
it('should create basic range operation without possible grouping', () => {
expect(createRangeOperation('test_range_operation')).toMatchObject({
id: 'test_range_operation',
name: 'Test range operation',
params: [{ name: 'Range', type: 'string' }],
defaultParams: ['$__interval'],
alternativesKey: 'range function',
category: LokiVisualQueryOperationCategory.RangeFunctions,
});
});
it('should create basic range operation with possible grouping', () => {
expect(createRangeOperation('test_range_operation', true)).toMatchObject({
id: 'test_range_operation',
name: 'Test range operation',
params: [
{ name: 'Range', type: 'string' },
{
name: 'By label',
type: 'string',
restParam: true,
optional: true,
},
],
defaultParams: ['$__interval'],
alternativesKey: 'range function',
category: LokiVisualQueryOperationCategory.RangeFunctions,
});
});
it('should create range operation for quantile_over_time', () => {
expect(createRangeOperation('quantile_over_time', true)).toMatchObject({
id: 'quantile_over_time',
name: 'Quantile over time',
params: [
{ name: 'Range', type: 'string' },
{ name: 'Quantile', type: 'number' },
{ name: 'By label', type: 'string', restParam: true, optional: true },
],
defaultParams: ['$__interval', '0.95'],
alternativesKey: 'range function',
category: LokiVisualQueryOperationCategory.RangeFunctions,
});
});
});
describe('createRangeOperationWithGrouping', () => {
it('returns correct operation definitions with overrides and params', () => {
const operations = createRangeOperationWithGrouping('quantile_over_time');
expect(operations).toHaveLength(3);
expect(operations[0]).toMatchObject({
id: 'quantile_over_time',
name: 'Quantile over time',
params: [
{ name: 'Range', type: 'string' },
{ name: 'Quantile', type: 'number' },
{ name: 'By label', type: 'string', restParam: true, optional: true },
],
defaultParams: ['$__interval', '0.95'],
alternativesKey: 'range function',
category: LokiVisualQueryOperationCategory.RangeFunctions,
});
expect(operations[1]).toMatchObject({
id: '__quantile_over_time_by',
name: 'Quantile over time by',
params: [
{ name: 'Range', type: 'string' },
{ name: 'Quantile', type: 'number' },
{ name: 'Label', type: 'string', restParam: true, optional: true },
],
defaultParams: ['$__interval', '0.95', ''],
alternativesKey: 'range function with grouping',
category: LokiVisualQueryOperationCategory.RangeFunctions,
});
expect(operations[2]).toMatchObject({
id: '__quantile_over_time_without',
name: 'Quantile over time without',
params: [
{ name: 'Range', type: 'string' },
{ name: 'Quantile', type: 'number' },
{ name: 'Label', type: 'string', restParam: true, optional: true },
],
defaultParams: ['$__interval', '0.95', ''],
alternativesKey: 'range function with grouping',
category: LokiVisualQueryOperationCategory.RangeFunctions,
});
});
it('returns correct query string using range operation definitions for quantile_over_time with by grouping', () => {
const operations = createRangeOperationWithGrouping('quantile_over_time');
const query = operations[1].renderer(
{ id: '__quantile_over_time_by', params: ['[5m]', '0.95', 'source', 'place'] },
operations[1],
'{job="grafana"}'
);
expect(query).toBe('quantile_over_time(0.95, {job="grafana"} [[5m]]) by (source, place)');
});
it('returns correct query string using range operation definitions for quantile_over_time with without grouping', () => {
const operations = createRangeOperationWithGrouping('quantile_over_time');
const query = operations[2].renderer(
{ id: '__quantile_over_time_without', params: ['[$__interval]', '0.91', 'source', 'place'] },
operations[2],
'{job="grafana"}'
);
expect(query).toBe('quantile_over_time(0.91, {job="grafana"} [[$__interval]]) without (source, place)');
});
it('returns correct query string using range operation definitions for avg_over_time with without grouping', () => {
const operations = createRangeOperationWithGrouping('avg_over_time');
const query = operations[2].renderer(
{ id: '__avg_over_time_without', params: ['[$__interval]', 'source'] },
operations[2],
'{job="grafana"}'
);
expect(query).toBe('avg_over_time({job="grafana"} [[$__interval]]) without (source)');
});
});
@@ -0,0 +1,270 @@
import { LabelParamEditor } from '../../prometheus/querybuilder/components/LabelParamEditor';
import {
getAggregationExplainer,
getLastLabelRemovedHandler,
getOnLabelAddedHandler,
getPromAndLokiOperationDisplayName,
} from '../../prometheus/querybuilder/shared/operationUtils';
import {
QueryBuilderOperation,
QueryBuilderOperationDef,
QueryBuilderOperationParamDef,
VisualQueryModeller,
} from '../../prometheus/querybuilder/shared/types';
import { FUNCTIONS } from '../syntax';
import { LokiOperationId, LokiOperationOrder, LokiVisualQuery, LokiVisualQueryOperationCategory } from './types';
export function createRangeOperation(name: string, isRangeOperationWithGrouping?: boolean): QueryBuilderOperationDef {
const params = [getRangeVectorParamDef()];
const defaultParams = ['$__interval'];
let paramChangedHandler = undefined;
if (name === LokiOperationId.QuantileOverTime) {
defaultParams.push('0.95');
params.push({
name: 'Quantile',
type: 'number',
});
}
if (isRangeOperationWithGrouping) {
params.push({
name: 'By label',
type: 'string',
restParam: true,
optional: true,
});
paramChangedHandler = getOnLabelAddedHandler(`__${name}_by`);
}
return {
id: name,
name: getPromAndLokiOperationDisplayName(name),
params: params,
defaultParams,
alternativesKey: 'range function',
category: LokiVisualQueryOperationCategory.RangeFunctions,
orderRank: LokiOperationOrder.RangeVectorFunction,
renderer: operationWithRangeVectorRenderer,
addOperationHandler: addLokiOperation,
paramChangedHandler,
explainHandler: (op, def) => {
let opDocs = FUNCTIONS.find((x) => x.insertText === op.id)?.documentation ?? '';
if (op.params[0] === '$__interval') {
return `${opDocs} \`$__interval\` is variable that will be replaced with a calculated interval based on **Max data points**, **Min interval** and query time range. You find these options you find under **Query options** at the right of the data source select dropdown.`;
} else {
return `${opDocs} The [range vector](https://grafana.com/docs/loki/latest/logql/metric_queries/#range-vector-aggregation) is set to \`${op.params[0]}\`.`;
}
},
};
}
export function createRangeOperationWithGrouping(name: string): QueryBuilderOperationDef[] {
const rangeOperation = createRangeOperation(name, true);
// Copy range operation params without the last param
const params = rangeOperation.params.slice(0, -1);
const operations: QueryBuilderOperationDef[] = [
rangeOperation,
{
id: `__${name}_by`,
name: `${getPromAndLokiOperationDisplayName(name)} by`,
params: [
...params,
{
name: 'Label',
type: 'string',
restParam: true,
optional: true,
editor: LabelParamEditor,
},
],
defaultParams: [...rangeOperation.defaultParams, ''],
alternativesKey: 'range function with grouping',
category: LokiVisualQueryOperationCategory.RangeFunctions,
renderer: getRangeAggregationWithGroupingRenderer(name, 'by'),
paramChangedHandler: getLastLabelRemovedHandler(name),
explainHandler: getAggregationExplainer(name, 'by'),
addOperationHandler: addLokiOperation,
hideFromList: true,
},
{
id: `__${name}_without`,
name: `${getPromAndLokiOperationDisplayName(name)} without`,
params: [
...params,
{
name: 'Label',
type: 'string',
restParam: true,
optional: true,
editor: LabelParamEditor,
},
],
defaultParams: [...rangeOperation.defaultParams, ''],
alternativesKey: 'range function with grouping',
category: LokiVisualQueryOperationCategory.RangeFunctions,
renderer: getRangeAggregationWithGroupingRenderer(name, 'without'),
paramChangedHandler: getLastLabelRemovedHandler(name),
explainHandler: getAggregationExplainer(name, 'without'),
addOperationHandler: addLokiOperation,
hideFromList: true,
},
];
return operations;
}
export function getRangeAggregationWithGroupingRenderer(aggregation: string, grouping: 'by' | 'without') {
return function aggregationRenderer(model: QueryBuilderOperation, def: QueryBuilderOperationDef, innerExpr: string) {
const restParamIndex = def.params.findIndex((param) => param.restParam);
const params = model.params.slice(0, restParamIndex);
const restParams = model.params.slice(restParamIndex);
if (params.length === 2 && aggregation === LokiOperationId.QuantileOverTime) {
return `${aggregation}(${params[1]}, ${innerExpr} [${params[0]}]) ${grouping} (${restParams.join(', ')})`;
}
return `${aggregation}(${innerExpr} [${params[0]}]) ${grouping} (${restParams.join(', ')})`;
};
}
function operationWithRangeVectorRenderer(
model: QueryBuilderOperation,
def: QueryBuilderOperationDef,
innerExpr: string
) {
const params = model.params ?? [];
const rangeVector = params[0] ?? '$__interval';
// QuantileOverTime is only range vector with more than one param
if (params.length === 2 && model.id === LokiOperationId.QuantileOverTime) {
const quantile = params[1];
return `${model.id}(${quantile}, ${innerExpr} [${rangeVector}])`;
}
return `${model.id}(${innerExpr} [${params[0] ?? '$__interval'}])`;
}
export function labelFilterRenderer(model: QueryBuilderOperation, def: QueryBuilderOperationDef, innerExpr: string) {
if (model.params[0] === '') {
return innerExpr;
}
if (model.params[1] === '<' || model.params[1] === '>') {
return `${innerExpr} | ${model.params[0]} ${model.params[1]} ${model.params[2]}`;
}
return `${innerExpr} | ${model.params[0]} ${model.params[1]} \`${model.params[2]}\``;
}
export function pipelineRenderer(model: QueryBuilderOperation, def: QueryBuilderOperationDef, innerExpr: string) {
return `${innerExpr} | ${model.id}`;
}
function isRangeVectorFunction(def: QueryBuilderOperationDef) {
return def.category === LokiVisualQueryOperationCategory.RangeFunctions;
}
function getIndexOfOrLast(
operations: QueryBuilderOperation[],
queryModeller: VisualQueryModeller,
condition: (def: QueryBuilderOperationDef) => boolean
) {
const index = operations.findIndex((x) => {
const opDef = queryModeller.getOperationDef(x.id);
if (!opDef) {
return false;
}
return condition(opDef);
});
return index === -1 ? operations.length : index;
}
export function addLokiOperation(
def: QueryBuilderOperationDef,
query: LokiVisualQuery,
modeller: VisualQueryModeller
): LokiVisualQuery {
const newOperation: QueryBuilderOperation = {
id: def.id,
params: def.defaultParams,
};
const operations = [...query.operations];
const existingRangeVectorFunction = operations.find((x) => {
const opDef = modeller.getOperationDef(x.id);
if (!opDef) {
return false;
}
return isRangeVectorFunction(opDef);
});
switch (def.category) {
case LokiVisualQueryOperationCategory.Aggregations:
case LokiVisualQueryOperationCategory.Functions:
// If we are adding a function but we have not range vector function yet add one
if (!existingRangeVectorFunction) {
const placeToInsert = getIndexOfOrLast(
operations,
modeller,
(def) => def.category === LokiVisualQueryOperationCategory.Functions
);
operations.splice(placeToInsert, 0, { id: LokiOperationId.Rate, params: ['$__interval'] });
}
operations.push(newOperation);
break;
case LokiVisualQueryOperationCategory.RangeFunctions:
// If adding a range function and range function is already added replace it
if (existingRangeVectorFunction) {
const index = operations.indexOf(existingRangeVectorFunction);
operations[index] = newOperation;
break;
}
// Add range functions after any formats, line filters and label filters
default:
const placeToInsert = getIndexOfOrLast(
operations,
modeller,
(x) => (def.orderRank ?? 100) < (x.orderRank ?? 100)
);
operations.splice(placeToInsert, 0, newOperation);
break;
}
return {
...query,
operations,
};
}
export function addNestedQueryHandler(def: QueryBuilderOperationDef, query: LokiVisualQuery): LokiVisualQuery {
return {
...query,
binaryQueries: [
...(query.binaryQueries ?? []),
{
operator: '/',
query,
},
],
};
}
export function getLineFilterRenderer(operation: string) {
return function lineFilterRenderer(model: QueryBuilderOperation, def: QueryBuilderOperationDef, innerExpr: string) {
return `${innerExpr} ${operation} \`${model.params[0]}\``;
};
}
function getRangeVectorParamDef(): QueryBuilderOperationParamDef {
return {
name: 'Range',
type: 'string',
options: ['$__interval', '$__range', '1m', '5m', '10m', '1h', '24h'],
};
}
@@ -1,19 +1,21 @@
import {
createAggregationOperation,
createAggregationOperationWithParam,
getPromAndLokiOperationDisplayName,
} from '../../prometheus/querybuilder/shared/operationUtils';
import {
QueryBuilderOperation,
QueryBuilderOperationDef,
QueryBuilderOperationParamDef,
VisualQueryModeller,
} from '../../prometheus/querybuilder/shared/types';
import { FUNCTIONS } from '../syntax';
import { QueryBuilderOperationDef } from '../../prometheus/querybuilder/shared/types';
import { binaryScalarOperations } from './binaryScalarOperations';
import { UnwrapParamEditor } from './components/UnwrapParamEditor';
import { LokiOperationId, LokiOperationOrder, LokiVisualQuery, LokiVisualQueryOperationCategory } from './types';
import {
addLokiOperation,
addNestedQueryHandler,
createRangeOperation,
createRangeOperationWithGrouping,
getLineFilterRenderer,
labelFilterRenderer,
pipelineRenderer,
} from './operationUtils';
import { LokiOperationId, LokiOperationOrder, LokiVisualQueryOperationCategory } from './types';
export function getOperationDefinitions(): QueryBuilderOperationDef[] {
const aggregations = [
@@ -45,23 +47,31 @@ export function getOperationDefinitions(): QueryBuilderOperationDef[] {
);
});
const list: QueryBuilderOperationDef[] = [
const rangeOperations = [
createRangeOperation(LokiOperationId.Rate),
createRangeOperation(LokiOperationId.CountOverTime),
createRangeOperation(LokiOperationId.SumOverTime),
createRangeOperation(LokiOperationId.BytesRate),
createRangeOperation(LokiOperationId.BytesOverTime),
createRangeOperation(LokiOperationId.AbsentOverTime),
createRangeOperation(LokiOperationId.AvgOverTime),
createRangeOperation(LokiOperationId.MaxOverTime),
createRangeOperation(LokiOperationId.MinOverTime),
createRangeOperation(LokiOperationId.FirstOverTime),
createRangeOperation(LokiOperationId.LastOverTime),
createRangeOperation(LokiOperationId.StdvarOverTime),
createRangeOperation(LokiOperationId.StddevOverTime),
createRangeOperation(LokiOperationId.QuantileOverTime),
];
const rangeOperationsWithGrouping = [
...createRangeOperationWithGrouping(LokiOperationId.AvgOverTime),
...createRangeOperationWithGrouping(LokiOperationId.MaxOverTime),
...createRangeOperationWithGrouping(LokiOperationId.MinOverTime),
...createRangeOperationWithGrouping(LokiOperationId.FirstOverTime),
...createRangeOperationWithGrouping(LokiOperationId.LastOverTime),
...createRangeOperationWithGrouping(LokiOperationId.StdvarOverTime),
...createRangeOperationWithGrouping(LokiOperationId.StddevOverTime),
...createRangeOperationWithGrouping(LokiOperationId.QuantileOverTime),
];
const list: QueryBuilderOperationDef[] = [
...aggregations,
...aggregationsWithParam,
...rangeOperations,
...rangeOperationsWithGrouping,
{
id: LokiOperationId.Json,
name: 'Json',
@@ -409,181 +419,3 @@ export function getOperationDefinitions(): QueryBuilderOperationDef[] {
return list;
}
function createRangeOperation(name: string): QueryBuilderOperationDef {
const params = [getRangeVectorParamDef()];
const defaultParams = ['$__interval'];
let renderer = operationWithRangeVectorRenderer;
if (name === LokiOperationId.QuantileOverTime) {
defaultParams.push('0.95');
params.push({
name: 'Quantile',
type: 'number',
});
renderer = operationWithRangeVectorRendererAndParam;
}
return {
id: name,
name: getPromAndLokiOperationDisplayName(name),
params,
defaultParams,
alternativesKey: 'range function',
category: LokiVisualQueryOperationCategory.RangeFunctions,
orderRank: LokiOperationOrder.RangeVectorFunction,
renderer,
addOperationHandler: addLokiOperation,
explainHandler: (op, def) => {
let opDocs = FUNCTIONS.find((x) => x.insertText === op.id)?.documentation ?? '';
if (op.params[0] === '$__interval') {
return `${opDocs} \`$__interval\` is variable that will be replaced with a calculated interval based on **Max data points**, **Min interval** and query time range. You find these options you find under **Query options** at the right of the data source select dropdown.`;
} else {
return `${opDocs} The [range vector](https://grafana.com/docs/loki/latest/logql/metric_queries/#range-vector-aggregation) is set to \`${op.params[0]}\`.`;
}
},
};
}
function getRangeVectorParamDef(): QueryBuilderOperationParamDef {
return {
name: 'Range',
type: 'string',
options: ['$__interval', '$__range', '1m', '5m', '10m', '1h', '24h'],
};
}
function operationWithRangeVectorRenderer(
model: QueryBuilderOperation,
def: QueryBuilderOperationDef,
innerExpr: string
) {
let rangeVector = (model.params ?? [])[0] ?? '$__interval';
return `${def.id}(${innerExpr} [${rangeVector}])`;
}
function operationWithRangeVectorRendererAndParam(
model: QueryBuilderOperation,
def: QueryBuilderOperationDef,
innerExpr: string
) {
const params = model.params ?? [];
const rangeVector = params[0] ?? '$__interval';
const param = params[1];
return `${def.id}(${param}, ${innerExpr} [${rangeVector}])`;
}
function getLineFilterRenderer(operation: string) {
return function lineFilterRenderer(model: QueryBuilderOperation, def: QueryBuilderOperationDef, innerExpr: string) {
return `${innerExpr} ${operation} \`${model.params[0]}\``;
};
}
function labelFilterRenderer(model: QueryBuilderOperation, def: QueryBuilderOperationDef, innerExpr: string) {
if (model.params[0] === '') {
return innerExpr;
}
if (model.params[1] === '<' || model.params[1] === '>') {
return `${innerExpr} | ${model.params[0]} ${model.params[1]} ${model.params[2]}`;
}
return `${innerExpr} | ${model.params[0]} ${model.params[1]} \`${model.params[2]}\``;
}
function pipelineRenderer(model: QueryBuilderOperation, def: QueryBuilderOperationDef, innerExpr: string) {
return `${innerExpr} | ${model.id}`;
}
function isRangeVectorFunction(def: QueryBuilderOperationDef) {
return def.category === LokiVisualQueryOperationCategory.RangeFunctions;
}
function getIndexOfOrLast(
operations: QueryBuilderOperation[],
queryModeller: VisualQueryModeller,
condition: (def: QueryBuilderOperationDef) => boolean
) {
const index = operations.findIndex((x) => {
const opDef = queryModeller.getOperationDef(x.id);
if (!opDef) {
return false;
}
return condition(opDef);
});
return index === -1 ? operations.length : index;
}
export function addLokiOperation(
def: QueryBuilderOperationDef,
query: LokiVisualQuery,
modeller: VisualQueryModeller
): LokiVisualQuery {
const newOperation: QueryBuilderOperation = {
id: def.id,
params: def.defaultParams,
};
const operations = [...query.operations];
const existingRangeVectorFunction = operations.find((x) => {
const opDef = modeller.getOperationDef(x.id);
if (!opDef) {
return false;
}
return isRangeVectorFunction(opDef);
});
switch (def.category) {
case LokiVisualQueryOperationCategory.Aggregations:
case LokiVisualQueryOperationCategory.Functions:
// If we are adding a function but we have not range vector function yet add one
if (!existingRangeVectorFunction) {
const placeToInsert = getIndexOfOrLast(
operations,
modeller,
(def) => def.category === LokiVisualQueryOperationCategory.Functions
);
operations.splice(placeToInsert, 0, { id: LokiOperationId.Rate, params: ['$__interval'] });
}
operations.push(newOperation);
break;
case LokiVisualQueryOperationCategory.RangeFunctions:
// If adding a range function and range function is already added replace it
if (existingRangeVectorFunction) {
const index = operations.indexOf(existingRangeVectorFunction);
operations[index] = newOperation;
break;
}
// Add range functions after any formats, line filters and label filters
default:
const placeToInsert = getIndexOfOrLast(
operations,
modeller,
(x) => (def.orderRank ?? 100) < (x.orderRank ?? 100)
);
operations.splice(placeToInsert, 0, newOperation);
break;
}
return {
...query,
operations,
};
}
function addNestedQueryHandler(def: QueryBuilderOperationDef, query: LokiVisualQuery): LokiVisualQuery {
return {
...query,
binaryQueries: [
...(query.binaryQueries ?? []),
{
operator: '/',
query,
},
],
};
}
@@ -457,7 +457,7 @@ describe('buildVisualQueryFromString', () => {
label: 'app',
},
],
operations: [{ id: LokiOperationId.QuantileOverTime, params: ['0.99', '1m'] }],
operations: [{ id: LokiOperationId.QuantileOverTime, params: ['1m', '0.99'] }],
})
);
});
@@ -404,7 +404,7 @@ function handleRangeAggregation(expr: string, node: SyntaxNode, context: Context
let match = getString(expr, node).match(/\[(.+)\]/);
if (match?.[1]) {
params.push(match[1]);
params.unshift(match[1]);
}
const op = {
@@ -261,7 +261,7 @@ function getAggregationWithoutRenderer(aggregation: string) {
/**
* Very simple poc implementation, needs to be modified to support all aggregation operators
*/
function getAggregationExplainer(aggregationName: string, mode: 'by' | 'without' | '') {
export function getAggregationExplainer(aggregationName: string, mode: 'by' | 'without' | '') {
return function aggregationExplainer(model: QueryBuilderOperation) {
const labels = model.params.map((label) => `\`${label}\``).join(' and ');
const labelWord = pluralize('label', model.params.length);
@@ -292,7 +292,7 @@ function getAggregationByRendererWithParameter(aggregation: string) {
/**
* This function will transform operations without labels to their plan aggregation operation
*/
function getLastLabelRemovedHandler(changeToOperationId: string) {
export function getLastLabelRemovedHandler(changeToOperationId: string) {
return function onParamChanged(index: number, op: QueryBuilderOperation, def: QueryBuilderOperationDef) {
// If definition has more params then is defined there are no optional rest params anymore.
// We then transform this operation into a different one
@@ -307,7 +307,7 @@ function getLastLabelRemovedHandler(changeToOperationId: string) {
};
}
function getOnLabelAddedHandler(changeToOperationId: string) {
export function getOnLabelAddedHandler(changeToOperationId: string) {
return function onParamChanged(index: number, op: QueryBuilderOperation, def: QueryBuilderOperationDef) {
// Check if we actually have the label param. As it's optional the aggregation can have one less, which is the
// case of just simple aggregation without label. When user adds the label it now has the same number of params