From 79540a20b99ac51e157dd3705368e6d377849a54 Mon Sep 17 00:00:00 2001 From: Darren Janeczek <38694490+darrenjaneczek@users.noreply.github.com> Date: Fri, 14 Jun 2024 12:36:35 -0400 Subject: [PATCH] datatrails: persist search terms and use them to reduce retrieved metric names (#87884) --- .betterer.results | 3 +- public/app/features/trails/DataTrail.tsx | 15 +- .../trails/MetricSelect/MetricSelectScene.tsx | 351 +++++++++--------- .../app/features/trails/MetricSelect/api.ts | 30 ++ .../app/features/trails/MetricSelect/util.ts | 51 +++ .../features/trails/TrailStore/TrailStore.ts | 4 +- public/app/features/trails/shared.ts | 1 - 7 files changed, 274 insertions(+), 181 deletions(-) create mode 100644 public/app/features/trails/MetricSelect/api.ts create mode 100644 public/app/features/trails/MetricSelect/util.ts diff --git a/.betterer.results b/.betterer.results index 2efe1b0e08a..33fec39f716 100644 --- a/.betterer.results +++ b/.betterer.results @@ -5537,7 +5537,8 @@ exports[`better eslint`] = { "public/app/features/trails/MetricSelect/MetricSelectScene.tsx:5381": [ [0, 0, 0, "No untranslated strings. Wrap text with ", "0"], [0, 0, 0, "No untranslated strings. Wrap text with ", "1"], - [0, 0, 0, "No untranslated strings. Wrap text with ", "2"] + [0, 0, 0, "No untranslated strings. Wrap text with ", "2"], + [0, 0, 0, "No untranslated strings. Wrap text with ", "3"] ], "public/app/features/trails/MetricsHeader.tsx:5381": [ [0, 0, 0, "No untranslated strings. Wrap text with ", "0"], diff --git a/public/app/features/trails/DataTrail.tsx b/public/app/features/trails/DataTrail.tsx index 3295af66bc8..7470d6543e2 100644 --- a/public/app/features/trails/DataTrail.tsx +++ b/public/app/features/trails/DataTrail.tsx @@ -52,10 +52,11 @@ export interface DataTrailState extends SceneObjectState { // Synced with url metric?: string; + metricSearch?: string; } export class DataTrail extends SceneObjectBase { - protected _urlSync = new SceneObjectUrlSyncConfig(this, { keys: ['metric'] }); + protected _urlSync = new SceneObjectUrlSyncConfig(this, { keys: ['metric', 'metricSearch'] }); public constructor(state: Partial) { super({ @@ -109,7 +110,7 @@ export class DataTrail extends SceneObjectBase { protected _variableDependency = new VariableDependencyConfig(this, { variableNames: [VAR_DATASOURCE], - onReferencedVariableValueChanged: (variable: SceneVariable) => { + onReferencedVariableValueChanged: async (variable: SceneVariable) => { const { name } = variable.state; if (name === VAR_DATASOURCE) { this.datasourceHelper.reset(); @@ -156,6 +157,7 @@ export class DataTrail extends SceneObjectBase { sceneUtils.cloneSceneObjectState(state, { history: this.state.history, metric: !state.metric ? undefined : state.metric, + metricSearch: !state.metricSearch ? undefined : state.metricSearch, }) ); @@ -184,7 +186,8 @@ export class DataTrail extends SceneObjectBase { } getUrlState() { - return { metric: this.state.metric }; + const { metric, metricSearch } = this.state; + return { metric, metricSearch }; } updateFromUrl(values: SceneObjectUrlValues) { @@ -199,6 +202,12 @@ export class DataTrail extends SceneObjectBase { stateUpdate.topScene = new MetricSelectScene({}); } + if (typeof values.metricSearch === 'string') { + stateUpdate.metricSearch = values.metricSearch; + } else if (values.metric == null) { + stateUpdate.metricSearch = undefined; + } + this.setState(stateUpdate); } diff --git a/public/app/features/trails/MetricSelect/MetricSelectScene.tsx b/public/app/features/trails/MetricSelect/MetricSelectScene.tsx index 2acd179d229..062c8ab732e 100644 --- a/public/app/features/trails/MetricSelect/MetricSelectScene.tsx +++ b/public/app/features/trails/MetricSelect/MetricSelectScene.tsx @@ -1,11 +1,12 @@ import { css } from '@emotion/css'; -import { debounce } from 'lodash'; -import React, { useCallback } from 'react'; +import { debounce, isEqual } from 'lodash'; +import React, { useReducer } from 'react'; -import { GrafanaTheme2, VariableRefresh } from '@grafana/data'; +import { GrafanaTheme2, RawTimeRange } from '@grafana/data'; +import { isFetchError } from '@grafana/runtime'; import { + AdHocFiltersVariable, PanelBuilders, - QueryVariable, SceneComponentProps, SceneCSSGridItem, SceneCSSGridLayout, @@ -15,12 +16,13 @@ import { SceneObjectBase, SceneObjectRef, SceneObjectState, + SceneObjectStateChangedEvent, + SceneTimeRange, SceneVariable, SceneVariableSet, VariableDependencyConfig, } from '@grafana/scenes'; -import { VariableHide } from '@grafana/schema'; -import { Input, InlineSwitch, Field, Alert, Icon, useStyles2 } from '@grafana/ui'; +import { InlineSwitch, Field, Alert, Icon, useStyles2, Tooltip, Input } from '@grafana/ui'; import { MetricScene } from '../MetricScene'; import { StatusWrapper } from '../StatusWrapper'; @@ -29,16 +31,17 @@ import { reportExploreMetrics } from '../interactions'; import { getVariablesWithMetricConstant, MetricSelectedEvent, - trailDS, VAR_DATASOURCE, - VAR_FILTERS_EXPR, - VAR_METRIC_NAMES, + VAR_DATASOURCE_EXPR, + VAR_FILTERS, } from '../shared'; -import { getFilters, getTrailFor } from '../utils'; +import { getFilters, getTrailFor, isSceneTimeRangeState } from '../utils'; import { SelectMetricAction } from './SelectMetricAction'; +import { getMetricNames } from './api'; import { getPreviewPanelFor } from './previewPanel'; import { sortRelatedMetrics } from './relatedMetrics'; +import { createJSRegExpFromSearchTerms, createPromRegExp, deriveSearchTermsFromInput } from './util'; interface MetricPanel { name: string; @@ -51,21 +54,25 @@ interface MetricPanel { export interface MetricSelectSceneState extends SceneObjectState { body: SceneCSSGridLayout; - searchQuery?: string; showPreviews?: boolean; - metricsAfterSearch?: string[]; + metricNames?: string[]; + metricNamesLoading?: boolean; + metricNamesError?: string; + metricNamesWarning?: string; } const ROW_PREVIEW_HEIGHT = '175px'; const ROW_CARD_HEIGHT = '64px'; +const MAX_METRIC_NAMES = 20000; + export class MetricSelectScene extends SceneObjectBase { private previewCache: Record = {}; private ignoreNextUpdate = false; constructor(state: Partial) { super({ - $variables: state.$variables ?? getMetricNamesVariableSet(), + $variables: state.$variables, body: state.body ?? new SceneCSSGridLayout({ @@ -82,19 +89,10 @@ export class MetricSelectScene extends SceneObjectBase { } protected _variableDependency = new VariableDependencyConfig(this, { - variableNames: [VAR_METRIC_NAMES, VAR_DATASOURCE], + variableNames: [VAR_DATASOURCE, VAR_FILTERS], onReferencedVariableValueChanged: (variable: SceneVariable) => { - const { name } = variable.state; - - if (name === VAR_DATASOURCE) { - // Clear all panels for the previous data source - this.state.body.setState({ children: [] }); - } else if (name === VAR_METRIC_NAMES) { - this.onMetricNamesChange(); - // Entire pipeline must be performed - this.updateMetrics(); - this.buildLayout(); - } + // In all cases, we want to reload the metric names + this._debounceRefreshMetricNames(); }, }); @@ -108,24 +106,111 @@ export class MetricSelectScene extends SceneObjectBase { const trail = getTrailFor(this); - const metricChangeSubscription = trail.subscribeToEvent(MetricSelectedEvent, (event) => { - const { steps, currentStep } = trail.state.history.state; - const prevStep = steps[currentStep].parentIndex; - const previousMetric = steps[prevStep].trailState.metric; - const isRelatedMetricSelector = previousMetric !== undefined; + this._subs.add( + trail.subscribeToEvent(MetricSelectedEvent, (event) => { + const { steps, currentStep } = trail.state.history.state; + const prevStep = steps[currentStep].parentIndex; + const previousMetric = steps[prevStep].trailState.metric; + const isRelatedMetricSelector = previousMetric !== undefined; - const terms = this.state.searchQuery?.split(splitSeparator).filter((part) => part.length > 0); - if (event.payload !== undefined) { - reportExploreMetrics('metric_selected', { - from: isRelatedMetricSelector ? 'related_metrics' : 'metric_list', - searchTermCount: terms?.length || 0, - }); + if (event.payload !== undefined) { + const metricSearch = getMetricSearch(trail); + const searchTermCount = deriveSearchTermsFromInput(metricSearch).length; + + reportExploreMetrics('metric_selected', { + from: isRelatedMetricSelector ? 'related_metrics' : 'metric_list', + searchTermCount, + }); + } + }) + ); + + this._subs.add( + trail.subscribeToEvent(SceneObjectStateChangedEvent, (evt) => { + if (evt.payload.changedObject instanceof SceneTimeRange) { + const { prevState, newState } = evt.payload; + + if (isSceneTimeRangeState(prevState) && isSceneTimeRangeState(newState)) { + if (prevState.from === newState.from && prevState.to === newState.to) { + return; + } + } + } + }) + ); + + this._subs.add( + trail.subscribeToState(({ metricSearch }, oldState) => { + const oldSearchTerms = deriveSearchTermsFromInput(oldState.metricSearch); + const newSearchTerms = deriveSearchTermsFromInput(metricSearch); + if (!isEqual(oldSearchTerms, newSearchTerms)) { + this._debounceRefreshMetricNames(); + } + }) + ); + + this.subscribeToState((newState, prevState) => { + if (newState.metricNames !== prevState.metricNames) { + this.onMetricNamesChanged(); } }); - return () => { - metricChangeSubscription.unsubscribe(); - }; + this._debounceRefreshMetricNames(); + } + + private _debounceRefreshMetricNames = debounce(() => this._refreshMetricNames(), 1000); + + private async _refreshMetricNames() { + const trail = getTrailFor(this); + const timeRange: RawTimeRange | undefined = trail.state.$timeRange?.state; + + if (!timeRange) { + return; + } + + const matchTerms = []; + + const filtersVar = sceneGraph.lookupVariable(VAR_FILTERS, this); + const hasFilters = filtersVar instanceof AdHocFiltersVariable && filtersVar.getValue()?.valueOf(); + if (hasFilters) { + matchTerms.push(sceneGraph.interpolate(trail, '${filters}')); + } + + const metricSearchRegex = createPromRegExp(trail.state.metricSearch); + if (metricSearchRegex) { + matchTerms.push(`__name__=~"${metricSearchRegex}"`); + } + + const match = `{${matchTerms.join(',')}}`; + const datasourceUid = sceneGraph.interpolate(trail, VAR_DATASOURCE_EXPR); + this.setState({ metricNamesLoading: true, metricNamesError: undefined, metricNamesWarning: undefined }); + + try { + const response = await getMetricNames(datasourceUid, timeRange, match, MAX_METRIC_NAMES); + const searchRegex = createJSRegExpFromSearchTerms(getMetricSearch(this)); + const metricNames = searchRegex + ? response.data.filter((metric) => !searchRegex || searchRegex.test(metric)) + : response.data; + + const metricNamesWarning = response.limitReached + ? `This feature will only return up to ${MAX_METRIC_NAMES} metric names for performance reasons. ` + + `This limit is being exceeded for the current data source. ` + + `Add search terms or label filters to narrow down the number of metric names returned.` + : undefined; + + this.setState({ metricNames, metricNamesLoading: false, metricNamesWarning, metricNamesError: response.error }); + } catch (err: unknown) { + let error = 'Unknown error'; + if (isFetchError(err)) { + if (err.cancelled) { + error = 'Request cancelled'; + } else if (err.statusText) { + error = err.statusText; + } + } + + this.setState({ metricNames: undefined, metricNamesLoading: false, metricNamesError: error }); + } } private sortedPreviewMetrics() { @@ -143,22 +228,10 @@ export class MetricSelectScene extends SceneObjectBase { }); } - private currentMetricNames = new Set(); + private onMetricNamesChanged() { + const metricNames = this.state.metricNames || []; - private onMetricNamesChange() { - // Get the datasource metrics list from the VAR_METRIC_NAMES variable - const variable = sceneGraph.lookupVariable(VAR_METRIC_NAMES, this); - - if (!(variable instanceof QueryVariable)) { - return; - } - - if (variable.state.loading) { - return; - } - - const nameList = variable.state.options.map((option) => option.value.toString()); - const nameSet = new Set(nameList); + const nameSet = new Set(metricNames); Object.values(this.previewCache).forEach((panel) => { if (!nameSet.has(panel.name)) { @@ -166,35 +239,6 @@ export class MetricSelectScene extends SceneObjectBase { } }); - this.currentMetricNames = nameSet; - this.buildLayout(); - } - - private applyMetricSearch() { - // This should only occur when the `searchQuery` changes, of if the `metricNames` change - const metricNames = Array.from(this.currentMetricNames); - if (metricNames == null) { - return; - } - const searchRegex = createSearchRegExp(this.state.searchQuery); - - if (!searchRegex) { - this.setState({ metricsAfterSearch: metricNames }); - } else { - const metricsAfterSearch = metricNames.filter((metric) => !searchRegex || searchRegex.test(metric)); - this.setState({ metricsAfterSearch }); - } - } - - private updateMetrics(applySearchAndFilter = true) { - if (applySearchAndFilter) { - // Set to false if these are not required (because they can be assumed to have been suitably called). - this.applyMetricSearch(); - } - - const { metricsAfterSearch } = this.state; - - const metricNames = metricsAfterSearch || []; const trail = getTrailFor(this); const sortedMetricNames = trail.state.metric !== undefined ? sortRelatedMetrics(metricNames, trail.state.metric) : metricNames; @@ -203,7 +247,7 @@ export class MetricSelectScene extends SceneObjectBase { // Clear absent metrics from cache Object.keys(this.previewCache).forEach((metric) => { - if (!this.currentMetricNames.has(metric)) { + if (!nameSet.has(metric)) { delete this.previewCache[metric]; } }); @@ -231,6 +275,7 @@ export class MetricSelectScene extends SceneObjectBase { } this.previewCache = metricsMap; + this.buildLayout(); } private async buildLayout() { @@ -240,20 +285,6 @@ export class MetricSelectScene extends SceneObjectBase { return; } - const variable = sceneGraph.lookupVariable(VAR_METRIC_NAMES, this); - - if (!(variable instanceof QueryVariable)) { - return; - } - - if (variable.state.loading) { - return; - } - - if (!Object.keys(this.previewCache).length) { - this.updateMetrics(); - } - const children: SceneFlexItem[] = []; const trail = getTrailFor(this); @@ -309,30 +340,32 @@ export class MetricSelectScene extends SceneObjectBase { }; public onSearchQueryChange = (evt: React.SyntheticEvent) => { - this.setState({ searchQuery: evt.currentTarget.value }); - this.searchQueryChangedDebounced(); + const metricSearch = evt.currentTarget.value; + const trail = getTrailFor(this); + // Update the variable + trail.setState({ metricSearch }); }; - private searchQueryChangedDebounced = debounce(() => { - this.updateMetrics(); // Need to repeat entire pipeline - this.buildLayout(); - }, 500); - public onTogglePreviews = () => { this.setState({ showPreviews: !this.state.showPreviews }); this.buildLayout(); }; public static Component = ({ model }: SceneComponentProps) => { - const { searchQuery, showPreviews, body } = model.useState(); + const { showPreviews, body, metricNames, metricNamesError, metricNamesLoading, metricNamesWarning } = + model.useState(); const { children } = body.useState(); + const trail = getTrailFor(model); const styles = useStyles2(getStyles); - const metricNamesStatus = useVariableStatus(VAR_METRIC_NAMES, model); - const tooStrict = children.length === 0 && searchQuery; - const noMetrics = !metricNamesStatus.isLoading && model.currentMetricNames.size === 0; + const [warningDismissed, dismissWarning] = useReducer(() => true, false); - const isLoading = metricNamesStatus.isLoading && children.length === 0; + const { metricSearch } = trail.useState(); + + const tooStrict = children.length === 0 && metricSearch; + const noMetrics = !metricNamesLoading && metricNames && metricNames.length === 0; + + const isLoading = metricNamesLoading && children.length === 0; const blockingMessage = isLoading ? undefined @@ -340,7 +373,18 @@ export class MetricSelectScene extends SceneObjectBase { (tooStrict && 'There are no results found. Try adjusting your search or filters.') || undefined; - const disableSearch = metricNamesStatus.error || metricNamesStatus.isLoading; + const metricNamesWarningIcon = metricNamesWarning ? ( + +

Unable to retrieve metric names

+

{metricNamesWarning}

+ + } + > + + + ) : undefined; return (
@@ -349,23 +393,27 @@ export class MetricSelectScene extends SceneObjectBase { } - value={searchQuery} + value={metricSearch} onChange={model.onSearchQueryChange} - disabled={disableSearch} + suffix={metricNamesWarningIcon} /> - +
- {metricNamesStatus.error && ( + {metricNamesError && (
We are unable to connect to your data source. Double check your data source URL and credentials.
-
({metricNamesStatus.error})
+
({metricNamesError})
+ + )} + {metricNamesWarning && !warningDismissed && ( + +
{metricNamesWarning}
 )} @@ -376,23 +424,6 @@ export class MetricSelectScene extends SceneObjectBase { }; } -function getMetricNamesVariableSet() { - return new SceneVariableSet({ - variables: [ - new QueryVariable({ - name: VAR_METRIC_NAMES, - datasource: trailDS, - hide: VariableHide.hideVariable, - includeAll: true, - defaultToAll: true, - skipUrlSync: true, - refresh: VariableRefresh.onTimeRangeChanged, - query: { query: `label_values(${VAR_FILTERS_EXPR},__name__)`, refId: 'A' }, - }), - ], - }); -} - function getCardPanelFor(metric: string, description?: string) { return PanelBuilders.text() .setTitle(metric) @@ -423,43 +454,13 @@ function getStyles(theme: GrafanaTheme2) { flexGrow: 1, marginBottom: 0, }), + warningIcon: css({ + color: theme.colors.warning.main, + }), }; } -// Consider any sequence of characters not permitted for metric names as a sepratator -const splitSeparator = /[^a-z0-9_:]+/; - -function createSearchRegExp(spaceSeparatedMetricNames?: string) { - if (!spaceSeparatedMetricNames) { - return null; - } - const searchParts = spaceSeparatedMetricNames - ?.toLowerCase() - .split(splitSeparator) - .filter((part) => part.length > 0) - .map((part) => `(?=(.*${part}.*))`); - - if (searchParts.length === 0) { - return null; - } - - const regex = searchParts.join(''); - // (?=(.*expr1.*))(?=().*expr2.*))... - // The ?=(...) lookahead allows us to match these in any order. - return new RegExp(regex, 'igy'); -} - -function useVariableStatus(name: string, sceneObject: SceneObject) { - const variable = sceneGraph.lookupVariable(name, sceneObject); - - const useVariableState = useCallback(() => { - if (variable) { - return variable.useState(); - } - return undefined; - }, [variable]); - - const { error, loading } = useVariableState() || {}; - - return { isLoading: !!loading, error }; +function getMetricSearch(scene: SceneObject) { + const trail = getTrailFor(scene); + return trail.state.metricSearch || ''; } diff --git a/public/app/features/trails/MetricSelect/api.ts b/public/app/features/trails/MetricSelect/api.ts new file mode 100644 index 00000000000..3ec86f83b50 --- /dev/null +++ b/public/app/features/trails/MetricSelect/api.ts @@ -0,0 +1,30 @@ +import { RawTimeRange } from '@grafana/data'; +import { getPrometheusTime } from '@grafana/prometheus/src/language_utils'; +import { getBackendSrv } from '@grafana/runtime'; + +type MetricValuesResponse = { + data: string[]; + status: 'success' | 'error'; + error?: 'string'; + warnings?: string[]; +}; + +const LIMIT_REACHED = 'results truncated due to limit'; + +export async function getMetricNames(dataSourceUid: string, timeRange: RawTimeRange, filters: string, limit?: number) { + const url = `/api/datasources/uid/${dataSourceUid}/resources/api/v1/label/__name__/values`; + const params: Record = { + start: getPrometheusTime(timeRange.from, false), + end: getPrometheusTime(timeRange.to, true), + ...(filters && filters !== '{}' ? { 'match[]': filters } : {}), + ...(limit ? { limit } : {}), + }; + + const response = await getBackendSrv().get(url, params, 'explore-metrics-names'); + + if (limit && response.warnings?.includes(LIMIT_REACHED)) { + return { ...response, limitReached: true }; + } + + return { ...response, limitReached: false }; +} diff --git a/public/app/features/trails/MetricSelect/util.ts b/public/app/features/trails/MetricSelect/util.ts new file mode 100644 index 00000000000..b71e98d04b4 --- /dev/null +++ b/public/app/features/trails/MetricSelect/util.ts @@ -0,0 +1,51 @@ +// Consider any sequence of characters not permitted for metric names as a sepratator +const splitSeparator = /[^a-z0-9_:]+/; + +export function deriveSearchTermsFromInput(whiteSpaceSeparatedTerms?: string) { + return ( + whiteSpaceSeparatedTerms + ?.toLowerCase() + .split(splitSeparator) + .filter((term) => term.length > 0) || [] + ); +} + +export function createJSRegExpFromSearchTerms(searchQuery?: string) { + const searchParts = deriveSearchTermsFromInput(searchQuery).map((part) => `(?=(.*${part.toLowerCase()}.*))`); + + if (searchParts.length === 0) { + return null; + } + + const regex = searchParts.join(''); + // (?=(.*expr1.*)(?=(.*expr2.*))... + // The ?=(...) lookahead allows us to match these in any order. + return new RegExp(regex, 'igy'); +} + +export function createPromRegExp(searchQuery?: string) { + const searchParts = getUniqueTerms(deriveSearchTermsFromInput(searchQuery)) + .filter((term) => term.length > 0) + .map((term) => `(.*${term.toLowerCase()}.*)`); + + const count = searchParts.length; + + if (searchParts.length === 0) { + // avoid match[] must contain at least one non-empty matcher + return null; //'..*'; + } + + const regex = `(?i:${searchParts.join('|')}){${count}}`; + // (?i:(.*expr_1.*)|.*expr_2.*)|...|.*expr_n.*){n} + // ?i: to ignore case + // {n} to ensure that it matches n times, one match per term + // - This isn't ideal, since it doesn't enforce that each unique term is matched, + // but it's the best we can do with the Prometheus / Go stdlib implementation of regex. + + return regex; +} + +function getUniqueTerms(terms: string[] = []) { + const set = new Set(terms.map((term) => term.toLowerCase().trim())); + return Array.from(set); +} diff --git a/public/app/features/trails/TrailStore/TrailStore.ts b/public/app/features/trails/TrailStore/TrailStore.ts index 4eeea064b32..69e2ff119b6 100644 --- a/public/app/features/trails/TrailStore/TrailStore.ts +++ b/public/app/features/trails/TrailStore/TrailStore.ts @@ -191,9 +191,11 @@ function getUrlStateForComparison(trail: DataTrail) { const urlState = getUrlSyncManager().getUrlState(trail); // Make a few corrections - // Omit some URL parameters that are not useful for state comparison + // Omit some URL parameters that are not useful for state comparison, + // as they can change in the URL without creating new steps delete urlState.actionView; delete urlState.layout; + delete urlState.metricSearch; // Populate defaults if (urlState['var-groupby'] === '') { diff --git a/public/app/features/trails/shared.ts b/public/app/features/trails/shared.ts index 691360a53c4..7c8ed5697b3 100644 --- a/public/app/features/trails/shared.ts +++ b/public/app/features/trails/shared.ts @@ -13,7 +13,6 @@ export interface ActionViewDefinition { export const TRAILS_ROUTE = '/explore/metrics/trail'; export const HOME_ROUTE = '/explore/metrics'; -export const VAR_METRIC_NAMES = 'metricNames'; export const VAR_FILTERS = 'filters'; export const VAR_FILTERS_EXPR = '{${filters}}'; export const VAR_METRIC = 'metric';