From a80517d6fecf245cb9366b5662ab6d8760200389 Mon Sep 17 00:00:00 2001 From: Bogdan Matei Date: Wed, 6 Nov 2024 15:11:33 +0200 Subject: [PATCH] Scopes: Add scopes to metrics explore (#94802) --- .../src/types/featureToggles.gen.ts | 1 + pkg/promlib/resource/resource.go | 7 +- pkg/services/featuremgmt/registry.go | 11 ++ pkg/services/featuremgmt/toggles_gen.csv | 1 + pkg/services/featuremgmt/toggles_gen.go | 4 + pkg/services/featuremgmt/toggles_gen.json | 14 +++ .../trails/Breakdown/LabelBreakdownScene.tsx | 9 ++ public/app/features/trails/DataTrail.tsx | 40 ++++++- public/app/features/trails/DataTrailsApp.tsx | 75 ++++++++++++- public/app/features/trails/MetricScene.tsx | 11 ++ .../trails/MetricSelect/MetricSelectScene.tsx | 82 +++++++------- .../app/features/trails/MetricSelect/api.ts | 95 ++++++++++++++-- public/app/features/trails/otel/api.test.ts | 4 +- public/app/features/trails/otel/api.ts | 84 ++++++++++---- public/app/features/trails/otel/util.ts | 5 +- public/app/features/trails/otel/utils.test.ts | 12 +- public/app/features/trails/shared.ts | 6 +- public/app/features/trails/utils.test.ts | 10 +- public/app/features/trails/utils.ts | 105 ++++++++++++++++-- 19 files changed, 471 insertions(+), 105 deletions(-) diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index 4b8a03abd9d..ab7c715ba8f 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -199,6 +199,7 @@ export interface FeatureToggles { failWrongDSUID?: boolean; zanzana?: boolean; reloadDashboardsOnParamsChange?: boolean; + enableScopesInMetricsExplore?: boolean; alertingApiServer?: boolean; cloudWatchRoundUpEndTime?: boolean; cloudwatchMetricInsightsCrossAccount?: boolean; diff --git a/pkg/promlib/resource/resource.go b/pkg/promlib/resource/resource.go index 2c9705ef80f..56988d10805 100644 --- a/pkg/promlib/resource/resource.go +++ b/pkg/promlib/resource/resource.go @@ -177,12 +177,17 @@ func (r *Resource) GetSuggestions(ctx context.Context, req *backend.CallResource } values := url.Values{} - for _, s := range selectorList { vs := parser.VectorSelector{Name: s, LabelMatchers: matchers} values.Add("match[]", vs.String()) } + // if no timeserie name is provided, but scopes are, the scope is still rendered and passed as match param. + if len(selectorList) == 0 && len(sugReq.Scopes) > 0 { + vs := parser.VectorSelector{LabelMatchers: matchers} + values.Add("match[]", vs.String()) + } + if sugReq.Start != "" { values.Add("start", sugReq.Start) } diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index 66a5c1d76b9..49b392cdff4 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -1373,6 +1373,17 @@ var ( HideFromDocs: true, HideFromAdminPage: true, }, + { + Name: "enableScopesInMetricsExplore", + Description: "Enables the scopes usage in Metrics Explore", + FrontendOnly: false, + Stage: FeatureStageExperimental, + Owner: grafanaDashboardsSquad, + RequiresRestart: false, + AllowSelfServe: false, + HideFromDocs: true, + HideFromAdminPage: true, + }, { Name: "alertingApiServer", Description: "Register Alerting APIs with the K8s API server", diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index 003d2be0097..8e761d96b00 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -180,6 +180,7 @@ ssoSettingsLDAP,preview,@grafana/identity-access-team,false,true,false failWrongDSUID,experimental,@grafana/plugins-platform-backend,false,false,false zanzana,experimental,@grafana/identity-access-team,false,false,false reloadDashboardsOnParamsChange,experimental,@grafana/dashboards-squad,false,false,false +enableScopesInMetricsExplore,experimental,@grafana/dashboards-squad,false,false,false alertingApiServer,experimental,@grafana/alerting-squad,false,true,false cloudWatchRoundUpEndTime,GA,@grafana/aws-datasources,false,false,false cloudwatchMetricInsightsCrossAccount,GA,@grafana/aws-datasources,false,false,true diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index 4013b71ca0c..b28e0548cae 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -731,6 +731,10 @@ const ( // Enables reload of dashboards on scopes, time range and variables changes FlagReloadDashboardsOnParamsChange = "reloadDashboardsOnParamsChange" + // FlagEnableScopesInMetricsExplore + // Enables the scopes usage in Metrics Explore + FlagEnableScopesInMetricsExplore = "enableScopesInMetricsExplore" + // FlagAlertingApiServer // Register Alerting APIs with the K8s API server FlagAlertingApiServer = "alertingApiServer" diff --git a/pkg/services/featuremgmt/toggles_gen.json b/pkg/services/featuremgmt/toggles_gen.json index 5e9f6020e3d..39ebc17db51 100644 --- a/pkg/services/featuremgmt/toggles_gen.json +++ b/pkg/services/featuremgmt/toggles_gen.json @@ -1231,6 +1231,20 @@ "hideFromAdminPage": true } }, + { + "metadata": { + "name": "enableScopesInMetricsExplore", + "resourceVersion": "1729765731452", + "creationTimestamp": "2024-10-24T10:28:51Z" + }, + "spec": { + "description": "Enables the scopes usage in Metrics Explore", + "stage": "experimental", + "codeowner": "@grafana/dashboards-squad", + "hideFromAdminPage": true, + "hideFromDocs": true + } + }, { "metadata": { "name": "exploreContentOutline", diff --git a/public/app/features/trails/Breakdown/LabelBreakdownScene.tsx b/public/app/features/trails/Breakdown/LabelBreakdownScene.tsx index ccf7a0af877..e2d370f164e 100644 --- a/public/app/features/trails/Breakdown/LabelBreakdownScene.tsx +++ b/public/app/features/trails/Breakdown/LabelBreakdownScene.tsx @@ -42,6 +42,7 @@ import { getSortByPreference } from '../services/store'; import { ALL_VARIABLE_VALUE } from '../services/variables'; import { MDP_METRIC_PREVIEW, + RefreshMetricsEvent, trailDS, VAR_FILTERS, VAR_GROUP_BY, @@ -98,6 +99,14 @@ export class LabelBreakdownScene extends SceneObjectBase { + this.updateBody(this.getVariable()); + }) + ); + } + variable.subscribeToState((newState, oldState) => { if ( newState.options !== oldState.options || diff --git a/public/app/features/trails/DataTrail.tsx b/public/app/features/trails/DataTrail.tsx index b924fba3740..448253d5819 100644 --- a/public/app/features/trails/DataTrail.tsx +++ b/public/app/features/trails/DataTrail.tsx @@ -10,6 +10,7 @@ import { VariableHide, urlUtil, } from '@grafana/data'; +import { PromQuery } from '@grafana/prometheus'; import { locationService, useChromeHeaderHeight } from '@grafana/runtime'; import { AdHocFiltersVariable, @@ -24,6 +25,7 @@ import { SceneObjectState, SceneObjectUrlSyncConfig, SceneObjectUrlValues, + SceneQueryRunner, SceneRefreshPicker, SceneTimePicker, SceneTimeRange, @@ -35,6 +37,7 @@ import { VariableValueSelectors, } from '@grafana/scenes'; import { useStyles2 } from '@grafana/ui'; +import { getSelectedScopes } from 'app/features/scopes'; import { DataTrailSettings } from './DataTrailSettings'; import { DataTrailHistory } from './DataTrailsHistory'; @@ -325,7 +328,7 @@ export class DataTrail extends SceneObjectBase { const datasourceUid = sceneGraph.interpolate(trail, VAR_DATASOURCE_EXPR); const otelTargets = await totalOtelResources(datasourceUid, timeRange); - const deploymentEnvironments = await getDeploymentEnvironments(datasourceUid, timeRange); + const deploymentEnvironments = await getDeploymentEnvironments(datasourceUid, timeRange, getSelectedScopes()); const hasOtelResources = otelTargets.jobs.length > 0 && otelTargets.instances.length > 0; if ( otelResourcesVariable instanceof AdHocFiltersVariable && @@ -469,7 +472,13 @@ export class DataTrail extends SceneObjectBase { values: GetTagResponse | MetricFindValue[]; }> => { // apply filters here - let values = await datasourceHelper.getTagKeys({ filters }); + // we're passing the queries so we get the labels that adhere to the queries + // we're also passing the scopes so we get the labels that adhere to the scopes filters + let values = await datasourceHelper.getTagKeys({ + filters, + scopes: getSelectedScopes(), + queries: this.getQueries(), + }); values = sortResources(values, filters.map((f) => f.key).concat(currentKey ?? '')); return { replace: true, values }; }, @@ -483,7 +492,14 @@ export class DataTrail extends SceneObjectBase { // apply filters here // remove current selected filter if refiltering filters = filters.filter((f) => f.key !== filter.key); - const values = await datasourceHelper.getTagValues({ key: filter.key, filters }); + // we're passing the queries so we get the label values that adhere to the queries + // we're also passing the scopes so we get the label values that adhere to the scopes filters + const values = await datasourceHelper.getTagValues({ + key: filter.key, + filters, + scopes: getSelectedScopes(), + queries: this.getQueries(), + }); return { replace: true, values }; }, hide: VariableHide.hideLabel, @@ -573,6 +589,22 @@ export class DataTrail extends SceneObjectBase { } } + public getQueries(): PromQuery[] { + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + const sqrs = sceneGraph.findAllObjects(this, (b) => b instanceof SceneQueryRunner) as SceneQueryRunner[]; + + return sqrs.reduce((acc, sqr) => { + acc.push( + ...sqr.state.queries.map((q) => ({ + ...q, + expr: sceneGraph.interpolate(sqr, q.expr), + })) + ); + + return acc; + }, []); + } + static Component = ({ model }: SceneComponentProps) => { const { controls, topScene, history, settings, useOtelExperience, hasOtelResources } = model.useState(); @@ -606,7 +638,7 @@ export class DataTrail extends SceneObjectBase { useEffect(() => { const filtersVariable = sceneGraph.lookupVariable(VAR_FILTERS, model); const datasourceHelper = model.datasourceHelper; - limitAdhocProviders(filtersVariable, datasourceHelper); + limitAdhocProviders(model, filtersVariable, datasourceHelper); }, [model]); return ( diff --git a/public/app/features/trails/DataTrailsApp.tsx b/public/app/features/trails/DataTrailsApp.tsx index 25115c4e213..f7f0112e456 100644 --- a/public/app/features/trails/DataTrailsApp.tsx +++ b/public/app/features/trails/DataTrailsApp.tsx @@ -1,15 +1,25 @@ +import { css } from '@emotion/css'; import { useEffect, useState } from 'react'; import { Routes, Route } from 'react-router-dom-v5-compat'; -import { PageLayoutType } from '@grafana/data'; -import { locationService } from '@grafana/runtime'; +import { + DataQueryRequest, + DataSourceGetTagKeysOptions, + DataSourceGetTagValuesOptions, + PageLayoutType, +} from '@grafana/data'; +import { config, locationService } from '@grafana/runtime'; import { SceneComponentProps, SceneObjectBase, SceneObjectState, UrlSyncContextProvider } from '@grafana/scenes'; +import { useStyles2 } from '@grafana/ui/'; import { Page } from 'app/core/components/Page/Page'; +import { getClosestScopesFacade, ScopesFacade, ScopesSelector } from 'app/features/scopes'; + +import { AppChromeUpdate } from '../../core/components/AppChrome/AppChromeUpdate'; import { DataTrail } from './DataTrail'; import { DataTrailsHome } from './DataTrailsHome'; import { getTrailStore } from './TrailStore/TrailStore'; -import { HOME_ROUTE, TRAILS_ROUTE } from './shared'; +import { HOME_ROUTE, RefreshMetricsEvent, TRAILS_ROUTE } from './shared'; import { getMetricName, getUrlForTrail, newMetricsTrail } from './utils'; export interface DataTrailsAppState extends SceneObjectState { @@ -18,8 +28,32 @@ export interface DataTrailsAppState extends SceneObjectState { } export class DataTrailsApp extends SceneObjectBase { + private _scopesFacade: ScopesFacade | null; + public constructor(state: DataTrailsAppState) { super(state); + + this._scopesFacade = getClosestScopesFacade(this); + } + + public enrichDataRequest(): Partial { + if (!config.featureToggles.promQLScope) { + return {}; + } + + return { + scopes: this._scopesFacade?.value, + }; + } + + public enrichFiltersRequest(): Partial { + if (!config.featureToggles.promQLScope) { + return {}; + } + + return { + scopes: this._scopesFacade?.value, + }; } goToUrlForTrail(trail: DataTrail) { @@ -54,6 +88,7 @@ export class DataTrailsApp extends SceneObjectBase { } function DataTrailView({ trail }: { trail: DataTrail }) { + const styles = useStyles2(getStyles); const [isInitialized, setIsInitialized] = useState(false); const { metric } = trail.useState(); @@ -73,6 +108,15 @@ function DataTrailView({ trail }: { trail: DataTrail }) { return ( + {config.featureToggles.singleTopNav && config.featureToggles.enableScopesInMetricsExplore && ( + + + + } + /> + )} @@ -83,11 +127,36 @@ let dataTrailsApp: DataTrailsApp; export function getDataTrailsApp() { if (!dataTrailsApp) { + const $behaviors = config.featureToggles.enableScopesInMetricsExplore + ? [ + new ScopesFacade({ + handler: (facade) => { + const trail = facade.parent && 'trail' in facade.parent.state ? facade.parent.state.trail : undefined; + + if (trail instanceof DataTrail) { + trail.publishEvent(new RefreshMetricsEvent()); + trail.checkDataSourceForOTelResources(); + } + }, + }), + ] + : undefined; + dataTrailsApp = new DataTrailsApp({ trail: newMetricsTrail(), home: new DataTrailsHome({}), + $behaviors, }); } return dataTrailsApp; } + +const getStyles = () => ({ + topNavContainer: css({ + width: '100%', + display: 'flex', + flexDirection: 'row', + justifyItems: 'flex-start', + }), +}); diff --git a/public/app/features/trails/MetricScene.tsx b/public/app/features/trails/MetricScene.tsx index b02e716902f..fe9161312b2 100644 --- a/public/app/features/trails/MetricScene.tsx +++ b/public/app/features/trails/MetricScene.tsx @@ -32,6 +32,7 @@ import { getVariablesWithMetricConstant, MakeOptional, MetricSelectedEvent, + RefreshMetricsEvent, trailDS, VAR_GROUP_BY, VAR_METRIC_EXPR, @@ -69,6 +70,16 @@ export class MetricScene extends SceneObjectBase { if (this.state.actionView === undefined) { this.setActionView('overview'); } + + if (config.featureToggles.enableScopesInMetricsExplore) { + // Push the scopes change event to the tabs + // The event is not propagated because the tabs are not part of the scene graph + this._subs.add( + this.subscribeToEvent(RefreshMetricsEvent, (event) => { + this.state.body.state.selectedTab?.publishEvent(event); + }) + ); + } } getUrlState() { diff --git a/public/app/features/trails/MetricSelect/MetricSelectScene.tsx b/public/app/features/trails/MetricSelect/MetricSelectScene.tsx index 1a28aac9146..f8e68f408dd 100644 --- a/public/app/features/trails/MetricSelect/MetricSelectScene.tsx +++ b/public/app/features/trails/MetricSelect/MetricSelectScene.tsx @@ -2,8 +2,8 @@ import { css } from '@emotion/css'; import { debounce, isEqual } from 'lodash'; import { SyntheticEvent, useReducer } from 'react'; -import { GrafanaTheme2, RawTimeRange, SelectableValue } from '@grafana/data'; -import { isFetchError } from '@grafana/runtime'; +import { AdHocVariableFilter, GrafanaTheme2, RawTimeRange, SelectableValue } from '@grafana/data'; +import { config, isFetchError } from '@grafana/runtime'; import { AdHocFiltersVariable, PanelBuilders, @@ -22,22 +22,22 @@ import { SceneObjectUrlValues, SceneObjectWithUrlSync, SceneTimeRange, - SceneVariable, SceneVariableSet, VariableDependencyConfig, } from '@grafana/scenes'; import { Alert, Field, Icon, IconButton, InlineSwitch, Input, Select, Tooltip, useStyles2 } from '@grafana/ui'; import { Trans } from 'app/core/internationalization'; +import { getSelectedScopes } from 'app/features/scopes'; import { MetricScene } from '../MetricScene'; import { StatusWrapper } from '../StatusWrapper'; import { Node, Parser } from '../groop/parser'; import { getMetricDescription } from '../helpers/MetricDatasourceHelper'; import { reportExploreMetrics } from '../interactions'; -import { limitOtelMatchTerms } from '../otel/util'; import { getVariablesWithMetricConstant, MetricSelectedEvent, + RefreshMetricsEvent, VAR_DATASOURCE, VAR_DATASOURCE_EXPR, VAR_FILTERS, @@ -104,7 +104,7 @@ export class MetricSelectScene extends SceneObjectBase i protected _urlSync = new SceneObjectUrlSyncConfig(this, { keys: ['metricPrefix'] }); protected _variableDependency = new VariableDependencyConfig(this, { variableNames: [VAR_DATASOURCE, VAR_FILTERS], - onReferencedVariableValueChanged: (variable: SceneVariable) => { + onReferencedVariableValueChanged: () => { // In all cases, we want to reload the metric names this._debounceRefreshMetricNames(); }, @@ -194,7 +194,7 @@ export class MetricSelectScene extends SceneObjectBase i ); this._subs.add( - trail.subscribeToState(({ useOtelExperience }, oldState) => { + trail.subscribeToState(() => { // users will most likely not switch this off but for now, // update metric names when changing useOtelExperience this._debounceRefreshMetricNames(); @@ -202,13 +202,21 @@ export class MetricSelectScene extends SceneObjectBase i ); this._subs.add( - trail.subscribeToState(({ showPreviews }, oldState) => { + trail.subscribeToState(() => { // move showPreviews into the settings // build layout when toggled this.buildLayout(); }) ); + if (config.featureToggles.enableScopesInMetricsExplore) { + this._subs.add( + trail.subscribeToEvent(RefreshMetricsEvent, () => { + this._debounceRefreshMetricNames(); + }) + ); + } + this._debounceRefreshMetricNames(); } @@ -220,45 +228,39 @@ export class MetricSelectScene extends SceneObjectBase i return; } - const matchTerms: string[] = []; + const filters: AdHocVariableFilter[] = []; const filtersVar = sceneGraph.lookupVariable(VAR_FILTERS, this); - const hasFilters = filtersVar instanceof AdHocFiltersVariable && filtersVar.getValue()?.valueOf(); - if (hasFilters) { - matchTerms.push(sceneGraph.interpolate(trail, '${filters}')); + const adhocFilters = filtersVar instanceof AdHocFiltersVariable ? (filtersVar?.state.filters ?? []) : []; + if (adhocFilters.length > 0) { + filters.push(...adhocFilters); } const metricSearchRegex = createPromRegExp(trail.state.metricSearch); if (metricSearchRegex) { - matchTerms.push(`__name__=~"${metricSearchRegex}"`); + filters.push({ + key: '__name__', + operator: '=~', + value: metricSearchRegex, + }); } - let noOtelMetrics = false; - let missingOtelTargets = false; - - if (trail.state.useOtelExperience) { - const jobsList = trail.state.otelTargets?.jobs; - const instancesList = trail.state.otelTargets?.instances; - // no targets have this combination of filters so there are no metrics that can be joined - // show no metrics - if (jobsList && jobsList.length > 0 && instancesList && instancesList.length > 0) { - const otelMatches = limitOtelMatchTerms(matchTerms, jobsList, instancesList, missingOtelTargets); - - missingOtelTargets = otelMatches.missingOtelTargets; - - matchTerms.push(otelMatches.jobsRegex); - matchTerms.push(otelMatches.instancesRegex); - } else { - noOtelMetrics = true; - } - } - - const match = `{${matchTerms.join(',')}}`; const datasourceUid = sceneGraph.interpolate(trail, VAR_DATASOURCE_EXPR); this.setState({ metricNamesLoading: true, metricNamesError: undefined, metricNamesWarning: undefined }); try { - const response = await getMetricNames(datasourceUid, timeRange, match, MAX_METRIC_NAMES); + const jobsList = trail.state.useOtelExperience ? (trail.state.otelTargets?.jobs ?? []) : []; + const instancesList = trail.state.useOtelExperience ? (trail.state.otelTargets?.instances ?? []) : []; + + const response = await getMetricNames( + datasourceUid, + timeRange, + getSelectedScopes(), + filters, + jobsList, + instancesList, + MAX_METRIC_NAMES + ); const searchRegex = createJSRegExpFromSearchTerms(getMetricSearch(this)); let metricNames = searchRegex ? response.data.filter((metric) => !searchRegex || searchRegex.test(metric)) @@ -281,21 +283,19 @@ export class MetricSelectScene extends SceneObjectBase i : undefined; // if there are no otel targets for otel resources, there will be no labels - if (noOtelMetrics) { + if (trail.state.useOtelExperience && (jobsList.length === 0 || instancesList.length === 0)) { metricNames = []; metricNamesWarning = undefined; } - if (missingOtelTargets) { + if (response.missingOtelTargets) { metricNamesWarning = `${metricNamesWarning ?? ''} The list of metrics is not complete. Select more OTel resource attributes to see a full list of metrics.`; } let bodyLayout = this.state.body; - let rootGroupNode = this.state.rootGroup; - // generate groups based on the search metrics input - rootGroupNode = await this.generateGroups(filteredMetricNames); + let rootGroupNode = await this.generateGroups(filteredMetricNames); this.setState({ metricNames, @@ -364,9 +364,7 @@ export class MetricSelectScene extends SceneObjectBase i const oldPanel = this.previewCache[metricName]; - const panel = oldPanel || { name: metricName, index, loaded: false }; - - metricsMap[metricName] = panel; + metricsMap[metricName] = oldPanel || { name: metricName, index, loaded: false }; } try { diff --git a/public/app/features/trails/MetricSelect/api.ts b/public/app/features/trails/MetricSelect/api.ts index 3ec86f83b50..c6d77c6958f 100644 --- a/public/app/features/trails/MetricSelect/api.ts +++ b/public/app/features/trails/MetricSelect/api.ts @@ -1,17 +1,48 @@ -import { RawTimeRange } from '@grafana/data'; +import { AdHocVariableFilter, RawTimeRange, Scope } from '@grafana/data'; import { getPrometheusTime } from '@grafana/prometheus/src/language_utils'; -import { getBackendSrv } from '@grafana/runtime'; +import { config, getBackendSrv } from '@grafana/runtime'; -type MetricValuesResponse = { - data: string[]; - status: 'success' | 'error'; - error?: 'string'; - warnings?: string[]; -}; +import { limitOtelMatchTerms } from '../otel/util'; +import { callSuggestionsApi, SuggestionsResponse } from '../utils'; const LIMIT_REACHED = 'results truncated due to limit'; -export async function getMetricNames(dataSourceUid: string, timeRange: RawTimeRange, filters: string, limit?: number) { +export async function getMetricNames( + dataSourceUid: string, + timeRange: RawTimeRange, + scopes: Scope[], + filters: AdHocVariableFilter[], + jobs: string[], + instances: string[], + limit?: number +): Promise { + if (!config.featureToggles.enableScopesInMetricsExplore) { + return await getMetricNamesWithoutScopes(dataSourceUid, timeRange, filters, jobs, instances, limit); + } + + return getMetricNamesWithScopes(dataSourceUid, timeRange, scopes, filters, jobs, instances, limit); +} + +export async function getMetricNamesWithoutScopes( + dataSourceUid: string, + timeRange: RawTimeRange, + adhocFilters: AdHocVariableFilter[], + jobs: string[], + instances: string[], + limit?: number +) { + const matchTerms = adhocFilters.map((filter) => `${filter.key}${filter.operator}"${filter.value}"`); + let missingOtelTargets = false; + + if (jobs.length > 0 && instances.length > 0) { + const otelMatches = limitOtelMatchTerms(matchTerms, jobs, instances); + missingOtelTargets = otelMatches.missingOtelTargets; + matchTerms.push(otelMatches.jobsRegex); + matchTerms.push(otelMatches.instancesRegex); + } + + const filters = `{${matchTerms.join(',')}}`; + const url = `/api/datasources/uid/${dataSourceUid}/resources/api/v1/label/__name__/values`; const params: Record = { start: getPrometheusTime(timeRange.from, false), @@ -20,11 +51,51 @@ export async function getMetricNames(dataSourceUid: string, timeRange: RawTimeRa ...(limit ? { limit } : {}), }; - const response = await getBackendSrv().get(url, params, 'explore-metrics-names'); + const response = await getBackendSrv().get(url, params, 'explore-metrics-names'); if (limit && response.warnings?.includes(LIMIT_REACHED)) { - return { ...response, limitReached: true }; + return { ...response, limitReached: true, missingOtelTargets }; } - return { ...response, limitReached: false }; + return { ...response, limitReached: false, missingOtelTargets }; +} + +export async function getMetricNamesWithScopes( + dataSourceUid: string, + timeRange: RawTimeRange, + scopes: Scope[], + filters: AdHocVariableFilter[], + jobs: string[], + instances: string[], + limit?: number +) { + const response = await callSuggestionsApi( + dataSourceUid, + timeRange, + scopes, + filters, + '__name__', + limit, + 'explore-metrics-names' + ); + + if (jobs.length > 0 && instances.length > 0) { + filters.push({ + key: 'job', + operator: '=~', + value: jobs?.join('|') || '', + }); + + filters.push({ + key: 'instance', + operator: '=~', + value: instances?.join('|') || '', + }); + } + + return { + ...response.data, + limitReached: !!limit && !!response.data.warnings?.includes(LIMIT_REACHED), + missingOtelTargets: false, + }; } diff --git a/public/app/features/trails/otel/api.test.ts b/public/app/features/trails/otel/api.test.ts index f1033344033..104fb4de2cd 100644 --- a/public/app/features/trails/otel/api.test.ts +++ b/public/app/features/trails/otel/api.test.ts @@ -10,7 +10,9 @@ import { } from './api'; jest.mock('@grafana/runtime', () => ({ + ...jest.requireActual('@grafana/runtime'), config: { + ...jest.requireActual('@grafana/runtime').config, publicDashboardAccessToken: '123', }, getBackendSrv: () => { @@ -107,7 +109,7 @@ describe('OTEL API', () => { describe('getDeploymentEnvironments', () => { it('should fetch deployment environments', async () => { - const environments = await getDeploymentEnvironments(dataSourceUid, timeRange); + const environments = await getDeploymentEnvironments(dataSourceUid, timeRange, []); expect(environments).toEqual(['env1', 'env2']); }); diff --git a/public/app/features/trails/otel/api.ts b/public/app/features/trails/otel/api.ts index 0e763d98187..6e1dc9cec61 100644 --- a/public/app/features/trails/otel/api.ts +++ b/public/app/features/trails/otel/api.ts @@ -1,6 +1,8 @@ -import { RawTimeRange } from '@grafana/data'; +import { RawTimeRange, Scope } from '@grafana/data'; import { getPrometheusTime } from '@grafana/prometheus/src/language_utils'; -import { getBackendSrv } from '@grafana/runtime'; +import { config, getBackendSrv } from '@grafana/runtime'; + +import { callSuggestionsApi } from '../utils'; import { OtelResponse, LabelResponse, OtelTargetType } from './types'; import { sortResources } from './util'; @@ -45,9 +47,7 @@ export async function getOtelResources( const response = await getBackendSrv().get(url, params, 'explore-metrics-otel-resources'); // exclude __name__ or deployment_environment or previously chosen filters - const resources = response.data?.filter((resource) => !allExcludedFilters.includes(resource)).map((el: string) => el); - - return resources; + return response.data?.filter((resource) => !allExcludedFilters.includes(resource)).map((el: string) => el); } /** @@ -56,7 +56,7 @@ export async function getOtelResources( * * @param dataSourceUid * @param timeRange - * @param expr + * @param filters * @returns */ export async function totalOtelResources( @@ -99,12 +99,10 @@ export async function totalOtelResources( } }); - const otelTargets: OtelTargetType = { + return { jobs, instances, }; - - return otelTargets; } /** @@ -115,14 +113,9 @@ export async function totalOtelResources( * * @param dataSourceUid * @param timeRange - * @param expr * @returns */ -export async function isOtelStandardization( - dataSourceUid: string, - timeRange: RawTimeRange, - expr?: string -): Promise { +export async function isOtelStandardization(dataSourceUid: string, timeRange: RawTimeRange): Promise { const url = `/api/datasources/uid/${dataSourceUid}/resources/api/v1/query`; const start = getPrometheusTime(timeRange.from, false); @@ -138,9 +131,27 @@ export async function isOtelStandardization( const response = await getBackendSrv().get(url, paramsTargets, 'explore-metrics-otel-check-standard'); // the response should be not greater than zero if it is standard - const checkStandard = !(response.data.result.length > 0); + return !(response.data.result.length > 0); +} - return checkStandard; +/** + * Query the DS for deployment environment label values. + * + * @param dataSourceUid + * @param timeRange + * @param scopes + * @returns string[], values for the deployment_environment label + */ +export async function getDeploymentEnvironments( + dataSourceUid: string, + timeRange: RawTimeRange, + scopes: Scope[] +): Promise { + if (!config.featureToggles.enableScopesInMetricsExplore) { + return getDeploymentEnvironmentsWithoutScopes(dataSourceUid, timeRange); + } + + return getDeploymentEnvironmentsWithScopes(dataSourceUid, timeRange, scopes); } /** @@ -150,7 +161,10 @@ export async function isOtelStandardization( * @param timeRange * @returns string[], values for the deployment_environment label */ -export async function getDeploymentEnvironments(dataSourceUid: string, timeRange: RawTimeRange): Promise { +export async function getDeploymentEnvironmentsWithoutScopes( + dataSourceUid: string, + timeRange: RawTimeRange +): Promise { const start = getPrometheusTime(timeRange.from, false); const end = getPrometheusTime(timeRange.to, true); @@ -168,9 +182,39 @@ export async function getDeploymentEnvironments(dataSourceUid: string, timeRange ); // exclude __name__ or deployment_environment or previously chosen filters - const resources = response.data; + return response.data; +} - return resources; +/** + * Query the DS for deployment environment label values. + * + * @param dataSourceUid + * @param timeRange + * @param scopes + * @returns string[], values for the deployment_environment label + */ +export async function getDeploymentEnvironmentsWithScopes( + dataSourceUid: string, + timeRange: RawTimeRange, + scopes: Scope[] +): Promise { + const response = await callSuggestionsApi( + dataSourceUid, + timeRange, + scopes, + [ + { + key: '__name__', + operator: '=', + value: 'target_info', + }, + ], + 'deployment_environment', + undefined, + 'explore-metrics-otel-resources-deployment-env' + ); + // exclude __name__ or deployment_environment or previously chosen filters + return response.data.data; } /** diff --git a/public/app/features/trails/otel/util.ts b/public/app/features/trails/otel/util.ts index 50cf26daf6d..dc76fbeb623 100644 --- a/public/app/features/trails/otel/util.ts +++ b/public/app/features/trails/otel/util.ts @@ -153,15 +153,14 @@ export function getOtelResourcesObject(scene: SceneObject, firstQueryVal?: strin * @param matchTerms __name__ and other Prom filters * @param jobsList list of jobs in target_info * @param instancesList list of instances in target_info - * @param missingOtelTargets flag to indicate truncated job and instance filters * @returns */ export function limitOtelMatchTerms( matchTerms: string[], jobsList: string[], - instancesList: string[], - missingOtelTargets: boolean + instancesList: string[] ): { missingOtelTargets: boolean; jobsRegex: string; instancesRegex: string } { + let missingOtelTargets = false; const charLimit = 2000; let initialCharAmount = matchTerms.join(',').length; diff --git a/public/app/features/trails/otel/utils.test.ts b/public/app/features/trails/otel/utils.test.ts index eaa54133afd..d8f72ecf587 100644 --- a/public/app/features/trails/otel/utils.test.ts +++ b/public/app/features/trails/otel/utils.test.ts @@ -152,9 +152,7 @@ describe('limitOtelMatchTerms', () => { const jobs = ['a', 'b', 'c']; const instances = ['d', 'e', 'f']; - const missingOtelTargets = false; - - const result = limitOtelMatchTerms(promMatchTerms, jobs, instances, missingOtelTargets); + const result = limitOtelMatchTerms(promMatchTerms, jobs, instances); expect(result.missingOtelTargets).toEqual(true); expect(result.jobsRegex).toEqual('job=~"a"'); @@ -179,9 +177,7 @@ describe('limitOtelMatchTerms', () => { const jobs = ['a', 'b', 'c']; const instances = ['d', 'e', 'f']; - const missingOtelTargets = false; - - const result = limitOtelMatchTerms(promMatchTerms, jobs, instances, missingOtelTargets); + const result = limitOtelMatchTerms(promMatchTerms, jobs, instances); expect(result.missingOtelTargets).toEqual(true); expect(result.jobsRegex).toEqual('job=~"a|b"'); @@ -195,9 +191,7 @@ describe('limitOtelMatchTerms', () => { const instances = ['instance1', 'instance2', 'instance3', 'instance4', 'instance5']; - const missingOtelTargets = false; - - const result = limitOtelMatchTerms(promMatchTerms, jobs, instances, missingOtelTargets); + const result = limitOtelMatchTerms(promMatchTerms, jobs, instances); expect(result.missingOtelTargets).toEqual(false); expect(result.jobsRegex).toEqual('job=~"job1|job2|job3|job4|job5"'); diff --git a/public/app/features/trails/shared.ts b/public/app/features/trails/shared.ts index a245e3c07ec..7edc2a5faf6 100644 --- a/public/app/features/trails/shared.ts +++ b/public/app/features/trails/shared.ts @@ -1,4 +1,4 @@ -import { BusEventWithPayload } from '@grafana/data'; +import { BusEventBase, BusEventWithPayload } from '@grafana/data'; import { ConstantVariable, SceneObject } from '@grafana/scenes'; import { VariableHide } from '@grafana/schema'; @@ -73,3 +73,7 @@ export function getVariablesWithOtelJoinQueryConstant(otelJoinQuery: string) { export class MetricSelectedEvent extends BusEventWithPayload { public static type = 'metric-selected-event'; } + +export class RefreshMetricsEvent extends BusEventBase { + public static type = 'refresh-metrics-event'; +} diff --git a/public/app/features/trails/utils.test.ts b/public/app/features/trails/utils.test.ts index 832b0173faa..e360d1c4da3 100644 --- a/public/app/features/trails/utils.test.ts +++ b/public/app/features/trails/utils.test.ts @@ -1,11 +1,13 @@ import { AdHocFiltersVariable } from '@grafana/scenes'; +import { DataTrail } from './DataTrail'; import { MetricDatasourceHelper } from './helpers/MetricDatasourceHelper'; import { limitAdhocProviders } from './utils'; describe('limitAdhocProviders', () => { let filtersVariable: AdHocFiltersVariable; let datasourceHelper: MetricDatasourceHelper; + let dataTrail: DataTrail; beforeEach(() => { // disable console.log called in Scenes for this test @@ -22,6 +24,10 @@ describe('limitAdhocProviders', () => { getTagKeys: jest.fn().mockResolvedValue(Array(20000).fill({ text: 'key' })), getTagValues: jest.fn().mockResolvedValue(Array(20000).fill({ text: 'value' })), } as unknown as MetricDatasourceHelper; + + dataTrail = { + getQueries: jest.fn().mockReturnValue([]), + } as unknown as DataTrail; }); afterAll(() => { @@ -29,7 +35,7 @@ describe('limitAdhocProviders', () => { }); it('should limit the number of tag keys returned in the variable to 10000', async () => { - limitAdhocProviders(filtersVariable, datasourceHelper); + limitAdhocProviders(dataTrail, filtersVariable, datasourceHelper); if (filtersVariable instanceof AdHocFiltersVariable && filtersVariable.state.getTagKeysProvider) { console.log = jest.fn(); @@ -41,7 +47,7 @@ describe('limitAdhocProviders', () => { }); it('should limit the number of tag values returned in the variable to 10000', async () => { - limitAdhocProviders(filtersVariable, datasourceHelper); + limitAdhocProviders(dataTrail, filtersVariable, datasourceHelper); if (filtersVariable instanceof AdHocFiltersVariable && filtersVariable.state.getTagValuesProvider) { const result = await filtersVariable.state.getTagValuesProvider(filtersVariable, { diff --git a/public/app/features/trails/utils.ts b/public/app/features/trails/utils.ts index 66bd90c30cc..317271dfbd6 100644 --- a/public/app/features/trails/utils.ts +++ b/public/app/features/trails/utils.ts @@ -1,5 +1,17 @@ -import { AdHocVariableFilter, GetTagResponse, MetricFindValue, urlUtil } from '@grafana/data'; -import { config, getDataSourceSrv } from '@grafana/runtime'; +import { lastValueFrom } from 'rxjs'; + +import { + AdHocVariableFilter, + GetTagResponse, + MetricFindValue, + RawTimeRange, + Scope, + scopeFilterOperatorMap, + ScopeSpecFilter, + urlUtil, +} from '@grafana/data'; +import { getPrometheusTime } from '@grafana/prometheus/src/language_utils'; +import { config, FetchResponse, getBackendSrv, getDataSourceSrv } from '@grafana/runtime'; import { AdHocFiltersVariable, sceneGraph, @@ -11,6 +23,7 @@ import { SceneVariable, SceneVariableState, } from '@grafana/scenes'; +import { getClosestScopesFacade } from 'app/features/scopes'; import { getDatasourceSrv } from '../plugins/datasource_srv'; @@ -129,10 +142,12 @@ const MAX_ADHOC_VARIABLE_OPTIONS = 10000; * The current provider functions for adhoc filter variables are the functions getTagKeys and getTagValues in the data source. * This function still uses these functions from inside the data source helper. * + * @param dataTrail * @param filtersVariable * @param datasourceHelper */ export function limitAdhocProviders( + dataTrail: DataTrail, filtersVariable: SceneVariable | null, datasourceHelper: MetricDatasourceHelper ) { @@ -155,7 +170,22 @@ export function limitAdhocProviders( // as the series match[] parameter in Prometheus labels endpoint const filters = filtersVariable.state.filters; // call getTagKeys and truncate the response - const values = (await datasourceHelper.getTagKeys({ filters })).slice(0, MAX_ADHOC_VARIABLE_OPTIONS); + // we're passing the queries so we get the labels that adhere to the queries + // we're also passing the scopes so we get the labels that adhere to the scopes filters + + const opts = { + filters, + scopes: getClosestScopesFacade(variable)?.value, + queries: dataTrail.getQueries(), + }; + + // if there are too many queries it takes to much time to process the requests. + // In this case we favour responsiveness over reducing the number of options. + if (opts.queries.length > 20) { + opts.queries = []; + } + + const values = (await datasourceHelper.getTagKeys(opts)).slice(0, MAX_ADHOC_VARIABLE_OPTIONS); // use replace: true to override the default lookup in adhoc filter variable return { replace: true, values }; }, @@ -175,12 +205,73 @@ export function limitAdhocProviders( // remove current selected filter if updating a chosen filter const filters = filtersValues.filter((f) => f.key !== filter.key); // call getTagValues and truncate the response - const values = (await datasourceHelper.getTagValues({ key: filter.key, filters })).slice( - 0, - MAX_ADHOC_VARIABLE_OPTIONS - ); + // we're passing the queries so we get the label values that adhere to the queries + // we're also passing the scopes so we get the label values that adhere to the scopes filters + + const opts = { + key: filter.key, + filters, + scopes: getClosestScopesFacade(variable)?.value, + queries: dataTrail.getQueries(), + }; + + // if there are too many queries it takes to much time to process the requests. + // In this case we favour responsiveness over reducing the number of options. + if (opts.queries.length > 20) { + opts.queries = []; + } + + const values = (await datasourceHelper.getTagValues(opts)).slice(0, MAX_ADHOC_VARIABLE_OPTIONS); // use replace: true to override the default lookup in adhoc filter variable return { replace: true, values }; }, }); } + +export type SuggestionsResponse = { + data: string[]; + status: 'success' | 'error'; + error?: 'string'; + warnings?: string[]; +}; + +// Suggestions API is an API that receives adhoc filters, scopes and queries and returns the labels or label values that match the provided parameters +// Under the hood it does exactly what the label and label values API where doing but the processing is done in the BE rather than in the FE +export async function callSuggestionsApi( + dataSourceUid: string, + timeRange: RawTimeRange, + scopes: Scope[], + adHocVariableFilters: AdHocVariableFilter[], + labelName: string | undefined, + limit: number | undefined, + requestId: string +): Promise> { + return await lastValueFrom( + getBackendSrv().fetch({ + url: `/api/datasources/uid/${dataSourceUid}/resources/suggestions`, + data: { + labelName, + queries: [], + scopes: scopes.reduce((acc, scope) => { + acc.push(...scope.spec.filters); + + return acc; + }, []), + adhocFilters: adHocVariableFilters.map((filter) => ({ + key: filter.key, + operator: scopeFilterOperatorMap[filter.operator], + value: filter.value, + values: filter.values, + })), + start: getPrometheusTime(timeRange.from, false).toString(), + end: getPrometheusTime(timeRange.to, true).toString(), + limit, + }, + requestId, + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + }) + ); +}