From 1347d85b5032aef1f51f2584c1283a9e32c1f1b1 Mon Sep 17 00:00:00 2001 From: Domas Date: Tue, 18 Jul 2023 13:11:12 +0300 Subject: [PATCH] Tempo Service Map: Optionally group nodes also by service.namespace (#70803) * tempo service map: optionally handle namespace * undo escape attempt * regen cue --- .../dataquery/x/TempoDataQuery_types.gen.ts | 4 + .../kinds/dataquery/types_dataquery_gen.go | 3 + .../plugins/datasource/tempo/dataquery.cue | 2 + .../plugins/datasource/tempo/dataquery.gen.ts | 4 + .../plugins/datasource/tempo/datasource.ts | 4 +- .../datasource/tempo/graphTransform.test.ts | 131 +++++++++++++----- .../datasource/tempo/graphTransform.ts | 38 +++-- public/app/plugins/panel/nodeGraph/Node.tsx | 6 +- 8 files changed, 138 insertions(+), 54 deletions(-) diff --git a/packages/grafana-schema/src/raw/composable/tempo/dataquery/x/TempoDataQuery_types.gen.ts b/packages/grafana-schema/src/raw/composable/tempo/dataquery/x/TempoDataQuery_types.gen.ts index 3ebd5260ebc..71cd1b3247b 100644 --- a/packages/grafana-schema/src/raw/composable/tempo/dataquery/x/TempoDataQuery_types.gen.ts +++ b/packages/grafana-schema/src/raw/composable/tempo/dataquery/x/TempoDataQuery_types.gen.ts @@ -35,6 +35,10 @@ export interface TempoQuery extends common.DataQuery { * Logfmt query to filter traces by their tags. Example: http.status_code=200 error=true */ search?: string; + /** + * Use service.namespace in addition to service.name to uniquely identify a service. + */ + serviceMapIncludeNamespace?: boolean; /** * Filters to be included in a PromQL query to select data for the service graph. Example: {client="app",service="app"} */ diff --git a/pkg/tsdb/tempo/kinds/dataquery/types_dataquery_gen.go b/pkg/tsdb/tempo/kinds/dataquery/types_dataquery_gen.go index c8eeed17a71..1fafb2c0048 100644 --- a/pkg/tsdb/tempo/kinds/dataquery/types_dataquery_gen.go +++ b/pkg/tsdb/tempo/kinds/dataquery/types_dataquery_gen.go @@ -110,6 +110,9 @@ type TempoQuery struct { // Logfmt query to filter traces by their tags. Example: http.status_code=200 error=true Search *string `json:"search,omitempty"` + // Use service.namespace in addition to service.name to uniquely identify a service. + ServiceMapIncludeNamespace *bool `json:"serviceMapIncludeNamespace,omitempty"` + // Filters to be included in a PromQL query to select data for the service graph. Example: {client="app",service="app"} ServiceMapQuery *string `json:"serviceMapQuery,omitempty"` diff --git a/public/app/plugins/datasource/tempo/dataquery.cue b/public/app/plugins/datasource/tempo/dataquery.cue index e1d4d287fbf..3a77559d9e9 100644 --- a/public/app/plugins/datasource/tempo/dataquery.cue +++ b/public/app/plugins/datasource/tempo/dataquery.cue @@ -44,6 +44,8 @@ composableKinds: DataQuery: { maxDuration?: string // Filters to be included in a PromQL query to select data for the service graph. Example: {client="app",service="app"} serviceMapQuery?: string + // Use service.namespace in addition to service.name to uniquely identify a service. + serviceMapIncludeNamespace?: bool // Defines the maximum number of traces that are returned from Tempo limit?: int64 // Use the streaming API to get partial results as they are available diff --git a/public/app/plugins/datasource/tempo/dataquery.gen.ts b/public/app/plugins/datasource/tempo/dataquery.gen.ts index 814d1a3dc57..3e353f16ad0 100644 --- a/public/app/plugins/datasource/tempo/dataquery.gen.ts +++ b/public/app/plugins/datasource/tempo/dataquery.gen.ts @@ -32,6 +32,10 @@ export interface TempoQuery extends common.DataQuery { * Logfmt query to filter traces by their tags. Example: http.status_code=200 error=true */ search?: string; + /** + * Use service.namespace in addition to service.name to uniquely identify a service. + */ + serviceMapIncludeNamespace?: boolean; /** * Filters to be included in a PromQL query to select data for the service graph. Example: {client="app",service="app"} */ diff --git a/public/app/plugins/datasource/tempo/datasource.ts b/public/app/plugins/datasource/tempo/datasource.ts index c3d7023036c..bc72734c7ea 100644 --- a/public/app/plugins/datasource/tempo/datasource.ts +++ b/public/app/plugins/datasource/tempo/datasource.ts @@ -788,12 +788,14 @@ function makePromServiceMapRequest(options: DataQueryRequest): DataQ return { ...options, targets: serviceMapMetrics.map((metric) => { + const { serviceMapQuery, serviceMapIncludeNamespace: serviceMapIncludeNamespace } = options.targets[0]; + const extraSumByFields = serviceMapIncludeNamespace ? ', client_service_namespace, server_service_namespace' : ''; return { format: 'table', refId: metric, // options.targets[0] is not correct here, but not sure what should happen if you have multiple queries for // service map at the same time anyway - expr: `sum by (client, server) (rate(${metric}${options.targets[0].serviceMapQuery || ''}[$__range]))`, + expr: `sum by (client, server${extraSumByFields}) (rate(${metric}${serviceMapQuery || ''}[$__range]))`, instant: true, }; }), diff --git a/public/app/plugins/datasource/tempo/graphTransform.test.ts b/public/app/plugins/datasource/tempo/graphTransform.test.ts index a81f776de51..7411e9a75c9 100644 --- a/public/app/plugins/datasource/tempo/graphTransform.test.ts +++ b/public/app/plugins/datasource/tempo/graphTransform.test.ts @@ -72,6 +72,7 @@ it('assigns correct field type even if values are numbers', async () => { expect(nodes.fields).toMatchObject([ { name: 'id', values: ['0', '1'], type: FieldType.string }, { name: 'title', values: ['0', '1'], type: FieldType.string }, + { name: 'subtitle', type: FieldType.string, values: [] }, { name: 'mainstat', values: [NaN, NaN], type: FieldType.number }, { name: 'secondarystat', values: [10, 20], type: FieldType.number }, { name: 'arc__success', values: [1, 1], type: FieldType.number }, @@ -86,7 +87,7 @@ describe('mapPromMetricsToServiceMap', () => { to: dateTime('2000-01-01T00:01:00'), }; const { nodes, edges } = mapPromMetricsToServiceMap( - [{ data: [totalsPromMetric, secondsPromMetric, failedPromMetric] }], + [{ data: [totalsPromMetric(), secondsPromMetric(), failedPromMetric()] }], { ...range, raw: range, @@ -96,6 +97,7 @@ describe('mapPromMetricsToServiceMap', () => { expect(nodes.fields).toMatchObject([ { name: 'id', values: ['db', 'app', 'lb'] }, { name: 'title', values: ['db', 'app', 'lb'] }, + { name: 'subtitle', values: [] }, { name: 'mainstat', values: [1000, 2000, NaN] }, { name: 'secondarystat', values: [10, 20, NaN] }, { name: 'arc__success', values: [0.8, 0.25, 1] }, @@ -110,6 +112,37 @@ describe('mapPromMetricsToServiceMap', () => { ]); }); + it('transforms prom metrics to service graph inlucding namespace', async () => { + const range = { + from: dateTime('2000-01-01T00:00:00'), + to: dateTime('2000-01-01T00:01:00'), + }; + const { nodes, edges } = mapPromMetricsToServiceMap( + [{ data: [totalsPromMetric(true), secondsPromMetric(true), failedPromMetric(true)] }], + { + ...range, + raw: range, + } + ); + + expect(nodes.fields).toMatchObject([ + { name: 'id', values: ['ns3/db', 'ns1/app', 'ns2/lb'] }, + { name: 'title', values: ['db', 'app', 'lb'] }, + { name: 'subtitle', values: ['ns3', 'ns1', 'ns2'] }, + { name: 'mainstat', values: [1000, 2000, NaN] }, + { name: 'secondarystat', values: [10, 20, NaN] }, + { name: 'arc__success', values: [0.8, 0.25, 1] }, + { name: 'arc__failed', values: [0.2, 0.75, 0] }, + ]); + expect(edges.fields).toMatchObject([ + { name: 'id', values: ['ns1/app_ns3/db', 'ns2/lb_ns1/app'] }, + { name: 'source', values: ['ns1/app', 'ns2/lb'] }, + { name: 'target', values: ['ns3/db', 'ns1/app'] }, + { name: 'mainstat', values: [1000, 2000] }, + { name: 'secondarystat', values: [10, 20] }, + ]); + }); + it('handles invalid failed count', () => { // If node.failed > node.total, the stat circle will render in the wrong position // Fixed this by limiting the failed value to the total value @@ -118,7 +151,7 @@ describe('mapPromMetricsToServiceMap', () => { to: dateTime('2000-01-01T00:01:00'), }; const { nodes } = mapPromMetricsToServiceMap( - [{ data: [totalsPromMetric, secondsPromMetric, invalidFailedPromMetric] }], + [{ data: [totalsPromMetric(), secondsPromMetric(), invalidFailedPromMetric] }], { ...range, raw: range, @@ -128,6 +161,7 @@ describe('mapPromMetricsToServiceMap', () => { expect(nodes.fields).toMatchObject([ { name: 'id', values: ['db', 'app', 'lb'] }, { name: 'title', values: ['db', 'app', 'lb'] }, + { name: 'subtitle', values: [] }, { name: 'mainstat', values: [1000, 2000, NaN] }, { name: 'secondarystat', values: [10, 20, NaN] }, { name: 'arc__success', values: [0, 0, 1] }, @@ -160,44 +194,65 @@ const missingSpanResponse = createDataFrame({ ], }); -const totalsPromMetric = createDataFrame({ - refId: 'traces_service_graph_request_total', - fields: [ - { name: 'Time', values: [1628169788000, 1628169788000] }, - { name: 'client', values: ['app', 'lb'] }, - { name: 'instance', values: ['127.0.0.1:12345', '127.0.0.1:12345'] }, - { name: 'job', values: ['local_scrape', 'local_scrape'] }, - { name: 'server', values: ['db', 'app'] }, - { name: 'tempo_config', values: ['default', 'default'] }, - { name: 'Value #traces_service_graph_request_total', values: [10, 20] }, - ], -}); +const totalsPromMetric = (namespace?: boolean) => + createDataFrame({ + refId: 'traces_service_graph_request_total', + fields: [ + { name: 'Time', values: [1628169788000, 1628169788000] }, + { name: 'client', values: ['app', 'lb'] }, + { name: 'instance', values: ['127.0.0.1:12345', '127.0.0.1:12345'] }, + { name: 'job', values: ['local_scrape', 'local_scrape'] }, + { name: 'server', values: ['db', 'app'] }, + { name: 'tempo_config', values: ['default', 'default'] }, + { name: 'Value #traces_service_graph_request_total', values: [10, 20] }, + ...(namespace + ? [ + { name: 'client_service_namespace', values: ['ns1', 'ns2'] }, + { name: 'server_service_namespace', values: ['ns3', 'ns1'] }, + ] + : []), + ], + }); -const secondsPromMetric = createDataFrame({ - refId: 'traces_service_graph_request_server_seconds_sum', - fields: [ - { name: 'Time', values: [1628169788000, 1628169788000] }, - { name: 'client', values: ['app', 'lb'] }, - { name: 'instance', values: ['127.0.0.1:12345', '127.0.0.1:12345'] }, - { name: 'job', values: ['local_scrape', 'local_scrape'] }, - { name: 'server', values: ['db', 'app'] }, - { name: 'tempo_config', values: ['default', 'default'] }, - { name: 'Value #traces_service_graph_request_server_seconds_sum', values: [10, 40] }, - ], -}); +const secondsPromMetric = (namespace?: boolean) => + createDataFrame({ + refId: 'traces_service_graph_request_server_seconds_sum', + fields: [ + { name: 'Time', values: [1628169788000, 1628169788000] }, + { name: 'client', values: ['app', 'lb'] }, + { name: 'instance', values: ['127.0.0.1:12345', '127.0.0.1:12345'] }, + { name: 'job', values: ['local_scrape', 'local_scrape'] }, + { name: 'server', values: ['db', 'app'] }, + { name: 'tempo_config', values: ['default', 'default'] }, + { name: 'Value #traces_service_graph_request_server_seconds_sum', values: [10, 40] }, + ...(namespace + ? [ + { name: 'client_service_namespace', values: ['ns1', 'ns2'] }, + { name: 'server_service_namespace', values: ['ns3', 'ns1'] }, + ] + : []), + ], + }); -const failedPromMetric = createDataFrame({ - refId: 'traces_service_graph_request_failed_total', - fields: [ - { name: 'Time', values: [1628169788000, 1628169788000] }, - { name: 'client', values: ['app', 'lb'] }, - { name: 'instance', values: ['127.0.0.1:12345', '127.0.0.1:12345'] }, - { name: 'job', values: ['local_scrape', 'local_scrape'] }, - { name: 'server', values: ['db', 'app'] }, - { name: 'tempo_config', values: ['default', 'default'] }, - { name: 'Value #traces_service_graph_request_failed_total', values: [2, 15] }, - ], -}); +const failedPromMetric = (namespace?: boolean) => + createDataFrame({ + refId: 'traces_service_graph_request_failed_total', + fields: [ + { name: 'Time', values: [1628169788000, 1628169788000] }, + { name: 'client', values: ['app', 'lb'] }, + { name: 'instance', values: ['127.0.0.1:12345', '127.0.0.1:12345'] }, + { name: 'job', values: ['local_scrape', 'local_scrape'] }, + { name: 'server', values: ['db', 'app'] }, + { name: 'tempo_config', values: ['default', 'default'] }, + { name: 'Value #traces_service_graph_request_failed_total', values: [2, 15] }, + ...(namespace + ? [ + { name: 'client_service_namespace', values: ['ns1', 'ns2'] }, + { name: 'server_service_namespace', values: ['ns3', 'ns1'] }, + ] + : []), + ], + }); const invalidFailedPromMetric = createDataFrame({ refId: 'traces_service_graph_request_failed_total', diff --git a/public/app/plugins/datasource/tempo/graphTransform.ts b/public/app/plugins/datasource/tempo/graphTransform.ts index eb36c6f8737..8c540eb66c6 100644 --- a/public/app/plugins/datasource/tempo/graphTransform.ts +++ b/public/app/plugins/datasource/tempo/graphTransform.ts @@ -173,7 +173,7 @@ export function mapPromMetricsToServiceMap( const frames = getMetricFrames(responses); // First just collect data from the metrics into a map with nodes and edges as keys - const nodesMap: Record = {}; + const nodesMap: Record = {}; const edgesMap: Record = {}; // At this moment we don't have any error/success or other counts so we just use these 2 collectMetricData(frames[totalsMetric], 'total', totalsMetric, nodesMap, edgesMap); @@ -191,6 +191,7 @@ function createServiceMapDataFrames() { const nodes = createDF('Nodes', [ { name: Fields.id, type: FieldType.string }, { name: Fields.title, type: FieldType.string, config: { displayName: 'Service name' } }, + { name: Fields.subTitle, type: FieldType.string, config: { displayName: 'Service namespace' } }, { name: Fields.mainStat, type: FieldType.number, config: { unit: 'ms/r', displayName: 'Average response time' } }, { name: Fields.secondaryStat, @@ -241,6 +242,11 @@ type ServiceMapStatistics = { failed?: number; }; +type NodeObject = ServiceMapStatistics & { + name: string; + namespace?: string; +}; + type EdgeObject = ServiceMapStatistics & { source: string; target: string; @@ -261,7 +267,7 @@ function collectMetricData( frame: DataFrameView | undefined, stat: keyof ServiceMapStatistics, metric: string, - nodesMap: Record, + nodesMap: Record, edgesMap: Record ) { if (!frame) { @@ -274,13 +280,16 @@ function collectMetricData( for (let i = 0; i < frame.length; i++) { const row = frame.get(i); - const edgeId = `${row.client}_${row.server}`; + const serverId = row.server_service_namespace ? `${row.server_service_namespace}/${row.server}` : row.server; + const clientId = row.client_service_namespace ? `${row.client_service_namespace}/${row.client}` : row.client; + + const edgeId = `${clientId}_${serverId}`; if (!edgesMap[edgeId]) { // Create edge as it does not exist yet edgesMap[edgeId] = { - target: row.server, - source: row.client, + target: serverId, + source: clientId, [stat]: row[valueName], }; } else { @@ -290,20 +299,24 @@ function collectMetricData( edgesMap[edgeId][stat] = (edgesMap[edgeId][stat] || 0) + row[valueName]; } - if (!nodesMap[row.server]) { + if (!nodesMap[serverId]) { // Create node for server - nodesMap[row.server] = { + nodesMap[serverId] = { + name: row.server, + namespace: row.server_service_namespace, [stat]: row[valueName], }; } else { // Add stat to server node. Sum up values if there are multiple edges targeting this server node. - nodesMap[row.server][stat] = (nodesMap[row.server][stat] || 0) + row[valueName]; + nodesMap[serverId][stat] = (nodesMap[serverId][stat] || 0) + row[valueName]; } - if (!nodesMap[row.client]) { + if (!nodesMap[clientId]) { // Create the client node but don't add the stat as edge stats are attributed to the server node. This means for // example that the number of requests in a node show how many requests it handled not how many it generated. - nodesMap[row.client] = { + nodesMap[clientId] = { + name: row.client, + namespace: row.client_service_namespace, [stat]: 0, }; } @@ -311,7 +324,7 @@ function collectMetricData( } function convertToDataFrames( - nodesMap: Record, + nodesMap: Record, edgesMap: Record, range: TimeRange ): { nodes: DataFrame; edges: DataFrame } { @@ -320,7 +333,8 @@ function convertToDataFrames( const node = nodesMap[nodeId]; nodes.add({ [Fields.id]: nodeId, - [Fields.title]: nodeId, + [Fields.title]: node.name, + [Fields.subTitle]: node.namespace, // NaN will not be shown in the node graph. This happens for a root client node which did not process // any requests itself. [Fields.mainStat]: node.total ? (node.seconds! / node.total) * 1000 : Number.NaN, // Average response time diff --git a/public/app/plugins/panel/nodeGraph/Node.tsx b/public/app/plugins/panel/nodeGraph/Node.tsx index a95b5c687ad..0b96f20a491 100644 --- a/public/app/plugins/panel/nodeGraph/Node.tsx +++ b/public/app/plugins/panel/nodeGraph/Node.tsx @@ -41,7 +41,7 @@ const getStyles = (theme: GrafanaTheme2, hovering: HoverState) => ({ overflow: hidden; white-space: nowrap; background-color: ${tinycolor(theme.colors.background.primary).setAlpha(0.6).toHex8String()}; - width: 100px; + width: 140px; `, statsText: css` @@ -90,9 +90,9 @@ export const Node = memo(function Node(props: {