From 974103c6fa20701c5467c67fb24f173aea9113fb Mon Sep 17 00:00:00 2001 From: Matias Chomicki Date: Wed, 16 Jul 2025 17:42:14 +0200 Subject: [PATCH] OTel: Add experimental formatting for OTel identified log lines (#108170) * OTel: create formats service * LogListContext: use OTel service to resolve the default displayed fields * Create otelLogsFormatting feature flag * LogListContext: conditionally use otel format based on feature flag * OTel logs: reformat body and improve displayed fields * formats: remove duplicated scope name * Update formats.ts * format: use regex rules to exclude fields * Add tests * Formatting * LogsPanel: expose setDisplayedFields * LogListContext: refactor otel displayed fields * Prettier * format: optimize identifyOTelLanguages --- .../src/types/featureToggles.gen.ts | 4 + .../logs/panelcfg/x/LogsPanelCfg_types.gen.ts | 1 + pkg/services/featuremgmt/registry.go | 7 ++ pkg/services/featuremgmt/toggles_gen.csv | 1 + pkg/services/featuremgmt/toggles_gen.go | 4 + pkg/services/featuremgmt/toggles_gen.json | 13 ++++ .../logs/components/otel/formats.test.ts | 49 ++++++++++++ .../features/logs/components/otel/formats.ts | 77 +++++++++++++++++++ .../logs/components/panel/LogListContext.tsx | 12 +++ .../logs/components/panel/processing.test.ts | 41 ++++++++++ .../logs/components/panel/processing.ts | 9 ++- public/app/plugins/panel/logs/LogsPanel.tsx | 9 ++- public/app/plugins/panel/logs/panelcfg.cue | 1 + public/app/plugins/panel/logs/panelcfg.gen.ts | 1 + public/app/plugins/panel/logs/types.ts | 17 ++-- 15 files changed, 237 insertions(+), 9 deletions(-) create mode 100644 public/app/features/logs/components/otel/formats.test.ts create mode 100644 public/app/features/logs/components/otel/formats.ts diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index 1832b95666f..8f3a12c6703 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -1026,4 +1026,8 @@ export interface FeatureToggles { * @default false */ enablePluginImporter?: boolean; + /** + * Applies OTel formatting templates to displayed logs + */ + otelLogsFormatting?: boolean; } diff --git a/packages/grafana-schema/src/raw/composable/logs/panelcfg/x/LogsPanelCfg_types.gen.ts b/packages/grafana-schema/src/raw/composable/logs/panelcfg/x/LogsPanelCfg_types.gen.ts index 73b7933cd79..956bff430f7 100644 --- a/packages/grafana-schema/src/raw/composable/logs/panelcfg/x/LogsPanelCfg_types.gen.ts +++ b/packages/grafana-schema/src/raw/composable/logs/panelcfg/x/LogsPanelCfg_types.gen.ts @@ -36,6 +36,7 @@ export interface Options { onLogOptionsChange?: unknown; onNewLogsReceived?: unknown; prettifyLogMessage: boolean; + setDisplayedFields?: unknown; showCommonLabels: boolean; showControls?: boolean; showLabels: boolean; diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index a82c3a77dbd..57084590310 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -1765,6 +1765,13 @@ var ( FrontendOnly: true, Expression: "false", }, + { + Name: "otelLogsFormatting", + Description: "Applies OTel formatting templates to displayed logs", + Stage: FeatureStageExperimental, + FrontendOnly: true, + Owner: grafanaObservabilityLogsSquad, + }, } ) diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index 81d6bf376f7..d64acfc637e 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -229,3 +229,4 @@ newInfluxDSConfigPageDesign,privatePreview,@grafana/partner-datasources,false,fa enableAppChromeExtensions,experimental,@grafana/plugins-platform-backend,false,false,true foldersAppPlatformAPI,experimental,@grafana/grafana-search-navigate-organise,false,false,true enablePluginImporter,experimental,@grafana/plugins-platform-backend,false,false,true +otelLogsFormatting,experimental,@grafana/observability-logs,false,false,true diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index 6069be22df0..197226456c5 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -926,4 +926,8 @@ const ( // FlagEnablePluginImporter // Set this to true to use the new PluginImporter functionality FlagEnablePluginImporter = "enablePluginImporter" + + // FlagOtelLogsFormatting + // Applies OTel formatting templates to displayed logs + FlagOtelLogsFormatting = "otelLogsFormatting" ) diff --git a/pkg/services/featuremgmt/toggles_gen.json b/pkg/services/featuremgmt/toggles_gen.json index 793dd264e1c..39af6383752 100644 --- a/pkg/services/featuremgmt/toggles_gen.json +++ b/pkg/services/featuremgmt/toggles_gen.json @@ -2225,6 +2225,19 @@ "expression": "true" } }, + { + "metadata": { + "name": "otelLogsFormatting", + "resourceVersion": "1750152612973", + "creationTimestamp": "2025-06-17T09:30:12Z" + }, + "spec": { + "description": "Applies OTel formatting templates to displayed logs", + "stage": "experimental", + "codeowner": "@grafana/observability-logs", + "frontend": true + } + }, { "metadata": { "name": "panelFilterVariable", diff --git a/public/app/features/logs/components/otel/formats.test.ts b/public/app/features/logs/components/otel/formats.test.ts new file mode 100644 index 00000000000..286654f8995 --- /dev/null +++ b/public/app/features/logs/components/otel/formats.test.ts @@ -0,0 +1,49 @@ +import { LOG_LINE_BODY_FIELD_NAME } from '../LogDetailsBody'; +import { createLogLine } from '../mocks/logRow'; + +import { getDisplayedFieldsForLogs, getOtelFormattedBody } from './formats'; + +describe('getDisplayedFieldsForLogs', () => { + test('Does not return displayed fields if not an OTel log line', () => { + const log = createLogLine({ labels: { place: 'luna' }, entry: `place="luna" 1ms 3 KB` }); + + expect(getDisplayedFieldsForLogs([log])).toEqual([]); + }); + + test('Does not return displayed fields if telemetry_sdk_language is empty', () => { + const log = createLogLine({ labels: { severity_number: '1' }, entry: `place="luna" 1ms 3 KB` }); + + expect(getDisplayedFieldsForLogs([log])).toEqual([]); + }); + + test('Does not return displayed fields if telemetry_sdk_language is empty', () => { + const log = createLogLine({ + labels: { severity_number: '1', telemetry_sdk_language: 'php', scope_name: 'scope' }, + entry: `place="luna" 1ms 3 KB`, + }); + + expect(getDisplayedFieldsForLogs([log])).toEqual(['scope_name', LOG_LINE_BODY_FIELD_NAME]); + }); +}); + +describe('getOtelFormattedBody', () => { + test('Does not modify non OTel logs', () => { + const log = createLogLine({ labels: { place: 'luna' }, entry: `place="luna" 1ms 3 KB` }); + expect(getOtelFormattedBody(log)).toEqual(`place="luna" 1ms 3 KB`); + }); + + test('Returns an OTel augmented log line body', () => { + const log = createLogLine({ + labels: { + severity_number: '1', + telemetry_sdk_language: 'php', + scope_name: 'scope', + aws_ignore: 'ignored', + key: 'value', + otel: 'otel', + }, + entry: `place="luna" 1ms 3 KB`, + }); + expect(getOtelFormattedBody(log)).toEqual(`place="luna" 1ms 3 KB key=value otel=otel`); + }); +}); diff --git a/public/app/features/logs/components/otel/formats.ts b/public/app/features/logs/components/otel/formats.ts new file mode 100644 index 00000000000..640009a39a3 --- /dev/null +++ b/public/app/features/logs/components/otel/formats.ts @@ -0,0 +1,77 @@ +import { LogRowModel } from '@grafana/data'; + +import { LOG_LINE_BODY_FIELD_NAME } from '../LogDetailsBody'; +import { LogListModel } from '../panel/processing'; + +/** + * The presence of this field along log fields determines OTel origin. + */ +const OTEL_PROBE_FIELD = 'severity_number'; +export function identifyOTelLanguages(logs: LogListModel[] | LogRowModel[]): string[] { + const languagesSet = new Set(); + logs.forEach((log) => { + const lang = identifyOTelLanguage(log); + if (lang !== undefined) { + languagesSet.add(lang); + } + }); + return [...languagesSet]; +} + +export function identifyOTelLanguage(log: LogListModel | LogRowModel): string | undefined { + if ('otelLanguage' in log && log.otelLanguage) { + return log.otelLanguage; + } + return log.labels[OTEL_PROBE_FIELD] !== undefined && log.labels.telemetry_sdk_language + ? log.labels.telemetry_sdk_language + : undefined; +} + +export function getDisplayedFieldsForLanguages(logs: LogListModel[] | LogRowModel[], languages: string[]) { + const displayedFields: string[] = []; + + languages.forEach((language) => { + const format = getDisplayFormatForLanguage(language) ?? getDefaultOTelDisplayFormat(); + format.forEach((field) => { + if (!displayedFields.includes(field)) { + displayedFields.push(field); + } + }); + }); + + return displayedFields.filter( + (field) => field === LOG_LINE_BODY_FIELD_NAME || logs.some((log) => log.labels[field] !== undefined) + ); +} + +export function getDisplayedFieldsForLogs(logs: LogListModel[] | LogRowModel[]): string[] { + return getDisplayedFieldsForLanguages(logs, identifyOTelLanguages(logs)); +} + +// Languages not implemented. +export function getDisplayFormatForLanguage(language: string) { + return undefined; +} + +export function getDefaultOTelDisplayFormat() { + return ['scope_name', 'thread_name', 'exception_type', 'exception_message', LOG_LINE_BODY_FIELD_NAME]; +} + +const OTEL_RESOURCE_ATTRS_REGEX = + /^(aws_|cloud_|cloudfoundry_|container_|deployment_|faas_|gcp_|host_|k8s_|os_|process_|service_|telemetry_)/; +const OTEL_LOG_FIELDS_REGEX = + /^(flags|observed_timestamp|scope_name|severity_number|severity_text|span_id|trace_id|detected_level)$/; + +export function getOtelFormattedBody(log: LogListModel) { + if (!log.otelLanguage) { + return log.raw; + } + const additionalFields = Object.keys(log.labels).filter( + (label) => !OTEL_RESOURCE_ATTRS_REGEX.test(label) && !OTEL_LOG_FIELDS_REGEX.test(label) + ); + return ( + log.raw + + ' ' + + additionalFields.map((field) => (log.labels[field] ? `${field}=${log.labels[field]}` : '')).join(' ') + ); +} diff --git a/public/app/features/logs/components/panel/LogListContext.tsx b/public/app/features/logs/components/panel/LogListContext.tsx index d4e8dfea793..4dc79aa2e80 100644 --- a/public/app/features/logs/components/panel/LogListContext.tsx +++ b/public/app/features/logs/components/panel/LogListContext.tsx @@ -22,9 +22,11 @@ import { shallowCompare, store, } from '@grafana/data'; +import { config } from '@grafana/runtime'; import { PopoverContent } from '@grafana/ui'; import { DownloadFormat, checkLogsError, checkLogsSampled, downloadLogs as download } from '../../utils'; +import { getDisplayedFieldsForLogs } from '../otel/formats'; import { LogLineDetailsMode } from './LogLineDetails'; import { GetRowContextQueryFn, LogLineMenuCustomItem } from './LogLineMenu'; @@ -239,6 +241,16 @@ export const LogListContextProvider = ({ const [detailsWidth, setDetailsWidthState] = useState(getDetailsWidth(containerElement, logOptionsStorageKey)); const [detailsMode, setDetailsMode] = useState(detailsModeProp ?? 'sidebar'); + useEffect(() => { + if (displayedFields.length > 0 || !config.featureToggles.otelLogsFormatting || !setDisplayedFields) { + return; + } + const otelDisplayedFields = getDisplayedFieldsForLogs(logs); + if (otelDisplayedFields.length) { + setDisplayedFields(otelDisplayedFields); + } + }, [displayedFields.length, logs, setDisplayedFields]); + useEffect(() => { // Props are updated in the context only of the panel is being externally controlled. if (showControls && app !== CoreApp.PanelEditor) { diff --git a/public/app/features/logs/components/panel/processing.test.ts b/public/app/features/logs/components/panel/processing.test.ts index 7ef61741624..32b2d40b0f0 100644 --- a/public/app/features/logs/components/panel/processing.test.ts +++ b/public/app/features/logs/components/panel/processing.test.ts @@ -1,4 +1,5 @@ import { createTheme, Field, FieldType, LogLevel, LogRowModel, LogsSortOrder, toDataFrame } from '@grafana/data'; +import { config } from '@grafana/runtime'; import { LOG_LINE_BODY_FIELD_NAME } from '../LogDetailsBody'; import { createLogLine, createLogRow } from '../mocks/logRow'; @@ -347,3 +348,43 @@ describe('preProcessLogs', () => { }); }); }); + +describe('OTel logs', () => { + let originalOtelLogsFormatting = config.featureToggles.otelLogsFormatting; + afterAll(() => { + config.featureToggles.otelLogsFormatting = originalOtelLogsFormatting; + }); + + test('Requires a feature flag', () => { + const log = createLogLine({ + labels: { + severity_number: '1', + telemetry_sdk_language: 'php', + scope_name: 'scope', + aws_ignore: 'ignored', + key: 'value', + otel: 'otel', + }, + entry: `place="luna" 1ms 3 KB`, + }); + expect(log.otelLanguage).toBeDefined(); + expect(log.body).toEqual(`place="luna" 1ms 3 KB`); + }); + + test('Augments OTel log lines', () => { + config.featureToggles.otelLogsFormatting = true; + const log = createLogLine({ + labels: { + severity_number: '1', + telemetry_sdk_language: 'php', + scope_name: 'scope', + aws_ignore: 'ignored', + key: 'value', + otel: 'otel', + }, + entry: `place="luna" 1ms 3 KB`, + }); + expect(log.otelLanguage).toBeDefined(); + expect(log.body).toEqual(`place="luna" 1ms 3 KB key=value otel=otel`); + }); +}); diff --git a/public/app/features/logs/components/panel/processing.ts b/public/app/features/logs/components/panel/processing.ts index 0872e241a30..ee6826aa900 100644 --- a/public/app/features/logs/components/panel/processing.ts +++ b/public/app/features/logs/components/panel/processing.ts @@ -3,11 +3,13 @@ import { parse, stringify } from 'lossless-json'; import Prism, { Grammar } from 'prismjs'; import { DataFrame, dateTimeFormat, Labels, LogLevel, LogRowModel, LogsSortOrder, textUtil } from '@grafana/data'; +import { config } from '@grafana/runtime'; import { GetFieldLinksFn } from 'app/plugins/panel/logs/types'; import { checkLogsError, checkLogsSampled, escapeUnescapedString, sortLogRows } from '../../utils'; import { LOG_LINE_BODY_FIELD_NAME } from '../LogDetailsBody'; import { FieldDef, getAllFields } from '../logParser'; +import { identifyOTelLanguage, getOtelFormattedBody } from '../otel/formats'; import { generateLogGrammar, generateTextMatchGrammar } from './grammar'; import { LogLineVirtualization } from './virtualization'; @@ -30,6 +32,7 @@ export class LogListModel implements LogRowModel { isSampled: boolean; labels: Labels; logLevel: LogLevel; + otelLanguage?: string; raw: string; rowIndex: number; rowId?: string | undefined; @@ -69,6 +72,7 @@ export class LogListModel implements LogRowModel { this.isSampled = !!checkLogsSampled(log); this.labels = log.labels; this.logLevel = log.logLevel; + this.otelLanguage = identifyOTelLanguage(log); this.rowIndex = log.rowIndex; this.rowId = log.rowId; this.searchWords = log.searchWords; @@ -115,9 +119,10 @@ export class LogListModel implements LogRowModel { this.raw = parsed; } } catch (error) {} + const raw = config.featureToggles.otelLogsFormatting && this.otelLanguage ? getOtelFormattedBody(this) : this.raw; this._body = this.collapsed - ? this.raw.substring(0, this._virtualization?.getTruncationLength(null) ?? TRUNCATION_DEFAULT_LENGTH) - : this.raw; + ? raw.substring(0, this._virtualization?.getTruncationLength(null) ?? TRUNCATION_DEFAULT_LENGTH) + : raw; if (!this._wrapLogMessage) { this._body = this._body.replace(NEWLINES_REGEX, ''); } diff --git a/public/app/plugins/panel/logs/LogsPanel.tsx b/public/app/plugins/panel/logs/LogsPanel.tsx index 971bb97365f..03f2ffbfc35 100644 --- a/public/app/plugins/panel/logs/LogsPanel.tsx +++ b/public/app/plugins/panel/logs/LogsPanel.tsx @@ -60,6 +60,7 @@ import { isOnLogOptionsChange, isOnNewLogsReceivedType, isReactNodeArray, + isSetDisplayedFields, onNewLogsReceivedType, Options, } from './types'; @@ -91,6 +92,9 @@ interface LogsPanelProps extends PanelProps { * Called from the "eye" icon in Log Details to request hiding the displayed field. If ommited, a default implementation is used. * onClickHideField?: (key: string) => void; * + * Called from the new Log Details Panel when fields are reordered. If ommited, a default implementation is used. + * setDisplayedFields?: (key: string) => void; + * * Passed to the LogRowMenuCell component to be rendered before the default actions in the menu. * logRowMenuIconsBefore?: ReactNode[]; * @@ -517,6 +521,9 @@ export const LogsPanel = ({ const onClickShowField = isOnClickShowField(options.onClickShowField) ? options.onClickShowField : showField; const onClickHideField = isOnClickHideField(options.onClickHideField) ? options.onClickHideField : hideField; + const setDisplayedFieldsFn = isSetDisplayedFields(options.setDisplayedFields) + ? options.setDisplayedFields + : setDisplayedFields; // In Dashboards, default to inline. Otherwise, let apps control or have automatic behavior. const detailsMode = detailsModeProp ? detailsModeProp : app === CoreApp.Dashboard ? 'inline' : undefined; @@ -576,7 +583,7 @@ export const LogsPanel = ({ onOpenContext={onOpenContext} onPermalinkClick={showPermaLink() ? onPermalinkClick : undefined} permalinkedLogId={getLogsPanelState()?.logs?.id ?? undefined} - setDisplayedFields={setDisplayedFields} + setDisplayedFields={setDisplayedFieldsFn} showControls={Boolean(showControls)} showTime={showTime} sortOrder={sortOrder} diff --git a/public/app/plugins/panel/logs/panelcfg.cue b/public/app/plugins/panel/logs/panelcfg.cue index 70ddf33b412..5f7ac972318 100644 --- a/public/app/plugins/panel/logs/panelcfg.cue +++ b/public/app/plugins/panel/logs/panelcfg.cue @@ -55,6 +55,7 @@ composableKinds: PanelCfg: { logLineMenuCustomItems?: _ onNewLogsReceived?: _ displayedFields?: [...string] + setDisplayedFields?: _ } @cuetsy(kind="interface") } }] diff --git a/public/app/plugins/panel/logs/panelcfg.gen.ts b/public/app/plugins/panel/logs/panelcfg.gen.ts index a07151c7fda..3d28ef5ff55 100644 --- a/public/app/plugins/panel/logs/panelcfg.gen.ts +++ b/public/app/plugins/panel/logs/panelcfg.gen.ts @@ -34,6 +34,7 @@ export interface Options { onLogOptionsChange?: unknown; onNewLogsReceived?: unknown; prettifyLogMessage: boolean; + setDisplayedFields?: unknown; showCommonLabels: boolean; showControls?: boolean; showLabels: boolean; diff --git a/public/app/plugins/panel/logs/types.ts b/public/app/plugins/panel/logs/types.ts index 506cc27c4bb..326a0168383 100644 --- a/public/app/plugins/panel/logs/types.ts +++ b/public/app/plugins/panel/logs/types.ts @@ -10,11 +10,12 @@ type onClickFilterLabelType = (key: string, value: string, frame?: DataFrame) => type onClickFilterOutLabelType = (key: string, value: string, frame?: DataFrame) => void; type onClickFilterValueType = (value: string, refId?: string) => void; type onClickFilterOutStringType = (value: string, refId?: string) => void; -type isFilterLabelActiveType = (key: string, value: string, refId?: string) => Promise; -type isOnClickShowFieldType = (value: string) => void; -type isOnClickHideFieldType = (value: string) => void; +type filterLabelActiveType = (key: string, value: string, refId?: string) => Promise; +type onClickShowFieldType = (value: string) => void; +type onClickHideFieldType = (value: string) => void; export type onNewLogsReceivedType = (allLogs: DataFrame[], newLogs: DataFrame[]) => void; type onLogOptionsChangeType = (option: keyof LogListControlOptions, value: string | boolean | string[]) => void; +type setDisplayedFieldsType = (fields: string[]) => void; export type GetFieldLinksFn = ( field: Field, @@ -39,15 +40,19 @@ export function isOnClickFilterOutString(callback: unknown): callback is onClick return typeof callback === 'function'; } -export function isIsFilterLabelActive(callback: unknown): callback is isFilterLabelActiveType { +export function isIsFilterLabelActive(callback: unknown): callback is filterLabelActiveType { return typeof callback === 'function'; } -export function isOnClickShowField(callback: unknown): callback is isOnClickShowFieldType { +export function isOnClickShowField(callback: unknown): callback is onClickShowFieldType { return typeof callback === 'function'; } -export function isOnClickHideField(callback: unknown): callback is isOnClickHideFieldType { +export function isOnClickHideField(callback: unknown): callback is onClickHideFieldType { + return typeof callback === 'function'; +} + +export function isSetDisplayedFields(callback: unknown): callback is setDisplayedFieldsType { return typeof callback === 'function'; }