From dd5c545df93bd854e378a3f37d45469e9c6acceb Mon Sep 17 00:00:00 2001 From: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> Date: Mon, 30 Jun 2025 16:09:44 +0200 Subject: [PATCH] Loki: Remove `lokiStructuredMetadata` feature toggle (#107292) Loki: Remove lokiStructuredMetadata feature toggle --- .../feature-toggles/index.md | 1 - .../src/types/featureToggles.gen.ts | 5 --- pkg/services/featuremgmt/registry.go | 8 ----- pkg/services/featuremgmt/toggles_gen.csv | 1 - pkg/services/featuremgmt/toggles_gen.go | 4 --- pkg/services/featuremgmt/toggles_gen.json | 3 +- pkg/tsdb/loki/api.go | 23 +++++-------- pkg/tsdb/loki/api_mock.go | 10 +++--- pkg/tsdb/loki/api_test.go | 34 +++++++++---------- pkg/tsdb/loki/framing_test.go | 6 ++-- pkg/tsdb/loki/loki.go | 9 +++-- pkg/tsdb/loki/loki_bench_test.go | 2 +- 12 files changed, 41 insertions(+), 65 deletions(-) diff --git a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md index 5c5fb21173b..48a2ca08a92 100644 --- a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md +++ b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md @@ -45,7 +45,6 @@ Most [generally available](https://grafana.com/docs/release-life-cycle/#general- | `panelMonitoring` | Enables panel monitoring through logs and measurements | Yes | | `formatString` | Enable format string transformer | Yes | | `kubernetesClientDashboardsFolders` | Route the folder and dashboard service requests to k8s | Yes | -| `lokiStructuredMetadata` | Enables the loki data source to request structured metadata from the Loki server | Yes | | `addFieldFromCalculationStatFunctions` | Add cumulative and window functions to the add field from calculation transformation | Yes | | `annotationPermissionUpdate` | Change the way annotation permissions work by scoping them to folders and dashboards. | Yes | | `dashboardSceneForViewers` | Enables dashboard rendering using Scenes for viewer roles | Yes | diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index 3d94d8c841d..6350852823a 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -340,11 +340,6 @@ export interface FeatureToggles { */ cloudWatchBatchQueries?: boolean; /** - * Enables the loki data source to request structured metadata from the Loki server - * @default true - */ - lokiStructuredMetadata?: boolean; - /** * If enabled, the caching backend gradually serializes query responses for the cache, comparing against the configured `[caching]max_value_mb` value as it goes. This can can help prevent Grafana from running out of memory while attempting to cache very large query responses. */ cachingOptimizeSerializationMemoryUsage?: boolean; diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index cf8ec3ca8d6..a92e51294b5 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -563,14 +563,6 @@ var ( Stage: FeatureStagePublicPreview, Owner: awsDatasourcesSquad, }, - { - Name: "lokiStructuredMetadata", - Description: "Enables the loki data source to request structured metadata from the Loki server", - Stage: FeatureStageGeneralAvailability, - FrontendOnly: false, - Owner: grafanaObservabilityLogsSquad, - Expression: "true", - }, { Name: "cachingOptimizeSerializationMemoryUsage", Description: "If enabled, the caching backend gradually serializes query responses for the cache, comparing against the configured `[caching]max_value_mb` value as it goes. This can can help prevent Grafana from running out of memory while attempting to cache very large query responses.", diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index 3a865903fd0..b9106003778 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -74,7 +74,6 @@ queryServiceRewrite,experimental,@grafana/grafana-datasources-core-services,fals queryServiceFromUI,experimental,@grafana/grafana-datasources-core-services,false,false,true queryServiceFromExplore,experimental,@grafana/grafana-datasources-core-services,false,false,true cloudWatchBatchQueries,preview,@grafana/aws-datasources,false,false,false -lokiStructuredMetadata,GA,@grafana/observability-logs,false,false,false cachingOptimizeSerializationMemoryUsage,experimental,@grafana/grafana-operator-experience-squad,false,false,false prometheusCodeModeMetricNamesSearch,experimental,@grafana/oss-big-tent,false,false,true addFieldFromCalculationStatFunctions,GA,@grafana/dataviz-squad,false,false,true diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index 7e01719bd7d..e7e939c6885 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -307,10 +307,6 @@ const ( // Runs CloudWatch metrics queries as separate batches FlagCloudWatchBatchQueries = "cloudWatchBatchQueries" - // FlagLokiStructuredMetadata - // Enables the loki data source to request structured metadata from the Loki server - FlagLokiStructuredMetadata = "lokiStructuredMetadata" - // FlagCachingOptimizeSerializationMemoryUsage // If enabled, the caching backend gradually serializes query responses for the cache, comparing against the configured `[caching]max_value_mb` value as it goes. This can can help prevent Grafana from running out of memory while attempting to cache very large query responses. FlagCachingOptimizeSerializationMemoryUsage = "cachingOptimizeSerializationMemoryUsage" diff --git a/pkg/services/featuremgmt/toggles_gen.json b/pkg/services/featuremgmt/toggles_gen.json index a32f6fca33f..aa066802f55 100644 --- a/pkg/services/featuremgmt/toggles_gen.json +++ b/pkg/services/featuremgmt/toggles_gen.json @@ -1927,7 +1927,8 @@ "metadata": { "name": "lokiStructuredMetadata", "resourceVersion": "1750434297879", - "creationTimestamp": "2023-11-16T16:06:14Z" + "creationTimestamp": "2023-11-16T16:06:14Z", + "deletionTimestamp": "2025-06-27T09:00:53Z" }, "spec": { "description": "Enables the loki data source to request structured metadata from the Loki server", diff --git a/pkg/tsdb/loki/api.go b/pkg/tsdb/loki/api.go index 4c1006fad33..464a267391a 100644 --- a/pkg/tsdb/loki/api.go +++ b/pkg/tsdb/loki/api.go @@ -27,11 +27,10 @@ import ( ) type LokiAPI struct { - client *http.Client - url string - log log.Logger - tracer trace.Tracer - requestStructuredMetadata bool + client *http.Client + url string + log log.Logger + tracer trace.Tracer } type RawLokiResponse struct { @@ -40,11 +39,11 @@ type RawLokiResponse struct { Encoding string } -func newLokiAPI(client *http.Client, url string, log log.Logger, tracer trace.Tracer, requestStructuredMetadata bool) *LokiAPI { - return &LokiAPI{client: client, url: url, log: log, tracer: tracer, requestStructuredMetadata: requestStructuredMetadata} +func newLokiAPI(client *http.Client, url string, log log.Logger, tracer trace.Tracer) *LokiAPI { + return &LokiAPI{client: client, url: url, log: log, tracer: tracer} } -func makeDataRequest(ctx context.Context, lokiDsUrl string, query lokiQuery, categorizeLabels bool) (*http.Request, error) { +func makeDataRequest(ctx context.Context, lokiDsUrl string, query lokiQuery) (*http.Request, error) { qs := url.Values{} qs.Set("query", query.Expr) @@ -103,11 +102,7 @@ func makeDataRequest(ctx context.Context, lokiDsUrl string, query lokiQuery, cat req.Header.Set("X-Query-Tags", "Source="+value) } } - - if categorizeLabels { - req.Header.Set("X-Loki-Response-Encoding-Flags", "categorize-labels") - } - + req.Header.Set("X-Loki-Response-Encoding-Flags", "categorize-labels") setXScopeOrgIDHeader(req, ctx) return req, nil @@ -164,7 +159,7 @@ func readLokiError(body io.ReadCloser) error { } func (api *LokiAPI) DataQuery(ctx context.Context, query lokiQuery, responseOpts ResponseOpts) (*backend.DataResponse, error) { - req, err := makeDataRequest(ctx, api.url, query, api.requestStructuredMetadata) + req, err := makeDataRequest(ctx, api.url, query) if err != nil { return nil, err } diff --git a/pkg/tsdb/loki/api_mock.go b/pkg/tsdb/loki/api_mock.go index a3e565bf351..7fd7568162f 100644 --- a/pkg/tsdb/loki/api_mock.go +++ b/pkg/tsdb/loki/api_mock.go @@ -57,16 +57,16 @@ func (mockedRT *mockedCompressedRoundTripper) RoundTrip(req *http.Request) (*htt }, nil } -func makeMockedAPI(statusCode int, contentType string, responseBytes []byte, requestCallback mockRequestCallback, structuredMetadata bool) *LokiAPI { - return makeMockedAPIWithUrl("http://localhost:9999", statusCode, contentType, responseBytes, requestCallback, structuredMetadata) +func makeMockedAPI(statusCode int, contentType string, responseBytes []byte, requestCallback mockRequestCallback) *LokiAPI { + return makeMockedAPIWithUrl("http://localhost:9999", statusCode, contentType, responseBytes, requestCallback) } -func makeMockedAPIWithUrl(url string, statusCode int, contentType string, responseBytes []byte, requestCallback mockRequestCallback, structuredMetadata bool) *LokiAPI { +func makeMockedAPIWithUrl(url string, statusCode int, contentType string, responseBytes []byte, requestCallback mockRequestCallback) *LokiAPI { client := http.Client{ Transport: &mockedRoundTripper{statusCode: statusCode, contentType: contentType, responseBytes: responseBytes, requestCallback: requestCallback}, } - return newLokiAPI(&client, url, backend.NewLoggerWith("logger", "test"), tracing.DefaultTracer(), structuredMetadata) + return newLokiAPI(&client, url, backend.NewLoggerWith("logger", "test"), tracing.DefaultTracer()) } func makeCompressedMockedAPIWithUrl(url string, statusCode int, contentType string, responseBytes []byte, requestCallback mockRequestCallback) *LokiAPI { @@ -74,5 +74,5 @@ func makeCompressedMockedAPIWithUrl(url string, statusCode int, contentType stri Transport: &mockedCompressedRoundTripper{statusCode: statusCode, contentType: contentType, responseBytes: responseBytes, requestCallback: requestCallback}, } - return newLokiAPI(&client, url, backend.NewLoggerWith("logger", "test"), tracing.DefaultTracer(), false) + return newLokiAPI(&client, url, backend.NewLoggerWith("logger", "test"), tracing.DefaultTracer()) } diff --git a/pkg/tsdb/loki/api_test.go b/pkg/tsdb/loki/api_test.go index d48bed634b1..252608780e6 100644 --- a/pkg/tsdb/loki/api_test.go +++ b/pkg/tsdb/loki/api_test.go @@ -28,7 +28,7 @@ func TestApiLogVolume(t *testing.T) { api := makeMockedAPI(200, "application/json", response, func(req *http.Request) { called = true require.Equal(t, "Source=logvolhist", req.Header.Get("X-Query-Tags")) - }, false) + }) _, err := api.DataQuery(context.Background(), lokiQuery{Expr: "", SupportingQueryType: SupportingQueryLogsVolume, QueryType: QueryTypeRange}, ResponseOpts{}) require.NoError(t, err) @@ -40,7 +40,7 @@ func TestApiLogVolume(t *testing.T) { api := makeMockedAPI(200, "application/json", response, func(req *http.Request) { called = true require.Equal(t, "Source=logsample", req.Header.Get("X-Query-Tags")) - }, false) + }) _, err := api.DataQuery(context.Background(), lokiQuery{Expr: "", SupportingQueryType: SupportingQueryLogsSample, QueryType: QueryTypeRange}, ResponseOpts{}) require.NoError(t, err) @@ -52,7 +52,7 @@ func TestApiLogVolume(t *testing.T) { api := makeMockedAPI(200, "application/json", response, func(req *http.Request) { called = true require.Equal(t, "Source=datasample", req.Header.Get("X-Query-Tags")) - }, false) + }) _, err := api.DataQuery(context.Background(), lokiQuery{Expr: "", SupportingQueryType: SupportingQueryDataSample, QueryType: QueryTypeRange}, ResponseOpts{}) require.NoError(t, err) @@ -64,7 +64,7 @@ func TestApiLogVolume(t *testing.T) { api := makeMockedAPI(200, "application/json", response, func(req *http.Request) { called = true require.Equal(t, "", req.Header.Get("X-Query-Tags")) - }, false) + }) _, err := api.DataQuery(context.Background(), lokiQuery{Expr: "", SupportingQueryType: SupportingQueryNone, QueryType: QueryTypeRange}, ResponseOpts{}) require.NoError(t, err) @@ -76,19 +76,19 @@ func TestApiLogVolume(t *testing.T) { api := makeMockedAPI(200, "application/json", response, func(req *http.Request) { called = true require.Equal(t, "Source=foo", req.Header.Get("X-Query-Tags")) - }, false) + }) _, err := api.DataQuery(context.Background(), lokiQuery{Expr: "", SupportingQueryType: SupportingQueryType("foo"), QueryType: QueryTypeRange}, ResponseOpts{}) require.NoError(t, err) require.True(t, called) }) - t.Run("with `structuredMetadata` should set correct http header", func(t *testing.T) { + t.Run("should set correct http header", func(t *testing.T) { called := false api := makeMockedAPI(200, "application/json", response, func(req *http.Request) { called = true require.Equal(t, "categorize-labels", req.Header.Get("X-Loki-Response-Encoding-Flags")) - }, true) + }) _, err := api.DataQuery(context.Background(), lokiQuery{Expr: "", SupportingQueryType: SupportingQueryLogsVolume, QueryType: QueryTypeRange}, ResponseOpts{}) require.NoError(t, err) @@ -112,7 +112,7 @@ func TestInfiniteScroll(t *testing.T) { api := makeMockedAPI(200, "application/json", response, func(req *http.Request) { called = true require.Equal(t, "Source=infinitescroll", req.Header.Get("X-Query-Tags")) - }, false) + }) _, err := api.DataQuery(context.Background(), lokiQuery{Expr: "", SupportingQueryType: dataquery.SupportingQueryTypeInfiniteScroll, QueryType: QueryTypeRange}, ResponseOpts{}) require.NoError(t, err) @@ -177,7 +177,7 @@ func TestApiUrlHandling(t *testing.T) { wantedPrefix := test.rangeQueryPrefix failMessage := fmt.Sprintf(`wanted prefix: [%s], got string [%s]`, wantedPrefix, urlString) require.True(t, strings.HasPrefix(urlString, wantedPrefix), failMessage) - }, false) + }) query := lokiQuery{ QueryType: QueryTypeRange, @@ -198,7 +198,7 @@ func TestApiUrlHandling(t *testing.T) { wantedPrefix := test.instantQueryPrefix failMessage := fmt.Sprintf(`wanted prefix: [%s], got string [%s]`, wantedPrefix, urlString) require.True(t, strings.HasPrefix(urlString, wantedPrefix), failMessage) - }, false) + }) query := lokiQuery{ QueryType: QueryTypeInstant, @@ -216,7 +216,7 @@ func TestApiUrlHandling(t *testing.T) { api := makeMockedAPIWithUrl(test.dsUrl, 200, "application/json", response, func(req *http.Request) { called = true require.Equal(t, test.metaUrl, req.URL.String()) - }, false) + }) _, err := api.RawQuery(context.Background(), "/loki/api/v1/labels?start=1&end=2") require.NoError(t, err) @@ -289,7 +289,7 @@ func TestErrorSources(t *testing.T) { called := false api := makeMockedAPI(400, "application/json", errorResponse, func(req *http.Request) { called = true - }, false) + }) res, err := api.DataQuery(context.Background(), lokiQuery{QueryType: QueryTypeRange}, ResponseOpts{}) require.NoError(t, err) @@ -302,7 +302,7 @@ func TestErrorSources(t *testing.T) { called := false api := makeMockedAPI(406, "application/json", errorResponse, func(req *http.Request) { called = true - }, false) + }) res, err := api.DataQuery(context.Background(), lokiQuery{QueryType: QueryTypeRange}, ResponseOpts{}) require.NoError(t, err) @@ -315,7 +315,7 @@ func TestErrorSources(t *testing.T) { called := false api := makeMockedAPI(500, "application/json", errorResponse, func(req *http.Request) { called = true - }, false) + }) res, err := api.DataQuery(context.Background(), lokiQuery{QueryType: QueryTypeRange}, ResponseOpts{}) require.NoError(t, err) @@ -328,7 +328,7 @@ func TestErrorSources(t *testing.T) { called := false api := makeMockedAPI(400, "application/json", errorResponse, func(req *http.Request) { called = true - }, false) + }) res, err := api.DataQuery(context.Background(), lokiQuery{QueryType: QueryTypeRange}, ResponseOpts{}) require.NoError(t, err) @@ -342,7 +342,7 @@ func TestErrorSources(t *testing.T) { called := false api := makeMockedAPI(400, "application/json", errorResponse, func(req *http.Request) { called = true - }, false) + }) res, err := api.RawQuery(context.Background(), "/loki/api/v1/labels") require.NoError(t, err) @@ -355,7 +355,7 @@ func TestErrorSources(t *testing.T) { called := false api := makeMockedAPI(500, "application/json", errorResponse, func(req *http.Request) { called = true - }, false) + }) _, err := api.RawQuery(context.Background(), "/loki/api/v1/labels") require.Error(t, err) diff --git a/pkg/tsdb/loki/framing_test.go b/pkg/tsdb/loki/framing_test.go index 8715f98e4d5..624922ca49d 100644 --- a/pkg/tsdb/loki/framing_test.go +++ b/pkg/tsdb/loki/framing_test.go @@ -66,7 +66,7 @@ func TestSuccessResponse(t *testing.T) { bytes, err := os.ReadFile(responseFileName) require.NoError(t, err) - dr, err := runQuery(context.Background(), makeMockedAPI(http.StatusOK, "application/json", bytes, nil, false), &query, responseOpts, backend.NewLoggerWith("logger", "test")) + dr, err := runQuery(context.Background(), makeMockedAPI(http.StatusOK, "application/json", bytes, nil), &query, responseOpts, backend.NewLoggerWith("logger", "test")) require.NoError(t, err) experimental.CheckGoldenJSONResponse(t, folder, goldenFileName, dr, false) @@ -125,7 +125,7 @@ func TestErrorResponse(t *testing.T) { for _, test := range tt { t.Run(test.name, func(t *testing.T) { - dr, err := runQuery(context.Background(), makeMockedAPI(400, test.contentType, test.body, nil, false), &lokiQuery{QueryType: QueryTypeRange, Direction: DirectionBackward}, ResponseOpts{}, backend.NewLoggerWith("logger", "test")) + dr, err := runQuery(context.Background(), makeMockedAPI(400, test.contentType, test.body, nil), &lokiQuery{QueryType: QueryTypeRange, Direction: DirectionBackward}, ResponseOpts{}, backend.NewLoggerWith("logger", "test")) require.NoError(t, err) require.Len(t, dr.Frames, 0) require.Equal(t, dr.Error.Error(), test.errorMessage) @@ -172,7 +172,7 @@ func TestErrorsFromResponseCodes(t *testing.T) { for _, test := range tt { t.Run(test.name, func(t *testing.T) { - dr, _ := runQuery(context.Background(), makeMockedAPI(test.statusCode, contentType, []byte(errorString), nil, false), &lokiQuery{QueryType: QueryTypeRange, Direction: DirectionBackward}, ResponseOpts{}, backend.NewLoggerWith("logger", "test")) + dr, _ := runQuery(context.Background(), makeMockedAPI(test.statusCode, contentType, []byte(errorString), nil), &lokiQuery{QueryType: QueryTypeRange, Direction: DirectionBackward}, ResponseOpts{}, backend.NewLoggerWith("logger", "test")) require.Len(t, dr.Frames, 0) require.Equal(t, dr.Error.Error(), errorString) require.Equal(t, dr.ErrorSource, test.errorSource) diff --git a/pkg/tsdb/loki/loki.go b/pkg/tsdb/loki/loki.go index 5114c514610..4b68c66b42a 100644 --- a/pkg/tsdb/loki/loki.go +++ b/pkg/tsdb/loki/loki.go @@ -29,7 +29,6 @@ import ( const ( flagLokiLogsDataplane = "lokiLogsDataplane" flagLokiRunQueriesInParallel = "lokiRunQueriesInParallel" - flagLokiStructuredMetadata = "lokiStructuredMetadata" flagLogQLScope = "logQLScope" flagLokiExperimentalStreaming = "lokiExperimentalStreaming" fromAlertHeaderName = "FromAlert" @@ -134,7 +133,7 @@ func callResource(ctx context.Context, req *backend.CallResourceRequest, sender )) defer span.End() - api := newLokiAPI(dsInfo.HTTPClient, dsInfo.URL, plog, tracer, false) + api := newLokiAPI(dsInfo.HTTPClient, dsInfo.URL, plog, tracer) var rawLokiResponse RawLokiResponse var err error @@ -186,13 +185,13 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest) logsDataplane: isFeatureEnabled(ctx, flagLokiLogsDataplane), } - return queryData(ctx, req, dsInfo, responseOpts, s.tracer, logger, isFeatureEnabled(ctx, flagLokiRunQueriesInParallel), isFeatureEnabled(ctx, flagLokiStructuredMetadata), isFeatureEnabled(ctx, flagLogQLScope)) + return queryData(ctx, req, dsInfo, responseOpts, s.tracer, logger, isFeatureEnabled(ctx, flagLokiRunQueriesInParallel), isFeatureEnabled(ctx, flagLogQLScope)) } -func queryData(ctx context.Context, req *backend.QueryDataRequest, dsInfo *datasourceInfo, responseOpts ResponseOpts, tracer trace.Tracer, plog log.Logger, runInParallel bool, requestStructuredMetadata, logQLScopes bool) (*backend.QueryDataResponse, error) { +func queryData(ctx context.Context, req *backend.QueryDataRequest, dsInfo *datasourceInfo, responseOpts ResponseOpts, tracer trace.Tracer, plog log.Logger, runInParallel bool, logQLScopes bool) (*backend.QueryDataResponse, error) { result := backend.NewQueryDataResponse() - api := newLokiAPI(dsInfo.HTTPClient, dsInfo.URL, plog, tracer, requestStructuredMetadata) + api := newLokiAPI(dsInfo.HTTPClient, dsInfo.URL, plog, tracer) start := time.Now() queries, err := parseQuery(req, logQLScopes) diff --git a/pkg/tsdb/loki/loki_bench_test.go b/pkg/tsdb/loki/loki_bench_test.go index f9c5e15f293..5819feee802 100644 --- a/pkg/tsdb/loki/loki_bench_test.go +++ b/pkg/tsdb/loki/loki_bench_test.go @@ -19,7 +19,7 @@ func BenchmarkMatrixJson(b *testing.B) { b.ResetTimer() for n := 0; n < b.N; n++ { - _, _ = runQuery(context.Background(), makeMockedAPI(http.StatusOK, "application/json", bytes, nil, false), &lokiQuery{}, ResponseOpts{}, backend.NewLoggerWith("logger", "test")) + _, _ = runQuery(context.Background(), makeMockedAPI(http.StatusOK, "application/json", bytes, nil), &lokiQuery{}, ResponseOpts{}, backend.NewLoggerWith("logger", "test")) } }