Loki: Remove lokiStructuredMetadata feature toggle (#107292)

Loki: Remove lokiStructuredMetadata feature toggle

docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md

@@ -45,7 +45,6 @@ Most [generally available](https://grafana.com/docs/release-life-cycle/#general-
 | `panelMonitoring`                      | Enables panel monitoring through logs and measurements                                                                             | Yes                |
 | `formatString`                         | Enable format string transformer                                                                                                   | Yes                |
 | `kubernetesClientDashboardsFolders`    | Route the folder and dashboard service requests to k8s                                                                             | Yes                |
-| `lokiStructuredMetadata`               | Enables the loki data source to request structured metadata from the Loki server                                                   | Yes                |
 | `addFieldFromCalculationStatFunctions` | Add cumulative and window functions to the add field from calculation transformation                                               | Yes                |
 | `annotationPermissionUpdate`           | Change the way annotation permissions work by scoping them to folders and dashboards.                                              | Yes                |
 | `dashboardSceneForViewers`             | Enables dashboard rendering using Scenes for viewer roles                                                                          | Yes                |

packages/grafana-data/src/types/featureToggles.gen.ts

@@ -340,11 +340,6 @@ export interface FeatureToggles {
   */
   cloudWatchBatchQueries?: boolean;
   /**
-  * Enables the loki data source to request structured metadata from the Loki server
-  * @default true
-  */
-  lokiStructuredMetadata?: boolean;
-  /**
   * If enabled, the caching backend gradually serializes query responses for the cache, comparing against the configured `[caching]max_value_mb` value as it goes. This can can help prevent Grafana from running out of memory while attempting to cache very large query responses.
   */
   cachingOptimizeSerializationMemoryUsage?: boolean;

pkg/services/featuremgmt/registry.go

@@ -563,14 +563,6 @@ var (
 			Stage:       FeatureStagePublicPreview,
 			Owner:       awsDatasourcesSquad,
 		},
-		{
-			Name:         "lokiStructuredMetadata",
-			Description:  "Enables the loki data source to request structured metadata from the Loki server",
-			Stage:        FeatureStageGeneralAvailability,
-			FrontendOnly: false,
-			Owner:        grafanaObservabilityLogsSquad,
-			Expression:   "true",
-		},
 		{
 			Name:         "cachingOptimizeSerializationMemoryUsage",
 			Description:  "If enabled, the caching backend gradually serializes query responses for the cache, comparing against the configured `[caching]max_value_mb` value as it goes. This can can help prevent Grafana from running out of memory while attempting to cache very large query responses.",

pkg/services/featuremgmt/toggles_gen.csv

@@ -74,7 +74,6 @@ queryServiceRewrite,experimental,@grafana/grafana-datasources-core-services,fals
 queryServiceFromUI,experimental,@grafana/grafana-datasources-core-services,false,false,true
 queryServiceFromExplore,experimental,@grafana/grafana-datasources-core-services,false,false,true
 cloudWatchBatchQueries,preview,@grafana/aws-datasources,false,false,false
-lokiStructuredMetadata,GA,@grafana/observability-logs,false,false,false
 cachingOptimizeSerializationMemoryUsage,experimental,@grafana/grafana-operator-experience-squad,false,false,false
 prometheusCodeModeMetricNamesSearch,experimental,@grafana/oss-big-tent,false,false,true
 addFieldFromCalculationStatFunctions,GA,@grafana/dataviz-squad,false,false,true

pkg/services/featuremgmt/toggles_gen.go

@@ -307,10 +307,6 @@ const (
 	// Runs CloudWatch metrics queries as separate batches
 	FlagCloudWatchBatchQueries = "cloudWatchBatchQueries"
 
-	// FlagLokiStructuredMetadata
-	// Enables the loki data source to request structured metadata from the Loki server
-	FlagLokiStructuredMetadata = "lokiStructuredMetadata"
-
 	// FlagCachingOptimizeSerializationMemoryUsage
 	// If enabled, the caching backend gradually serializes query responses for the cache, comparing against the configured `[caching]max_value_mb` value as it goes. This can can help prevent Grafana from running out of memory while attempting to cache very large query responses.
 	FlagCachingOptimizeSerializationMemoryUsage = "cachingOptimizeSerializationMemoryUsage"

pkg/services/featuremgmt/toggles_gen.json

@@ -1927,7 +1927,8 @@
       "metadata": {
         "name": "lokiStructuredMetadata",
         "resourceVersion": "1750434297879",
-        "creationTimestamp": "2023-11-16T16:06:14Z"
+        "creationTimestamp": "2023-11-16T16:06:14Z",
+        "deletionTimestamp": "2025-06-27T09:00:53Z"
       },
       "spec": {
         "description": "Enables the loki data source to request structured metadata from the Loki server",

pkg/tsdb/loki/api.go

@@ -27,11 +27,10 @@ import (
 )
 
 type LokiAPI struct {
-	client                    *http.Client
-	url                       string
-	log                       log.Logger
-	tracer                    trace.Tracer
-	requestStructuredMetadata bool
+	client *http.Client
+	url    string
+	log    log.Logger
+	tracer trace.Tracer
 }
 
 type RawLokiResponse struct {
@@ -40,11 +39,11 @@ type RawLokiResponse struct {
 	Encoding string
 }
 
-func newLokiAPI(client *http.Client, url string, log log.Logger, tracer trace.Tracer, requestStructuredMetadata bool) *LokiAPI {
-	return &LokiAPI{client: client, url: url, log: log, tracer: tracer, requestStructuredMetadata: requestStructuredMetadata}
+func newLokiAPI(client *http.Client, url string, log log.Logger, tracer trace.Tracer) *LokiAPI {
+	return &LokiAPI{client: client, url: url, log: log, tracer: tracer}
 }
 
-func makeDataRequest(ctx context.Context, lokiDsUrl string, query lokiQuery, categorizeLabels bool) (*http.Request, error) {
+func makeDataRequest(ctx context.Context, lokiDsUrl string, query lokiQuery) (*http.Request, error) {
 	qs := url.Values{}
 	qs.Set("query", query.Expr)
 
@@ -103,11 +102,7 @@ func makeDataRequest(ctx context.Context, lokiDsUrl string, query lokiQuery, cat
 			req.Header.Set("X-Query-Tags", "Source="+value)
 		}
 	}
-
-	if categorizeLabels {
-		req.Header.Set("X-Loki-Response-Encoding-Flags", "categorize-labels")
-	}
-
+	req.Header.Set("X-Loki-Response-Encoding-Flags", "categorize-labels")
 	setXScopeOrgIDHeader(req, ctx)
 
 	return req, nil
@@ -164,7 +159,7 @@ func readLokiError(body io.ReadCloser) error {
 }
 
 func (api *LokiAPI) DataQuery(ctx context.Context, query lokiQuery, responseOpts ResponseOpts) (*backend.DataResponse, error) {
-	req, err := makeDataRequest(ctx, api.url, query, api.requestStructuredMetadata)
+	req, err := makeDataRequest(ctx, api.url, query)
 	if err != nil {
 		return nil, err
 	}

pkg/tsdb/loki/api_mock.go

@@ -57,16 +57,16 @@ func (mockedRT *mockedCompressedRoundTripper) RoundTrip(req *http.Request) (*htt
 	}, nil
 }
 
-func makeMockedAPI(statusCode int, contentType string, responseBytes []byte, requestCallback mockRequestCallback, structuredMetadata bool) *LokiAPI {
-	return makeMockedAPIWithUrl("http://localhost:9999", statusCode, contentType, responseBytes, requestCallback, structuredMetadata)
+func makeMockedAPI(statusCode int, contentType string, responseBytes []byte, requestCallback mockRequestCallback) *LokiAPI {
+	return makeMockedAPIWithUrl("http://localhost:9999", statusCode, contentType, responseBytes, requestCallback)
 }
 
-func makeMockedAPIWithUrl(url string, statusCode int, contentType string, responseBytes []byte, requestCallback mockRequestCallback, structuredMetadata bool) *LokiAPI {
+func makeMockedAPIWithUrl(url string, statusCode int, contentType string, responseBytes []byte, requestCallback mockRequestCallback) *LokiAPI {
 	client := http.Client{
 		Transport: &mockedRoundTripper{statusCode: statusCode, contentType: contentType, responseBytes: responseBytes, requestCallback: requestCallback},
 	}
 
-	return newLokiAPI(&client, url, backend.NewLoggerWith("logger", "test"), tracing.DefaultTracer(), structuredMetadata)
+	return newLokiAPI(&client, url, backend.NewLoggerWith("logger", "test"), tracing.DefaultTracer())
 }
 
 func makeCompressedMockedAPIWithUrl(url string, statusCode int, contentType string, responseBytes []byte, requestCallback mockRequestCallback) *LokiAPI {
@@ -74,5 +74,5 @@ func makeCompressedMockedAPIWithUrl(url string, statusCode int, contentType stri
 		Transport: &mockedCompressedRoundTripper{statusCode: statusCode, contentType: contentType, responseBytes: responseBytes, requestCallback: requestCallback},
 	}
 
-	return newLokiAPI(&client, url, backend.NewLoggerWith("logger", "test"), tracing.DefaultTracer(), false)
+	return newLokiAPI(&client, url, backend.NewLoggerWith("logger", "test"), tracing.DefaultTracer())
 }

pkg/tsdb/loki/api_test.go

@@ -28,7 +28,7 @@ func TestApiLogVolume(t *testing.T) {
 		api := makeMockedAPI(200, "application/json", response, func(req *http.Request) {
 			called = true
 			require.Equal(t, "Source=logvolhist", req.Header.Get("X-Query-Tags"))
-		}, false)
+		})
 
 		_, err := api.DataQuery(context.Background(), lokiQuery{Expr: "", SupportingQueryType: SupportingQueryLogsVolume, QueryType: QueryTypeRange}, ResponseOpts{})
 		require.NoError(t, err)
@@ -40,7 +40,7 @@ func TestApiLogVolume(t *testing.T) {
 		api := makeMockedAPI(200, "application/json", response, func(req *http.Request) {
 			called = true
 			require.Equal(t, "Source=logsample", req.Header.Get("X-Query-Tags"))
-		}, false)
+		})
 
 		_, err := api.DataQuery(context.Background(), lokiQuery{Expr: "", SupportingQueryType: SupportingQueryLogsSample, QueryType: QueryTypeRange}, ResponseOpts{})
 		require.NoError(t, err)
@@ -52,7 +52,7 @@ func TestApiLogVolume(t *testing.T) {
 		api := makeMockedAPI(200, "application/json", response, func(req *http.Request) {
 			called = true
 			require.Equal(t, "Source=datasample", req.Header.Get("X-Query-Tags"))
-		}, false)
+		})
 
 		_, err := api.DataQuery(context.Background(), lokiQuery{Expr: "", SupportingQueryType: SupportingQueryDataSample, QueryType: QueryTypeRange}, ResponseOpts{})
 		require.NoError(t, err)
@@ -64,7 +64,7 @@ func TestApiLogVolume(t *testing.T) {
 		api := makeMockedAPI(200, "application/json", response, func(req *http.Request) {
 			called = true
 			require.Equal(t, "", req.Header.Get("X-Query-Tags"))
-		}, false)
+		})
 
 		_, err := api.DataQuery(context.Background(), lokiQuery{Expr: "", SupportingQueryType: SupportingQueryNone, QueryType: QueryTypeRange}, ResponseOpts{})
 		require.NoError(t, err)
@@ -76,19 +76,19 @@ func TestApiLogVolume(t *testing.T) {
 		api := makeMockedAPI(200, "application/json", response, func(req *http.Request) {
 			called = true
 			require.Equal(t, "Source=foo", req.Header.Get("X-Query-Tags"))
-		}, false)
+		})
 
 		_, err := api.DataQuery(context.Background(), lokiQuery{Expr: "", SupportingQueryType: SupportingQueryType("foo"), QueryType: QueryTypeRange}, ResponseOpts{})
 		require.NoError(t, err)
 		require.True(t, called)
 	})
 
-	t.Run("with `structuredMetadata` should set correct http header", func(t *testing.T) {
+	t.Run("should set correct http header", func(t *testing.T) {
 		called := false
 		api := makeMockedAPI(200, "application/json", response, func(req *http.Request) {
 			called = true
 			require.Equal(t, "categorize-labels", req.Header.Get("X-Loki-Response-Encoding-Flags"))
-		}, true)
+		})
 
 		_, err := api.DataQuery(context.Background(), lokiQuery{Expr: "", SupportingQueryType: SupportingQueryLogsVolume, QueryType: QueryTypeRange}, ResponseOpts{})
 		require.NoError(t, err)
@@ -112,7 +112,7 @@ func TestInfiniteScroll(t *testing.T) {
 		api := makeMockedAPI(200, "application/json", response, func(req *http.Request) {
 			called = true
 			require.Equal(t, "Source=infinitescroll", req.Header.Get("X-Query-Tags"))
-		}, false)
+		})
 
 		_, err := api.DataQuery(context.Background(), lokiQuery{Expr: "", SupportingQueryType: dataquery.SupportingQueryTypeInfiniteScroll, QueryType: QueryTypeRange}, ResponseOpts{})
 		require.NoError(t, err)
@@ -177,7 +177,7 @@ func TestApiUrlHandling(t *testing.T) {
 				wantedPrefix := test.rangeQueryPrefix
 				failMessage := fmt.Sprintf(`wanted prefix: [%s], got string [%s]`, wantedPrefix, urlString)
 				require.True(t, strings.HasPrefix(urlString, wantedPrefix), failMessage)
-			}, false)
+			})
 
 			query := lokiQuery{
 				QueryType: QueryTypeRange,
@@ -198,7 +198,7 @@ func TestApiUrlHandling(t *testing.T) {
 				wantedPrefix := test.instantQueryPrefix
 				failMessage := fmt.Sprintf(`wanted prefix: [%s], got string [%s]`, wantedPrefix, urlString)
 				require.True(t, strings.HasPrefix(urlString, wantedPrefix), failMessage)
-			}, false)
+			})
 
 			query := lokiQuery{
 				QueryType: QueryTypeInstant,
@@ -216,7 +216,7 @@ func TestApiUrlHandling(t *testing.T) {
 			api := makeMockedAPIWithUrl(test.dsUrl, 200, "application/json", response, func(req *http.Request) {
 				called = true
 				require.Equal(t, test.metaUrl, req.URL.String())
-			}, false)
+			})
 
 			_, err := api.RawQuery(context.Background(), "/loki/api/v1/labels?start=1&end=2")
 			require.NoError(t, err)
@@ -289,7 +289,7 @@ func TestErrorSources(t *testing.T) {
 		called := false
 		api := makeMockedAPI(400, "application/json", errorResponse, func(req *http.Request) {
 			called = true
-		}, false)
+		})
 
 		res, err := api.DataQuery(context.Background(), lokiQuery{QueryType: QueryTypeRange}, ResponseOpts{})
 		require.NoError(t, err)
@@ -302,7 +302,7 @@ func TestErrorSources(t *testing.T) {
 		called := false
 		api := makeMockedAPI(406, "application/json", errorResponse, func(req *http.Request) {
 			called = true
-		}, false)
+		})
 
 		res, err := api.DataQuery(context.Background(), lokiQuery{QueryType: QueryTypeRange}, ResponseOpts{})
 		require.NoError(t, err)
@@ -315,7 +315,7 @@ func TestErrorSources(t *testing.T) {
 		called := false
 		api := makeMockedAPI(500, "application/json", errorResponse, func(req *http.Request) {
 			called = true
-		}, false)
+		})
 
 		res, err := api.DataQuery(context.Background(), lokiQuery{QueryType: QueryTypeRange}, ResponseOpts{})
 		require.NoError(t, err)
@@ -328,7 +328,7 @@ func TestErrorSources(t *testing.T) {
 		called := false
 		api := makeMockedAPI(400, "application/json", errorResponse, func(req *http.Request) {
 			called = true
-		}, false)
+		})
 
 		res, err := api.DataQuery(context.Background(), lokiQuery{QueryType: QueryTypeRange}, ResponseOpts{})
 		require.NoError(t, err)
@@ -342,7 +342,7 @@ func TestErrorSources(t *testing.T) {
 		called := false
 		api := makeMockedAPI(400, "application/json", errorResponse, func(req *http.Request) {
 			called = true
-		}, false)
+		})
 
 		res, err := api.RawQuery(context.Background(), "/loki/api/v1/labels")
 		require.NoError(t, err)
@@ -355,7 +355,7 @@ func TestErrorSources(t *testing.T) {
 		called := false
 		api := makeMockedAPI(500, "application/json", errorResponse, func(req *http.Request) {
 			called = true
-		}, false)
+		})
 
 		_, err := api.RawQuery(context.Background(), "/loki/api/v1/labels")
 		require.Error(t, err)

pkg/tsdb/loki/framing_test.go

@@ -66,7 +66,7 @@ func TestSuccessResponse(t *testing.T) {
 		bytes, err := os.ReadFile(responseFileName)
 		require.NoError(t, err)
 
-		dr, err := runQuery(context.Background(), makeMockedAPI(http.StatusOK, "application/json", bytes, nil, false), &query, responseOpts, backend.NewLoggerWith("logger", "test"))
+		dr, err := runQuery(context.Background(), makeMockedAPI(http.StatusOK, "application/json", bytes, nil), &query, responseOpts, backend.NewLoggerWith("logger", "test"))
 		require.NoError(t, err)
 
 		experimental.CheckGoldenJSONResponse(t, folder, goldenFileName, dr, false)
@@ -125,7 +125,7 @@ func TestErrorResponse(t *testing.T) {
 
 	for _, test := range tt {
 		t.Run(test.name, func(t *testing.T) {
-			dr, err := runQuery(context.Background(), makeMockedAPI(400, test.contentType, test.body, nil, false), &lokiQuery{QueryType: QueryTypeRange, Direction: DirectionBackward}, ResponseOpts{}, backend.NewLoggerWith("logger", "test"))
+			dr, err := runQuery(context.Background(), makeMockedAPI(400, test.contentType, test.body, nil), &lokiQuery{QueryType: QueryTypeRange, Direction: DirectionBackward}, ResponseOpts{}, backend.NewLoggerWith("logger", "test"))
 			require.NoError(t, err)
 			require.Len(t, dr.Frames, 0)
 			require.Equal(t, dr.Error.Error(), test.errorMessage)
@@ -172,7 +172,7 @@ func TestErrorsFromResponseCodes(t *testing.T) {
 
 	for _, test := range tt {
 		t.Run(test.name, func(t *testing.T) {
-			dr, _ := runQuery(context.Background(), makeMockedAPI(test.statusCode, contentType, []byte(errorString), nil, false), &lokiQuery{QueryType: QueryTypeRange, Direction: DirectionBackward}, ResponseOpts{}, backend.NewLoggerWith("logger", "test"))
+			dr, _ := runQuery(context.Background(), makeMockedAPI(test.statusCode, contentType, []byte(errorString), nil), &lokiQuery{QueryType: QueryTypeRange, Direction: DirectionBackward}, ResponseOpts{}, backend.NewLoggerWith("logger", "test"))
 			require.Len(t, dr.Frames, 0)
 			require.Equal(t, dr.Error.Error(), errorString)
 			require.Equal(t, dr.ErrorSource, test.errorSource)

pkg/tsdb/loki/loki.go

@@ -29,7 +29,6 @@ import (
 const (
 	flagLokiLogsDataplane         = "lokiLogsDataplane"
 	flagLokiRunQueriesInParallel  = "lokiRunQueriesInParallel"
-	flagLokiStructuredMetadata    = "lokiStructuredMetadata"
 	flagLogQLScope                = "logQLScope"
 	flagLokiExperimentalStreaming = "lokiExperimentalStreaming"
 	fromAlertHeaderName           = "FromAlert"
@@ -134,7 +133,7 @@ func callResource(ctx context.Context, req *backend.CallResourceRequest, sender
 	))
 	defer span.End()
 
-	api := newLokiAPI(dsInfo.HTTPClient, dsInfo.URL, plog, tracer, false)
+	api := newLokiAPI(dsInfo.HTTPClient, dsInfo.URL, plog, tracer)
 
 	var rawLokiResponse RawLokiResponse
 	var err error
@@ -186,13 +185,13 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest)
 		logsDataplane: isFeatureEnabled(ctx, flagLokiLogsDataplane),
 	}
 
-	return queryData(ctx, req, dsInfo, responseOpts, s.tracer, logger, isFeatureEnabled(ctx, flagLokiRunQueriesInParallel), isFeatureEnabled(ctx, flagLokiStructuredMetadata), isFeatureEnabled(ctx, flagLogQLScope))
+	return queryData(ctx, req, dsInfo, responseOpts, s.tracer, logger, isFeatureEnabled(ctx, flagLokiRunQueriesInParallel), isFeatureEnabled(ctx, flagLogQLScope))
 }
 
-func queryData(ctx context.Context, req *backend.QueryDataRequest, dsInfo *datasourceInfo, responseOpts ResponseOpts, tracer trace.Tracer, plog log.Logger, runInParallel bool, requestStructuredMetadata, logQLScopes bool) (*backend.QueryDataResponse, error) {
+func queryData(ctx context.Context, req *backend.QueryDataRequest, dsInfo *datasourceInfo, responseOpts ResponseOpts, tracer trace.Tracer, plog log.Logger, runInParallel bool, logQLScopes bool) (*backend.QueryDataResponse, error) {
 	result := backend.NewQueryDataResponse()
 
-	api := newLokiAPI(dsInfo.HTTPClient, dsInfo.URL, plog, tracer, requestStructuredMetadata)
+	api := newLokiAPI(dsInfo.HTTPClient, dsInfo.URL, plog, tracer)
 
 	start := time.Now()
 	queries, err := parseQuery(req, logQLScopes)

pkg/tsdb/loki/loki_bench_test.go

@@ -19,7 +19,7 @@ func BenchmarkMatrixJson(b *testing.B) {
 
 	b.ResetTimer()
 	for n := 0; n < b.N; n++ {
-		_, _ = runQuery(context.Background(), makeMockedAPI(http.StatusOK, "application/json", bytes, nil, false), &lokiQuery{}, ResponseOpts{}, backend.NewLoggerWith("logger", "test"))
+		_, _ = runQuery(context.Background(), makeMockedAPI(http.StatusOK, "application/json", bytes, nil), &lokiQuery{}, ResponseOpts{}, backend.NewLoggerWith("logger", "test"))
 	}
 }