[v11.5.x] TransformationFilter: Include transformation outputs in transformation filtering options (#99878 )

TransformationFilter: Include transformation outputs in transformation filtering options (#98323) * wip: include transformation output as filtering option * add refId to joinByField transformation * clean up * add refId to transformations that create new data frames * adjust duplicate query removal for filtering options * refactor transformation input/output subscription effect * adjust input data frame filtering logic to include transformations as input for debug view * transformation filter can only filter on output of previous transformation (cherry picked from commit a32eed1d13) Co-authored-by: Sergej-Vlasov <37613182+Sergej-Vlasov@users.noreply.github.com>
[v11.5.x] CodeEditor: Fix cursor alignment (#99863 )
2025-01-31 15:37:32 +00:00 · 2025-01-31 15:37:16 +00:00 · 2025-01-24 21:45:00 -05:00 · 2025-01-24 20:37:56 +00:00
30 changed files with 208 additions and 90 deletions
--- a/go.mod
+++ b/go.mod
@@ -64,7 +64,7 @@ require (
 	github.com/googleapis/gax-go/v2 v2.14.1 // @grafana/grafana-backend-group
 	github.com/gorilla/mux v1.8.1 // @grafana/grafana-backend-group
 	github.com/gorilla/websocket v1.5.3 // @grafana/grafana-app-platform-squad
-	github.com/grafana/alerting v0.0.0-20250110220613-267368fd1968 // @grafana/alerting-backend
+	github.com/grafana/alerting v0.0.0-20250123190916-7b528a0bc1d5 // @grafana/alerting-backend
 	github.com/grafana/authlib v0.0.0-20250108202437-7a039176d884 // @grafana/identity-access-team
 	github.com/grafana/authlib/claims v0.0.0-20241202085737-df90af04f335 // @grafana/identity-access-team
 	github.com/grafana/dataplane/examples v0.0.1 // @grafana/observability-metrics
--- a/go.sum
+++ b/go.sum
@@ -1489,8 +1489,8 @@ github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7Fsg
 github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
 github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
 github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/grafana/alerting v0.0.0-20250110220613-267368fd1968 h1:dSA0aOMzNnpBmYcmwv2OT5Is4kE7rubdSxo9GZSePAY=
-github.com/grafana/alerting v0.0.0-20250110220613-267368fd1968/go.mod h1:QsnoKX/iYZxA4Cv+H+wC7uxutBD8qi8ZW5UJvD2TYmU=
+github.com/grafana/alerting v0.0.0-20250123190916-7b528a0bc1d5 h1:mZezO6ccQl6AZv55f9JsPMph3eoHCofIJra2yhKzYMo=
+github.com/grafana/alerting v0.0.0-20250123190916-7b528a0bc1d5/go.mod h1:QsnoKX/iYZxA4Cv+H+wC7uxutBD8qi8ZW5UJvD2TYmU=
 github.com/grafana/authlib v0.0.0-20250108202437-7a039176d884 h1:MSRBiQrSJZ+iowjU4Tgtq8+uC5/cs9XdtUdSWCNHrNE=
 github.com/grafana/authlib v0.0.0-20250108202437-7a039176d884/go.mod h1:x7df73G3xuSD35Xv9cjaMLyPJCgM9Z/Wj5ISouoAfiI=
 github.com/grafana/authlib/claims v0.0.0-20241202085737-df90af04f335 h1:3DHH81RJCi8Bcgn2MdBh7vgWUshmAFjZzBCVuxiQ0uk=
--- a/go.work.sum
+++ b/go.work.sum
@@ -1697,6 +1697,7 @@ github.com/gomarkdown/markdown v0.0.0-20230922112808-5421fefb8386/go.mod h1:JDGc
 github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws=
 github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
 github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
+github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
 github.com/google/cel-go v0.17.1/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY=
 github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
 github.com/google/go-jsonnet v0.18.0 h1:/6pTy6g+Jh1a1I2UMoAODkqELFiVIdOxbNwv0DDzoOg=
--- a/package.json
+++ b/package.json
@@ -65,7 +65,7 @@
    "generate-apis": "rtk-query-codegen-openapi ./scripts/generate-rtk-apis.ts"
  },
  "grafana": {
-    "whatsNewUrl": "https://grafana.com/docs/grafana/next/whatsnew/whats-new-in-v11-4/",
+    "whatsNewUrl": "https://grafana.com/docs/grafana/next/whatsnew/whats-new-in-v11-5/",
    "releaseNotesUrl": "https://grafana.com/docs/grafana/next/release-notes/"
  },
  "devDependencies": {
--- a/packages/grafana-data/src/transformations/transformers/ensureColumns.test.ts
+++ b/packages/grafana-data/src/transformations/transformers/ensureColumns.test.ts
@@ -42,7 +42,10 @@ describe('ensureColumns transformer', () => {
      options: {},
    };

-    const data = [seriesA, seriesBC];
+    const data = [
+      { refId: 'A', ...seriesA },
+      { refId: 'B', ...seriesBC },
+    ];

    await expect(transformDataFrame([cfg], data)).toEmitValuesWith((received) => {
      const filtered = received[0];
@@ -109,6 +112,7 @@ describe('ensureColumns transformer', () => {
            },
          ],
          "length": 2,
+          "refId": "joinByField-A-B",
        }
      `);
    });
--- a/packages/grafana-data/src/transformations/transformers/histogram.ts
+++ b/packages/grafana-data/src/transformations/transformers/histogram.ts
@@ -592,5 +592,6 @@ export function histogramFieldsToFrame(info: HistogramFields, theme?: GrafanaThe
      type: DataFrameType.Histogram,
    },
    fields: [info.xMin, info.xMax, ...info.counts],
+    refId: `${DataTransformerID.histogram}`,
  };
 }
--- a/packages/grafana-data/src/transformations/transformers/joinByField.ts
+++ b/packages/grafana-data/src/transformations/transformers/joinByField.ts
@@ -42,6 +42,7 @@ export const joinByFieldTransformer: SynchronousDataTransformerInfo<JoinByFieldO
        }
        const joined = joinDataFrames({ frames: data, joinBy, mode: options.mode });
        if (joined) {
+          joined.refId = `${DataTransformerID.joinByField}-${data.map((frame) => frame.refId).join('-')}`;
          return [joined];
        }
      }
--- a/packages/grafana-data/src/transformations/transformers/merge.ts
+++ b/packages/grafana-data/src/transformations/transformers/merge.ts
@@ -43,7 +43,10 @@ export const mergeTransformer: DataTransformerInfo<MergeTransformerOptions> = {
        const fieldNames = new Set<string>();
        const fieldIndexByName: Record<string, Record<number, number>> = {};
        const fieldNamesForKey: string[] = [];
-        const dataFrame = new MutableDataFrame();
+        const dataFrame = new MutableDataFrame({
+          refId: `${DataTransformerID.merge}-${data.map((frame) => frame.refId).join('-')}`,
+          fields: [],
+        });

        for (let frameIndex = 0; frameIndex < data.length; frameIndex++) {
          const frame = data[frameIndex];
--- a/packages/grafana-data/src/transformations/transformers/reduce.ts
+++ b/packages/grafana-data/src/transformations/transformers/reduce.ts
@@ -56,7 +56,9 @@ export const reduceTransformer: DataTransformerInfo<ReduceTransformerOptions> =

        // Add a row for each series
        const res = reduceSeriesToRows(data, matcher, options.reducers, options.labelsToFields);
-        return res ? [res] : [];
+        return res
+          ? [{ ...res, refId: `${DataTransformerID.reduce}-${data.map((frame) => frame.refId).join('-')}` }]
+          : [];
      })
    ),
 };
--- a/packages/grafana-data/src/transformations/transformers/seriesToRows.ts
+++ b/packages/grafana-data/src/transformations/transformers/seriesToRows.ts
@@ -37,7 +37,10 @@ export const seriesToRowsTransformer: DataTransformerInfo<SeriesToRowsTransforme

        const timeFieldByIndex: Record<number, number> = {};
        const targetFields = new Set<string>();
-        const dataFrame = new MutableDataFrame();
+        const dataFrame = new MutableDataFrame({
+          refId: `${DataTransformerID.seriesToRows}-${data.map((frame) => frame.refId).join('-')}`,
+          fields: [],
+        });
        const metricField: Field = {
          name: TIME_SERIES_METRIC_FIELD_NAME,
          values: [],
--- a/packages/grafana-data/src/transformations/transformers/transpose.ts
+++ b/packages/grafana-data/src/transformations/transformers/transpose.ts
@@ -80,6 +80,7 @@ function transposeDataFrame(options: TransposeTransformerOptions, data: DataFram
      ...frame,
      fields: newFields,
      length: Math.max(...newFields.map((field) => field.values.length)),
+      refId: `${DataTransformerID.transpose}-${frame.refId}`,
    };
  });
 }
--- a/packages/grafana-ui/src/components/Monaco/ReactMonacoEditor.tsx
+++ b/packages/grafana-ui/src/components/Monaco/ReactMonacoEditor.tsx
@@ -11,7 +11,7 @@ import type { ReactMonacoEditorProps } from './types';
 monacoEditorLoader.config({ monaco });

 export const ReactMonacoEditor = (props: ReactMonacoEditorProps) => {
-  const { beforeMount, options, ...restProps } = props;
+  const { beforeMount, onMount, options, ...restProps } = props;

  const theme = useTheme2();
  const onMonacoBeforeMount = useCallback(
@@ -31,6 +31,15 @@ export const ReactMonacoEditor = (props: ReactMonacoEditorProps) => {
      }}
      theme={theme.isDark ? 'grafana-dark' : 'grafana-light'}
      beforeMount={onMonacoBeforeMount}
+      onMount={(editor, monaco) => {
+        // we use a custom font in our monaco editor
+        // we need monaco to remeasure the fonts after they are loaded to prevent alignment issues
+        // see https://github.com/microsoft/monaco-editor/issues/648#issuecomment-564978560
+        document.fonts.ready.then(() => {
+          monaco.editor.remeasureFonts();
+        });
+        onMount?.(editor, monaco);
+      }}
    />
  );
 };
--- a/pkg/services/ngalert/api/tooling/definitions/contact_points.go
+++ b/pkg/services/ngalert/api/tooling/definitions/contact_points.go
@@ -267,7 +267,7 @@ type ThreemaIntegration struct {
 type VictoropsIntegration struct {
 	DisableResolveMessage *bool `json:"-" yaml:"-" hcl:"disable_resolve_message"`

-	URL string `json:"url" yaml:"url" hcl:"url"`
+	URL Secret `json:"url" yaml:"url" hcl:"url"`

 	MessageType *string `json:"messageType,omitempty" yaml:"messageType,omitempty" hcl:"message_type"`
 	Title       *string `json:"title,omitempty" yaml:"title,omitempty" hcl:"title"`
--- a/pkg/services/ngalert/notifier/channels_config/available_channels.go
+++ b/pkg/services/ngalert/notifier/channels_config/available_channels.go
@@ -388,6 +388,7 @@ func GetAvailableNotifiers() []*NotifierPlugin {
 					Placeholder:  "VictorOps url",
 					PropertyName: "url",
 					Required:     true,
+					Secure:       true,
 				},
 				{ // New in 8.0.
 					Label:        "Message Type",
--- a/pkg/services/ngalert/notifier/channels_config/available_channels_test.go
+++ b/pkg/services/ngalert/notifier/channels_config/available_channels_test.go
@@ -15,7 +15,7 @@ func TestGetSecretKeysForContactPointType(t *testing.T) {
 		{receiverType: "kafka", expectedSecretFields: []string{"password"}},
 		{receiverType: "email", expectedSecretFields: []string{}},
 		{receiverType: "pagerduty", expectedSecretFields: []string{"integrationKey"}},
-		{receiverType: "victorops", expectedSecretFields: []string{}},
+		{receiverType: "victorops", expectedSecretFields: []string{"url"}},
 		{receiverType: "oncall", expectedSecretFields: []string{"password", "authorization_credentials"}},
 		{receiverType: "pushover", expectedSecretFields: []string{"apiToken", "userKey"}},
 		{receiverType: "slack", expectedSecretFields: []string{"token", "url"}},
--- a/pkg/services/ngalert/notifier/receiver_svc.go
+++ b/pkg/services/ngalert/notifier/receiver_svc.go
@@ -485,6 +485,14 @@ func (rs *ReceiverService) UpdateReceiver(ctx context.Context, r *models.Receive
 		return nil, err
 	}

+	// We re-encrypt the existing receiver to ensure any unencrypted secure fields that are correctly encrypted, note this should NOT re-encrypt secure fields that are already encrypted.
+	// This is rare, but can happen if a receiver is created with unencrypted secure fields and then the secure option is added later.
+	// Preferably, this would be handled by receiver config versions and migrations but for now this is a good safety net.
+	err = existing.Encrypt(rs.encryptor(ctx))
+	if err != nil {
+		return nil, err
+	}
+
 	span.AddEvent("Loaded current receiver", trace.WithAttributes(
 		attribute.String("concurrency_token", revision.ConcurrencyToken),
 		attribute.String("receiver", existing.Name),
--- a/pkg/services/ngalert/notifier/receiver_svc_test.go
+++ b/pkg/services/ngalert/notifier/receiver_svc_test.go
@@ -583,6 +583,44 @@ func TestReceiverService_Update(t *testing.T) {
 			), rm.Encrypted(models.Base64Enrypt)),
 			expectedProvenances: map[string]models.Provenance{slackIntegration.UID: models.ProvenanceNone},
 		},
+		{
+			name: "encrypts previously unencrypted secure fields",
+			user: writer,
+			receiver: models.CopyReceiverWith(baseReceiver, rm.WithIntegrations(
+				models.CopyIntegrationWith(slackIntegration, im.AddSetting("token", "unencryptedValue"))),
+			),
+			existing: util.Pointer(models.CopyReceiverWith(baseReceiver, rm.WithIntegrations(
+				models.CopyIntegrationWith(slackIntegration,
+					im.AddSetting("token", "unencryptedValue"), // This will get encrypted.
+				),
+			))),
+			expectedUpdate: models.CopyReceiverWith(baseReceiver, rm.WithIntegrations(
+				models.CopyIntegrationWith(slackIntegration,
+					im.AddSecureSetting("token", "dW5lbmNyeXB0ZWRWYWx1ZQ==")),
+			), rm.Encrypted(models.Base64Enrypt)),
+			expectedProvenances: map[string]models.Provenance{slackIntegration.UID: models.ProvenanceNone},
+		},
+		{
+			// This test is important for covering the rare case when an existing field is marked as secure.
+			// The UI will receive the field as secure and, if unchanged, will pass it back on update as a secureField instead of a Setting.
+			name: "encrypts previously unencrypted secure fields when passed in as secureFields",
+			user: writer,
+			receiver: models.CopyReceiverWith(baseReceiver, rm.WithIntegrations(
+				models.CopyIntegrationWith(slackIntegration, im.AddSetting("newField", "newValue"))),
+			),
+			secureFields: map[string][]string{slackIntegration.UID: {"token"}},
+			existing: util.Pointer(models.CopyReceiverWith(baseReceiver, rm.WithIntegrations(
+				models.CopyIntegrationWith(slackIntegration,
+					im.AddSetting("token", "unencryptedValue"), // This will get encrypted.
+				),
+			))),
+			expectedUpdate: models.CopyReceiverWith(baseReceiver, rm.WithIntegrations(
+				models.CopyIntegrationWith(slackIntegration,
+					im.AddSetting("newField", "newValue"),
+					im.AddSecureSetting("token", "dW5lbmNyeXB0ZWRWYWx1ZQ==")),
+			), rm.Encrypted(models.Base64Enrypt)),
+			expectedProvenances: map[string]models.Provenance{slackIntegration.UID: models.ProvenanceNone},
+		},
 		{
 			name: "doesn't copy existing unsecure fields",
 			user: writer,
@@ -684,8 +722,22 @@ func TestReceiverService_Update(t *testing.T) {
 			sut := createReceiverServiceSut(t, &secretsService)

 			if tc.existing != nil {
-				created, err := sut.CreateReceiver(context.Background(), tc.existing, tc.user.GetOrgID(), tc.user)
+				// Create route after receivers as they will be referenced.
+				revision, err := sut.cfgStore.Get(context.Background(), tc.user.GetOrgID())
 				require.NoError(t, err)
+				result, err := revision.CreateReceiver(tc.existing)
+				require.NoError(t, err)
+
+				created, err := PostableApiReceiverToReceiver(result, tc.existing.Provenance)
+				require.NoError(t, err)
+				err = sut.cfgStore.Save(context.Background(), revision, tc.user.GetOrgID())
+				require.NoError(t, err)
+
+				for _, integration := range created.Integrations {
+					target := definitions.EmbeddedContactPoint{UID: integration.UID}
+					err = sut.provisioningStore.SetProvenance(context.Background(), &target, tc.user.GetOrgID(), created.Provenance)
+					require.NoError(t, err)
+				}

 				if tc.version == "" {
 					tc.version = created.Version
--- a/pkg/storage/unified/apistore/go.mod
+++ b/pkg/storage/unified/apistore/go.mod
@@ -169,7 +169,7 @@ require (
 	github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
 	github.com/googleapis/gax-go/v2 v2.14.1 // indirect
 	github.com/gorilla/mux v1.8.1 // indirect
-	github.com/grafana/alerting v0.0.0-20250110220613-267368fd1968 // indirect
+	github.com/grafana/alerting v0.0.0-20250123190916-7b528a0bc1d5 // indirect
 	github.com/grafana/authlib v0.0.0-20250108202437-7a039176d884 // indirect
 	github.com/grafana/dataplane/sdata v0.0.9 // indirect
 	github.com/grafana/dskit v0.0.0-20241105154643-a6b453a88040 // indirect
--- a/pkg/storage/unified/apistore/go.sum
+++ b/pkg/storage/unified/apistore/go.sum
@@ -543,8 +543,8 @@ github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
 github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
 github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
 github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/grafana/alerting v0.0.0-20250110220613-267368fd1968 h1:dSA0aOMzNnpBmYcmwv2OT5Is4kE7rubdSxo9GZSePAY=
-github.com/grafana/alerting v0.0.0-20250110220613-267368fd1968/go.mod h1:QsnoKX/iYZxA4Cv+H+wC7uxutBD8qi8ZW5UJvD2TYmU=
+github.com/grafana/alerting v0.0.0-20250123190916-7b528a0bc1d5 h1:mZezO6ccQl6AZv55f9JsPMph3eoHCofIJra2yhKzYMo=
+github.com/grafana/alerting v0.0.0-20250123190916-7b528a0bc1d5/go.mod h1:QsnoKX/iYZxA4Cv+H+wC7uxutBD8qi8ZW5UJvD2TYmU=
 github.com/grafana/authlib v0.0.0-20250108202437-7a039176d884 h1:MSRBiQrSJZ+iowjU4Tgtq8+uC5/cs9XdtUdSWCNHrNE=
 github.com/grafana/authlib v0.0.0-20250108202437-7a039176d884/go.mod h1:x7df73G3xuSD35Xv9cjaMLyPJCgM9Z/Wj5ISouoAfiI=
 github.com/grafana/authlib/claims v0.0.0-20241202085737-df90af04f335 h1:3DHH81RJCi8Bcgn2MdBh7vgWUshmAFjZzBCVuxiQ0uk=
--- a/pkg/storage/unified/resource/go.mod
+++ b/pkg/storage/unified/resource/go.mod
@@ -116,7 +116,7 @@ require (
 	github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
 	github.com/googleapis/gax-go/v2 v2.14.1 // indirect
 	github.com/gorilla/mux v1.8.1 // indirect
-	github.com/grafana/alerting v0.0.0-20250110220613-267368fd1968 // indirect
+	github.com/grafana/alerting v0.0.0-20250123190916-7b528a0bc1d5 // indirect
 	github.com/grafana/dataplane/sdata v0.0.9 // indirect
 	github.com/grafana/grafana-aws-sdk v0.31.5 // indirect
 	github.com/grafana/grafana-azure-sdk-go/v2 v2.1.2 // indirect
--- a/pkg/storage/unified/resource/go.sum
+++ b/pkg/storage/unified/resource/go.sum
@@ -398,8 +398,8 @@ github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z
 github.com/gorilla/mux v1.7.1/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
 github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
-github.com/grafana/alerting v0.0.0-20250110220613-267368fd1968 h1:dSA0aOMzNnpBmYcmwv2OT5Is4kE7rubdSxo9GZSePAY=
-github.com/grafana/alerting v0.0.0-20250110220613-267368fd1968/go.mod h1:QsnoKX/iYZxA4Cv+H+wC7uxutBD8qi8ZW5UJvD2TYmU=
+github.com/grafana/alerting v0.0.0-20250123190916-7b528a0bc1d5 h1:mZezO6ccQl6AZv55f9JsPMph3eoHCofIJra2yhKzYMo=
+github.com/grafana/alerting v0.0.0-20250123190916-7b528a0bc1d5/go.mod h1:QsnoKX/iYZxA4Cv+H+wC7uxutBD8qi8ZW5UJvD2TYmU=
 github.com/grafana/authlib v0.0.0-20250108202437-7a039176d884 h1:MSRBiQrSJZ+iowjU4Tgtq8+uC5/cs9XdtUdSWCNHrNE=
 github.com/grafana/authlib v0.0.0-20250108202437-7a039176d884/go.mod h1:x7df73G3xuSD35Xv9cjaMLyPJCgM9Z/Wj5ISouoAfiI=
 github.com/grafana/authlib/claims v0.0.0-20241202085737-df90af04f335 h1:3DHH81RJCi8Bcgn2MdBh7vgWUshmAFjZzBCVuxiQ0uk=
--- a/public/app/features/dashboard/components/TransformationsEditor/TransformationEditor.tsx
+++ b/public/app/features/dashboard/components/TransformationsEditor/TransformationEditor.tsx
@@ -1,26 +1,17 @@
 import { css } from '@emotion/css';
-import { createElement, useEffect, useMemo, useState } from 'react';
-import { mergeMap } from 'rxjs/operators';
+import { createElement, useMemo } from 'react';

-import {
-  DataFrame,
-  DataTransformerConfig,
-  GrafanaTheme2,
-  transformDataFrame,
-  TransformerRegistryItem,
-  getFrameMatchers,
-  DataTransformContext,
-} from '@grafana/data';
+import { DataFrame, DataTransformerConfig, GrafanaTheme2, TransformerRegistryItem } from '@grafana/data';
 import { selectors } from '@grafana/e2e-selectors';
-import { getTemplateSrv } from '@grafana/runtime';
 import { Icon, JSONFormatter, useStyles2, Drawer } from '@grafana/ui';

 import { TransformationsEditorTransformation } from './types';

 interface TransformationEditorProps {
+  input: DataFrame[];
+  output: DataFrame[];
  debugMode?: boolean;
  index: number;
-  data: DataFrame[];
  uiConfig: TransformerRegistryItem;
  configs: TransformationsEditorTransformation[];
  onChange: (index: number, config: DataTransformerConfig) => void;
@@ -28,45 +19,18 @@ interface TransformationEditorProps {
 }

 export const TransformationEditor = ({
+  input,
+  output,
  debugMode,
  index,
-  data,
  uiConfig,
  configs,
  onChange,
  toggleShowDebug,
 }: TransformationEditorProps) => {
  const styles = useStyles2(getStyles);
-  const [input, setInput] = useState<DataFrame[]>([]);
-  const [output, setOutput] = useState<DataFrame[]>([]);
  const config = useMemo(() => configs[index], [configs, index]);

-  useEffect(() => {
-    const config = configs[index].transformation;
-    const matcher = config.filter?.options ? getFrameMatchers(config.filter) : undefined;
-    const inputTransforms = configs.slice(0, index).map((t) => t.transformation);
-    const outputTransforms = configs.slice(index, index + 1).map((t) => t.transformation);
-
-    const ctx: DataTransformContext = {
-      interpolate: (v: string) => getTemplateSrv().replace(v),
-    };
-
-    const inputSubscription = transformDataFrame(inputTransforms, data, ctx).subscribe((v) => {
-      if (matcher) {
-        v = data.filter((v) => matcher(v));
-      }
-      setInput(v);
-    });
-    const outputSubscription = transformDataFrame(inputTransforms, data, ctx)
-      .pipe(mergeMap((before) => transformDataFrame(outputTransforms, before, ctx)))
-      .subscribe(setOutput);
-
-    return function unsubscribe() {
-      inputSubscription.unsubscribe();
-      outputSubscription.unsubscribe();
-    };
-  }, [index, data, configs]);
-
  const editor = useMemo(
    () =>
      createElement(uiConfig.editor, {
--- a/public/app/features/dashboard/components/TransformationsEditor/TransformationFilter.tsx
+++ b/public/app/features/dashboard/components/TransformationsEditor/TransformationFilter.tsx
@@ -1,40 +1,35 @@
 import { css } from '@emotion/css';
 import { useMemo } from 'react';

-import {
-  DataTransformerConfig,
-  GrafanaTheme2,
-  StandardEditorContext,
-  StandardEditorsRegistryItem,
-} from '@grafana/data';
+import { DataFrame, DataTransformerConfig, GrafanaTheme2 } from '@grafana/data';
 import { DataTopic } from '@grafana/schema';
 import { Field, Select, useStyles2 } from '@grafana/ui';
 import { FrameMultiSelectionEditor } from 'app/plugins/panel/geomap/editor/FrameSelectionEditor';

-import { TransformationData } from './TransformationsEditor';
-
 interface TransformationFilterProps {
+  /** data frames from the output of previous transformation */
+  data: DataFrame[];
  index: number;
  config: DataTransformerConfig;
-  data: TransformationData;
+  annotations?: DataFrame[];
  onChange: (index: number, config: DataTransformerConfig) => void;
 }

-export const TransformationFilter = ({ index, data, config, onChange }: TransformationFilterProps) => {
+export const TransformationFilter = ({ index, annotations, config, onChange, data }: TransformationFilterProps) => {
  const styles = useStyles2(getStyles);

  const opts = useMemo(() => {
    return {
      // eslint-disable-next-line
-      context: { data: data.series } as StandardEditorContext<unknown>,
-      showTopic: true || data.annotations?.length || config.topic?.length,
+      context: { data },
+      showTopic: true || annotations?.length || config.topic?.length,
      showFilter: config.topic !== DataTopic.Annotations,
      source: [
-        { value: DataTopic.Series, label: `Query results` },
+        { value: DataTopic.Series, label: `Query and Transformation results` },
        { value: DataTopic.Annotations, label: `Annotation data` },
      ],
    };
-  }, [data, config.topic]);
+  }, [data, annotations?.length, config.topic]);

  return (
    <div className={styles.wrapper}>
@@ -59,8 +54,6 @@ export const TransformationFilter = ({ index, data, config, onChange }: Transfor
            <FrameMultiSelectionEditor
              value={config.filter!}
              context={opts.context}
-              // eslint-disable-next-line
-              item={{} as StandardEditorsRegistryItem}
              onChange={(filter) => onChange(index, { ...config, filter })}
            />
          )}
--- a/public/app/features/dashboard/components/TransformationsEditor/TransformationOperationRow.tsx
+++ b/public/app/features/dashboard/components/TransformationsEditor/TransformationOperationRow.tsx
@@ -1,10 +1,18 @@
-import { useCallback } from 'react';
-import * as React from 'react';
+import { useCallback, useEffect, useState } from 'react';
 import { useToggle } from 'react-use';
+import { mergeMap } from 'rxjs';

-import { DataTransformerConfig, TransformerRegistryItem, FrameMatcherID, DataTopic } from '@grafana/data';
+import {
+  DataTransformerConfig,
+  TransformerRegistryItem,
+  FrameMatcherID,
+  DataTransformContext,
+  getFrameMatchers,
+  transformDataFrame,
+  DataFrame,
+} from '@grafana/data';
 import { selectors } from '@grafana/e2e-selectors';
-import { reportInteraction } from '@grafana/runtime';
+import { getTemplateSrv, reportInteraction } from '@grafana/runtime';
 import { ConfirmModal } from '@grafana/ui';
 import {
  QueryOperationAction,
@@ -46,6 +54,10 @@ export const TransformationOperationRow = ({
  const topic = configs[index].transformation.topic;
  const showFilterEditor = configs[index].transformation.filter != null || topic != null;
  const showFilterToggle = showFilterEditor || data.series.length > 0 || (data.annotations?.length ?? 0) > 0;
+  const [input, setInput] = useState<DataFrame[]>([]);
+  const [output, setOutput] = useState<DataFrame[]>([]);
+  // output of previous transformation
+  const [prevOutput, setPrevOutput] = useState<DataFrame[]>([]);

  const onDisableToggle = useCallback(
    (index: number) => {
@@ -92,6 +104,48 @@ export const TransformationOperationRow = ({
    [configs, index]
  );

+  useEffect(() => {
+    const config = configs[index].transformation;
+    const matcher = config.filter?.options ? getFrameMatchers(config.filter) : undefined;
+    // we need previous transformation index to get its outputs
+    //    to be used in this transforms inputs
+    const prevTransformIndex = index - 1;
+
+    let prevInputTransforms: Array<DataTransformerConfig<{}>> = [];
+    let prevOutputTransforms: Array<DataTransformerConfig<{}>> = [];
+
+    if (prevTransformIndex >= 0) {
+      prevInputTransforms = configs.slice(0, prevTransformIndex).map((t) => t.transformation);
+      prevOutputTransforms = configs.slice(prevTransformIndex, index).map((t) => t.transformation);
+    }
+
+    const inputTransforms = configs.slice(0, index).map((t) => t.transformation);
+    const outputTransforms = configs.slice(index, index + 1).map((t) => t.transformation);
+
+    const ctx: DataTransformContext = {
+      interpolate: (v: string) => getTemplateSrv().replace(v),
+    };
+
+    const inputSubscription = transformDataFrame(inputTransforms, data.series, ctx).subscribe((data) => {
+      if (matcher) {
+        data = data.filter((frame) => matcher(frame));
+      }
+      setInput(data);
+    });
+    const outputSubscription = transformDataFrame(inputTransforms, data.series, ctx)
+      .pipe(mergeMap((before) => transformDataFrame(outputTransforms, before, ctx)))
+      .subscribe(setOutput);
+    const prevOutputSubscription = transformDataFrame(prevInputTransforms, data.series, ctx)
+      .pipe(mergeMap((before) => transformDataFrame(prevOutputTransforms, before, ctx)))
+      .subscribe(setPrevOutput);
+
+    return function unsubscribe() {
+      inputSubscription.unsubscribe();
+      outputSubscription.unsubscribe();
+      prevOutputSubscription.unsubscribe();
+    };
+  }, [index, data, configs]);
+
  const renderActions = () => {
    return (
      <>
@@ -162,13 +216,20 @@ export const TransformationOperationRow = ({
        }}
      >
        {showFilterEditor && (
-          <TransformationFilter index={index} config={configs[index].transformation} data={data} onChange={onChange} />
+          <TransformationFilter
+            data={prevOutput}
+            index={index}
+            config={configs[index].transformation}
+            annotations={data.annotations}
+            onChange={onChange}
+          />
        )}

        <TransformationEditor
+          input={input}
+          output={output}
          debugMode={showDebug}
          index={index}
-          data={topic === DataTopic.Annotations ? (data.annotations ?? []) : data.series}
          configs={configs}
          uiConfig={uiConfig}
          onChange={onChange}
--- a/public/app/features/transformers/joinByLabels/joinByLabels.ts
+++ b/public/app/features/transformers/joinByLabels/joinByLabels.ts
@@ -35,7 +35,7 @@ interface JoinValues {

 export function joinByLabels(options: JoinByLabelsTransformOptions, data: DataFrame[]): DataFrame {
  if (!options.value?.length) {
-    return getErrorFrame('No value labele configured');
+    return getErrorFrame('No value label configured');
  }
  const distinctLabels = getDistinctLabels(data);
  if (distinctLabels.size < 1) {
@@ -104,7 +104,11 @@ export function joinByLabels(options: JoinByLabelsTransformOptions, data: DataFr
    }
  }

-  const frame: DataFrame = { fields: [], length: nameValues[0].length };
+  const frame: DataFrame = {
+    fields: [],
+    length: nameValues[0].length,
+    refId: `${DataTransformerID.joinByLabels}-${data.map((frame) => frame.refId).join('-')}`,
+  };
  for (let i = 0; i < join.length; i++) {
    frame.fields.push({
      name: join[i],
--- a/public/app/features/transformers/rowsToFields/rowsToFields.test.ts
+++ b/public/app/features/transformers/rowsToFields/rowsToFields.test.ts
@@ -12,6 +12,7 @@ describe('Rows to fields', () => {
        { name: 'Miiin', type: FieldType.number, values: [3, 100] },
        { name: 'max', type: FieldType.string, values: [15, 200] },
      ],
+      refId: 'A',
    });

    const result = rowsToFields(
@@ -57,6 +58,7 @@ describe('Rows to fields', () => {
          },
        ],
        "length": 1,
+        "refId": "rowsToFields-A",
      }
    `);
  });
--- a/public/app/features/transformers/rowsToFields/rowsToFields.ts
+++ b/public/app/features/transformers/rowsToFields/rowsToFields.ts
@@ -64,6 +64,7 @@ export function rowsToFields(options: RowToFieldsTransformOptions, data: DataFra
  return {
    fields: outFields,
    length: 1,
+    refId: `${DataTransformerID.rowsToFields}-${data.refId}`,
  };
 }

--- a/public/app/plugins/panel/datagrid/DataGridPanel.test.tsx
+++ b/public/app/plugins/panel/datagrid/DataGridPanel.test.tsx
@@ -59,15 +59,17 @@ describe('DataGrid', () => {
      jest.clearAllMocks();
    });

-    it('converts dataframe values to cell values properly', () => {
+    it('converts dataframe values to cell values properly', async () => {
      jest.useFakeTimers();
      render(<DataGridPanel {...props} />);
      prep(false);

-      expect(screen.getByTestId('glide-cell-1-0')).toHaveTextContent('1');
-      expect(screen.getByTestId('glide-cell-2-1')).toHaveTextContent('b');
-      expect(screen.getByTestId('glide-cell-3-2')).toHaveTextContent('c');
-      expect(screen.getByTestId('glide-cell-3-3')).toHaveTextContent('d');
+      await waitFor(() => {
+        expect(screen.getByTestId('glide-cell-1-0')).toHaveTextContent('1');
+        expect(screen.getByTestId('glide-cell-2-1')).toHaveTextContent('b');
+        expect(screen.getByTestId('glide-cell-3-2')).toHaveTextContent('c');
+        expect(screen.getByTestId('glide-cell-3-3')).toHaveTextContent('d');
+      });
    });

    it('should open context menu on right click', async () => {
--- a/public/app/plugins/panel/geomap/editor/FrameSelectionEditor.tsx
+++ b/public/app/plugins/panel/geomap/editor/FrameSelectionEditor.tsx
@@ -29,7 +29,9 @@ export const FrameSelectionEditor = ({ value, context, onChange }: Props) => {
  );
 };

-export const FrameMultiSelectionEditor = ({ value, context, onChange }: Props) => {
+type FrameMultiSelectionEditorProps = Omit<StandardEditorProps<MatcherConfig>, 'item'>;
+
+export const FrameMultiSelectionEditor = ({ value, context, onChange }: FrameMultiSelectionEditorProps) => {
  const onFilterChange = useCallback(
    (v: string[]) => {
      onChange(
--- a/public/test/jest-setup.ts
+++ b/public/test/jest-setup.ts
@@ -61,6 +61,9 @@ const mockIntersectionObserver = jest
    disconnect: jest.fn(),
  }));
 global.IntersectionObserver = mockIntersectionObserver;
+Object.defineProperty(document, 'fonts', {
+  value: { ready: Promise.resolve({}) },
+});

 global.TextEncoder = TextEncoder;
 global.TextDecoder = TextDecoder;
Author	SHA1	Message	Date
grafana-delivery-bot[bot]	c6c701cf5b	[v11.5.x] TransformationFilter: Include transformation outputs in transformation filtering options (#99878 ) TransformationFilter: Include transformation outputs in transformation filtering options (#98323) * wip: include transformation output as filtering option * add refId to joinByField transformation * clean up * add refId to transformations that create new data frames * adjust duplicate query removal for filtering options * refactor transformation input/output subscription effect * adjust input data frame filtering logic to include transformations as input for debug view * transformation filter can only filter on output of previous transformation (cherry picked from commit `a32eed1d13`) Co-authored-by: Sergej-Vlasov <37613182+Sergej-Vlasov@users.noreply.github.com>	2025-01-31 15:37:32 +00:00
grafana-delivery-bot[bot]	c42e00bdac	[v11.5.x] CodeEditor: Fix cursor alignment (#99863 ) CodeEditor: Fix cursor alignment (#99090) * remeasure fonts once they've loaded * add test mock * fix unit test * remeasure fonts after the editor has mounted just to be safe (cherry picked from commit `8e59f618c1`) Co-authored-by: Ashley Harrison <ashley.harrison@grafana.com>	2025-01-31 15:37:16 +00:00
Brian Gann	f7a938db9a	update whats-new	2025-01-24 21:45:00 -05:00
github-actions[bot]	b9513983ce	apply security patch: v11.5.x/305-202501232115.patch commit 874ce8d12caad3742857ca86d2da7d5f81f3f825 Author: Matt Jacobson <matthew.jacobson@grafana.com> Date: Thu Jan 23 16:14:28 2025 -0500 linting commit c4b6d9194cc8b79e252e562a27a2d09a42d7a5e8 Author: Matt Jacobson <matthew.jacobson@grafana.com> Date: Thu Jan 23 14:56:35 2025 -0500 CVE-2024-11741 - victorops url	2025-01-24 20:37:56 +00:00