Dashboard Schema v2: Transformation to v2 supports annotations (#96787)

* Annotations * Default to configured default DS when datasource is undefined --------- Co-authored-by: Ivan Ortega <ivanortegaalba@gmail.com> Co-authored-by: alexandra vargas <alexa1866@gmail.com>
2024-11-28 12:36:09 +02:00
parent a4fff11438
commit 4721d5164b
7 changed files with 271 additions and 40 deletions
@@ -463,11 +463,9 @@ export const defaultVizConfigKind = (): VizConfigKind => ({
 });

 export interface AnnotationQuerySpec {
-	datasource: DataSourceRef;
+	datasource?: DataSourceRef;
 	query: DataQueryKind;
-	// TODO: Should be figured out based on datasource (Grafana ds)
-	// builtIn?: int
-	// Below are currently existing options for annotation queries
+	builtIn?: boolean;
 	enable: boolean;
 	filter: AnnotationPanelFilter;
 	hide: boolean;
@@ -476,7 +474,6 @@ export interface AnnotationQuerySpec {
 }

 export const defaultAnnotationQuerySpec = (): AnnotationQuerySpec => ({
-	datasource: defaultDataSourceRef(),
 	query: defaultDataQueryKind(),
 	enable: false,
 	filter: defaultAnnotationPanelFilter(),
@@ -521,14 +518,13 @@ export const defaultDataQueryKind = (): DataQueryKind => ({

 export interface PanelQuerySpec {
 	query: DataQueryKind;
-	datasource: DataSourceRef;
+	datasource?: DataSourceRef;
 	refId: string;
 	hidden: boolean;
 }

 export const defaultPanelQuerySpec = (): PanelQuerySpec => ({
 	query: defaultDataQueryKind(),
-	datasource: defaultDataSourceRef(),
 	refId: "",
 	hidden: false,
 });
@@ -859,7 +855,7 @@ export interface QueryVariableSpec {
 	refresh: VariableRefresh;
 	skipUrlSync: boolean;
 	description?: string;
-	datasource: DataSourceRef;
+	datasource?: DataSourceRef;
 	query: string | DataQueryKind;
 	regex: string;
 	sort: VariableSort;
@@ -877,7 +873,6 @@ export const defaultQueryVariableSpec = (): QueryVariableSpec => ({
 	hide: "dontHide",
 	refresh: "never",
 	skipUrlSync: false,
-	datasource: defaultDataSourceRef(),
 	query: "",
 	regex: "",
 	sort: "disabled",
@@ -1080,7 +1075,7 @@ export const defaultCustomVariableKind = (): CustomVariableKind => ({
 // GroupBy variable specification
 export interface GroupByVariableSpec {
 	name: string;
-	datasource: DataSourceRef;
+	datasource?: DataSourceRef;
 	current: VariableOption;
 	options: VariableOption[];
 	multi: boolean;
@@ -1094,7 +1089,6 @@ export interface GroupByVariableSpec {

 export const defaultGroupByVariableSpec = (): GroupByVariableSpec => ({
 	name: "",
-	datasource: defaultDataSourceRef(),
 	current: { text: "", value: "", },
 	options: [],
 	multi: false,
@@ -1117,7 +1111,7 @@ export const defaultGroupByVariableKind = (): GroupByVariableKind => ({
 // Adhoc variable specification
 export interface AdhocVariableSpec {
 	name: string;
-	datasource: DataSourceRef;
+	datasource?: DataSourceRef;
 	baseFilters: AdHocFilterWithLabels[];
 	filters: AdHocFilterWithLabels[];
 	defaultKeys: MetricFindValue[];
@@ -1129,7 +1123,6 @@ export interface AdhocVariableSpec {

 export const defaultAdhocVariableSpec = (): AdhocVariableSpec => ({
 	name: "",
-	datasource: defaultDataSourceRef(),
 	baseFilters: [],
 	filters: [],
 	defaultKeys: [],
@@ -362,12 +362,9 @@ VizConfigKind: {
 }

 AnnotationQuerySpec: {
-  datasource: DataSourceRef
+  datasource?: DataSourceRef
  query: DataQueryKind
-
-  // TODO: Should be figured out based on datasource (Grafana ds)
-  // builtIn?: int
-  // Below are currently existing options for annotation queries
+  builtIn?: bool
  enable: bool
  filter: AnnotationPanelFilter
  hide: bool
@@ -398,7 +395,7 @@ DataQueryKind: {

 PanelQuerySpec: {
  query: DataQueryKind
-  datasource: DataSourceRef
+  datasource?: DataSourceRef

  refId: string
  hidden: bool
@@ -598,7 +595,7 @@ QueryVariableSpec: {
  refresh: VariableRefresh
  skipUrlSync: bool | *false
  description?: string
-  datasource: DataSourceRef | *{}
+  datasource?: DataSourceRef
  query: string | DataQueryKind | *""
  regex: string | *""
  sort: VariableSort
@@ -732,7 +729,7 @@ CustomVariableKind: {
 // GroupBy variable specification
 GroupByVariableSpec: {
  name: string | *""
-  datasource: DataSourceRef | *{}
+  datasource?: DataSourceRef
  current: VariableOption | *{
    text: ""
    value: ""
@@ -756,7 +753,7 @@ GroupByVariableKind: {
 // Adhoc variable specification
 AdhocVariableSpec: {
  name: string | *""
-  datasource: DataSourceRef | *{}
+  datasource?: DataSourceRef
  baseFilters: [...AdHocFilterWithLabels] | *[]
  filters: [...AdHocFilterWithLabels] | *[]
  defaultKeys: [...MetricFindValue] | *[]
@@ -2,7 +2,86 @@

 exports[`transformSceneToSaveModelSchemaV2 should transform scene to save model schema v2 1`] = `
 {
-  "annotations": [],
+  "annotations": [
+    {
+      "kind": "AnnotationQuery",
+      "spec": {
+        "builtIn": false,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --",
+        },
+        "enable": true,
+        "filter": {
+          "exclude": false,
+          "ids": [],
+        },
+        "hide": false,
+        "iconColor": "red",
+        "name": "query1",
+        "query": {
+          "kind": "grafana",
+          "spec": {
+            "enable": true,
+            "iconColor": "red",
+            "name": "query1",
+          },
+        },
+      },
+    },
+    {
+      "kind": "AnnotationQuery",
+      "spec": {
+        "builtIn": false,
+        "datasource": {
+          "type": "prometheus",
+          "uid": "abcdef",
+        },
+        "enable": true,
+        "filter": {
+          "exclude": false,
+          "ids": [],
+        },
+        "hide": true,
+        "iconColor": "blue",
+        "name": "query2",
+        "query": {
+          "kind": "prometheus",
+          "spec": {
+            "enable": true,
+            "iconColor": "blue",
+            "name": "query2",
+          },
+        },
+      },
+    },
+    {
+      "kind": "AnnotationQuery",
+      "spec": {
+        "builtIn": false,
+        "datasource": {
+          "type": "loki",
+          "uid": "Loki",
+        },
+        "enable": true,
+        "filter": {
+          "exclude": false,
+          "ids": [],
+        },
+        "hide": true,
+        "iconColor": "green",
+        "name": "query3",
+        "query": {
+          "kind": "loki",
+          "spec": {
+            "enable": true,
+            "iconColor": "green",
+            "name": "query3",
+          },
+        },
+      },
+    },
+  ],
  "cursorSync": "Crosshair",
  "description": "Test Description",
  "editable": true,
@@ -185,13 +185,11 @@ describe('transformSaveModelSchemaV2ToScene', () => {
    expect(getQueryRunnerFor(vizPanels[0])?.state.datasource).toBeUndefined();
  });

-  // Skipping the test because the schema doesn't accept the ds to be undefined.
-  // In future PR, we will mark it as optional so, this test should pass and the runtime code should be updated.
-  it.skip('should set panel ds as undefined if it is not mixed DS', () => {
+  it('should set panel ds as mixed if one ds is undefined', () => {
    const dashboard = cloneDeep(defaultDashboard);
+
    dashboard.spec.elements['test-panel-uid'].spec.data.spec.queries.push({
      kind: 'PanelQuery',
-      // @ts-expect-error TODO: When marking DS as optional, this should be fixed
      spec: {
        refId: 'A',
        hidden: false,
@@ -208,6 +206,7 @@ describe('transformSaveModelSchemaV2ToScene', () => {

    const vizPanels = (scene.state.body as DashboardLayoutManager).getVizPanels();
    expect(vizPanels.length).toBe(1);
-    expect(getQueryRunnerFor(vizPanels[0])?.state.datasource).toBeUndefined();
+    expect(getQueryRunnerFor(vizPanels[0])?.state.datasource?.type).toBe('mixed');
+    expect(getQueryRunnerFor(vizPanels[0])?.state.datasource?.uid).toBe(MIXED_DATASOURCE_NAME);
  });
 });
@@ -101,7 +101,10 @@ export function transformSaveModelSchemaV2ToScene(dto: DashboardWithAccessInfo<D
  const annotationLayers = dashboard.annotations.map((annotation) => {
    return new DashboardAnnotationsDataLayer({
      key: uniqueId('annotations-'),
-      query: annotation.spec,
+      query: {
+        ...annotation.spec,
+        builtIn: annotation.spec.builtIn ? 1 : 0,
+      },
      name: annotation.spec.name,
      isEnabled: Boolean(annotation.spec.enable),
      isHidden: Boolean(annotation.spec.hide),
@@ -297,7 +300,7 @@ function getPanelDataSource(panel: PanelKind): DataSourceRef | undefined {
  panel.spec.data.spec.queries.forEach((query) => {
    if (!datasource) {
      datasource = query.spec.datasource;
-    } else if (datasource.uid !== query.spec.datasource.uid || datasource.type !== query.spec.datasource.type) {
+    } else if (datasource.uid !== query.spec.datasource?.uid || datasource.type !== query.spec.datasource?.type) {
      isMixedDatasource = true;
    }
  });
@@ -24,7 +24,9 @@ import {
 } from '@grafana/schema/dist/esm/index.gen';

 import { DashboardEditPane } from '../edit-pane/DashboardEditPane';
+import { DashboardAnnotationsDataLayer } from '../scene/DashboardAnnotationsDataLayer';
 import { DashboardControls } from '../scene/DashboardControls';
+import { DashboardDataLayerSet } from '../scene/DashboardDataLayerSet';
 import { DashboardScene, DashboardSceneState } from '../scene/DashboardScene';
 import { DashboardGridItem } from '../scene/layout-default/DashboardGridItem';
 import { DefaultGridLayoutManager } from '../scene/layout-default/DefaultGridLayoutManager';
@@ -35,6 +37,35 @@ function setupDashboardScene(state: DashboardSceneState): DashboardScene {
  return new DashboardScene(state);
 }

+jest.mock('@grafana/runtime', () => ({
+  ...jest.requireActual('@grafana/runtime'),
+  config: {
+    ...jest.requireActual('@grafana/runtime').config,
+    bootData: {
+      settings: {
+        defaultDatasource: 'loki',
+        datasources: {
+          Prometheus: {
+            name: 'Prometheus',
+            meta: { id: 'prometheus' },
+            type: 'datasource',
+          },
+          '-- Grafana --': {
+            name: 'Grafana',
+            meta: { id: 'grafana' },
+            type: 'datasource',
+          },
+          loki: {
+            name: 'Loki',
+            meta: { id: 'loki' },
+            type: 'datasource',
+          },
+        },
+      },
+    },
+  },
+}));
+
 describe('transformSceneToSaveModelSchemaV2', () => {
  let dashboardScene: DashboardScene;
  let prevFeatureToggleValue: boolean;
@@ -52,6 +83,7 @@ describe('transformSceneToSaveModelSchemaV2', () => {
    // The intention is to have a complete dashboard scene
    // with all the possible properties set
    dashboardScene = setupDashboardScene({
+      $data: new DashboardDataLayerSet({ annotationLayers }),
      title: 'Test Dashboard',
      description: 'Test Description',
      preload: true,
@@ -281,7 +313,58 @@ describe('transformSceneToSaveModelSchemaV2', () => {

  it('should transform scene to save model schema v2', () => {
    const result = transformSceneToSaveModelSchemaV2(dashboardScene);
-
    expect(result).toMatchSnapshot();
+
+    // Check that the annotation layers are correctly transformed
+    expect(result.annotations).toHaveLength(3);
+    // check annotation layer 3 with no datasource has the default datasource defined as type
+    expect(result.annotations?.[2].spec.query.kind).toBe('loki');
  });
 });
+
+const annotationLayer1 = new DashboardAnnotationsDataLayer({
+  key: 'layer1',
+  query: {
+    datasource: {
+      type: 'grafana',
+      uid: '-- Grafana --',
+    },
+    name: 'query1',
+    enable: true,
+    iconColor: 'red',
+  },
+  name: 'layer1',
+  isEnabled: true,
+  isHidden: false,
+});
+
+const annotationLayer2 = new DashboardAnnotationsDataLayer({
+  key: 'layer2',
+  query: {
+    datasource: {
+      type: 'prometheus',
+      uid: 'abcdef',
+    },
+    name: 'query2',
+    enable: true,
+    iconColor: 'blue',
+  },
+  name: 'layer2',
+  isEnabled: true,
+  isHidden: true,
+});
+
+// this could happen if a dahboard was created from code and the datasource was not defined
+const annotationLayer3NoDsDefined = new DashboardAnnotationsDataLayer({
+  key: 'layer3',
+  query: {
+    name: 'query3',
+    enable: true,
+    iconColor: 'green',
+  },
+  name: 'layer3',
+  isEnabled: true,
+  isHidden: true,
+});
+
+const annotationLayers = [annotationLayer1, annotationLayer2, annotationLayer3NoDsDefined];
@@ -1,4 +1,18 @@
-import { behaviors, SceneDataQuery, SceneDataTransformer, SceneVariableSet, VizPanel } from '@grafana/scenes';
+import { omit } from 'lodash';
+
+import { AnnotationQuery } from '@grafana/data';
+import { config } from '@grafana/runtime';
+import {
+  behaviors,
+  dataLayers,
+  SceneDataQuery,
+  SceneDataTransformer,
+  SceneVariableSet,
+  VizPanel,
+} from '@grafana/scenes';
+import { DataSourceRef } from '@grafana/schema';
+import { DASHBOARD_SCHEMA_VERSION } from 'app/features/dashboard/state/DashboardMigrator';
+
 import {
  DashboardV2Spec,
  defaultDashboardV2Spec,
@@ -11,7 +25,6 @@ import {
  DataTransformerConfig,
  PanelQuerySpec,
  DataQueryKind,
-  defaultDataSourceRef,
  GridLayoutItemKind,
  QueryOptionsSpec,
  QueryVariableKind,
@@ -22,9 +35,11 @@ import {
  ConstantVariableKind,
  GroupByVariableKind,
  AdhocVariableKind,
-} from '@grafana/schema/src/schema/dashboard/v2alpha0/dashboard.gen';
-import { DASHBOARD_SCHEMA_VERSION } from 'app/features/dashboard/state/DashboardMigrator';
-
+  AnnotationQueryKind,
+  defaultAnnotationPanelFilter,
+  defaultAnnotationQuerySpec,
+} from '../../../../../packages/grafana-schema/src/schema/dashboard/v2alpha0/dashboard.gen';
+import { DashboardDataLayerSet } from '../scene/DashboardDataLayerSet';
 import { DashboardScene, DashboardSceneState } from '../scene/DashboardScene';
 import { PanelTimeRange } from '../scene/PanelTimeRange';
 import { DashboardGridItem } from '../scene/layout-default/DashboardGridItem';
@@ -86,7 +101,7 @@ export function transformSceneToSaveModelSchemaV2(scene: DashboardScene, isSnaps
    // EOF elements

    // annotations
-    annotations: [], //FIXME
+    annotations: getAnnotations(oldDash),
    // EOF annotations

    // layout
@@ -251,7 +266,7 @@ function getVizPanelQueries(vizPanel: VizPanel): PanelQueryKind[] {
        spec: query,
      };
      const querySpec: PanelQuerySpec = {
-        datasource: datasource ?? defaultDataSourceRef(),
+        datasource: datasource ?? getDefaultDataSourceRef(),
        query: dataQuery,
        refId: query.refId,
        hidden: query.hidden,
@@ -266,8 +281,8 @@ function getVizPanelQueries(vizPanel: VizPanel): PanelQueryKind[] {
 }

 export function getDataQueryKind(query: SceneDataQuery): string {
-  // If the query has a datasource, use the datasource type, otherwise use 'default'
-  return query.datasource?.type ?? 'default';
+  // If the query has a datasource, use the datasource type, otherwise return empty kind
+  return query.datasource?.type ?? getDefaultDataSourceRef()?.type ?? '';
 }

 export function getDataQuerySpec(query: SceneDataQuery): Record<string, any> {
@@ -369,6 +384,68 @@ function getVariables(oldDash: DashboardSceneState) {
  return variables;
 }

+function getAnnotations(state: DashboardSceneState): AnnotationQueryKind[] {
+  const data = state.$data;
+  if (!(data instanceof DashboardDataLayerSet)) {
+    return [];
+  }
+  const annotations: AnnotationQueryKind[] = [];
+  for (const layer of data.state.annotationLayers) {
+    if (!(layer instanceof dataLayers.AnnotationsDataLayer)) {
+      continue;
+    }
+    const result: AnnotationQueryKind = {
+      kind: 'AnnotationQuery',
+      spec: {
+        name: layer.state.query.name,
+        datasource: layer.state.query.datasource || getDefaultDataSourceRef(),
+        query: {
+          kind: getAnnotationQueryKind(layer.state.query),
+          spec: omit(layer.state.query, 'datasource'),
+        },
+        enable: Boolean(layer.state.isEnabled),
+        hide: Boolean(layer.state.isHidden),
+        filter: layer.state.query.filter ?? defaultAnnotationPanelFilter(),
+        iconColor: layer.state.query.iconColor,
+        builtIn:
+          layer.state.query.builtIn === undefined
+            ? Boolean(layer.state.query.builtIn)
+            : defaultAnnotationQuerySpec().builtIn,
+      },
+    };
+    annotations.push(result);
+  }
+  return annotations;
+}
+
+export function getAnnotationQueryKind(annotationQuery: AnnotationQuery): string {
+  if (annotationQuery.datasource?.type) {
+    return annotationQuery.datasource.type;
+  } else {
+    const ds = getDefaultDataSourceRef();
+    if (ds) {
+      return ds.type!; // in the datasource list from bootData "id" is the type
+    }
+    // if we can't find the default datasource, return grafana as default
+    return 'grafana';
+  }
+}
+
+function getDefaultDataSourceRef(): DataSourceRef | undefined {
+  // we need to return the default datasource configured in the BootConfig
+  const defaultDatasource = config.bootData.settings.defaultDatasource;
+
+  // get default datasource type
+  const dsList = config.bootData.settings.datasources;
+  const ds = dsList[defaultDatasource];
+
+  if (ds) {
+    return { type: ds.meta.id, uid: ds.name }; // in the datasource list from bootData "id" is the type
+  }
+
+  return undefined;
+}
+
 // Function to know if the dashboard transformed is a valid DashboardV2Spec
 function isDashboardSchemaV2(dash: any): dash is DashboardV2Spec {
  if (typeof dash !== 'object' || dash === null) {