POC/Provisioning: Load local files (yaml or json) (#96653)

.gitignore

@@ -94,6 +94,9 @@ conf/custom.ini
 /conf/provisioning/**/*.yaml
 !/conf/provisioning/**/sample.yaml
 
+# sample yaml files
+!/conf/provisioning/sample/*.yaml
+
 /conf/ldap_dev.toml
 /conf/ldap_freeipa.toml
 profile.cov

conf/provisioning/sample/classic-dashboard.json

@@ -0,0 +1,321 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "datasource",
+          "uid": "grafana"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 1348,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "testdata",
+        "uid": "PD8C576611E62080A"
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 4,
+      "options": {
+        "code": {
+          "language": "plaintext",
+          "showLineNumbers": false,
+          "showMiniMap": false
+        },
+        "content": "## Data center = $datacenter\n\n### server = $server\n\n#### pod = $pod\n\n---\ntext = $Text",
+        "mode": "markdown"
+      },
+      "pluginVersion": "9.2.0-pre",
+      "targets": [
+        {
+          "datasource": {
+            "type": "testdata",
+            "uid": "PD8C576611E62080A"
+          },
+          "refId": "A",
+          "scenarioId": "random_walk"
+        }
+      ],
+      "title": "Markdown (with variables)",
+      "type": "text"
+    },
+    {
+      "datasource": {
+        "type": "testdata",
+        "uid": "PD8C576611E62080A"
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 5,
+      "options": {
+        "code": {
+          "language": "json",
+          "showLineNumbers": true,
+          "showMiniMap": false
+        },
+        "content": "{\n  \"datacenter\": $datacenter,\n  \"server\": $server,\n  \"pod\": $pod\n  \"text\": $Text\n}\n",
+        "mode": "code"
+      },
+      "pluginVersion": "9.2.0-pre",
+      "targets": [
+        {
+          "datasource": {
+            "type": "testdata",
+            "uid": "PD8C576611E62080A"
+          },
+          "refId": "A",
+          "scenarioId": "random_walk"
+        }
+      ],
+      "title": "JSON (with variables)",
+      "type": "text"
+    },
+    {
+      "datasource": {
+        "type": "testdata",
+        "uid": "PD8C576611E62080A"
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 9
+      },
+      "id": 6,
+      "options": {
+        "code": {
+          "language": "plaintext",
+          "showLineNumbers": false,
+          "showMiniMap": false
+        },
+        "content": "<h3>Data center</h3>\n<p>$datacenter</p>\n\n<h3>server</h3>\n<p>$server</p>\n\n<h3>pod</h3>\n<p>$pod</p>\n\n<h3>Text</h3>\n<p>$Text</p>",
+        "mode": "html"
+      },
+      "pluginVersion": "9.2.0-pre",
+      "targets": [
+        {
+          "datasource": {
+            "type": "testdata",
+            "uid": "PD8C576611E62080A"
+          },
+          "refId": "A",
+          "scenarioId": "random_walk"
+        }
+      ],
+      "title": "HTML (with variables)",
+      "type": "text"
+    },
+    {
+      "datasource": {
+        "type": "testdata",
+        "uid": "PD8C576611E62080A"
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 12,
+        "y": 9
+      },
+      "id": 7,
+      "options": {
+        "code": {
+          "language": "markdown",
+          "showLineNumbers": true,
+          "showMiniMap": true
+        },
+        "content": "## Data center\n$datacenter\n\n### server\n$server\n\n#### pod = \n$pod\n",
+        "mode": "code"
+      },
+      "pluginVersion": "9.2.0-pre",
+      "targets": [
+        {
+          "datasource": {
+            "type": "testdata",
+            "uid": "PD8C576611E62080A"
+          },
+          "refId": "A",
+          "scenarioId": "random_walk"
+        }
+      ],
+      "title": "Markdown (code w/ with variables)",
+      "type": "text"
+    }
+  ],
+  "refresh": false,
+  "schemaVersion": 37,
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false,
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
+        },
+        "datasource": {
+          "type": "testdata",
+          "uid": "PD8C576611E62080A"
+        },
+        "definition": "*",
+        "hide": 0,
+        "includeAll": true,
+        "multi": true,
+        "name": "datacenter",
+        "options": [],
+        "query": {
+          "query": "*",
+          "refId": "gdev-testdata-datacenter-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "current": {
+          "selected": false,
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
+        },
+        "datasource": {
+          "type": "testdata",
+          "uid": "PD8C576611E62080A"
+        },
+        "definition": "$datacenter.*",
+        "hide": 0,
+        "includeAll": true,
+        "multi": true,
+        "name": "server",
+        "options": [],
+        "query": {
+          "query": "$datacenter.*",
+          "refId": "gdev-testdata-server-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "current": {
+          "selected": false,
+          "text": [
+            "AAA",
+            "ACB"
+          ],
+          "value": [
+            "AAA",
+            "ACB"
+          ]
+        },
+        "datasource": {
+          "type": "testdata",
+          "uid": "PD8C576611E62080A"
+        },
+        "definition": "$datacenter.$server.*",
+        "hide": 0,
+        "includeAll": true,
+        "multi": true,
+        "name": "pod",
+        "options": [],
+        "query": {
+          "query": "$datacenter.$server.*",
+          "refId": "gdev-testdata-pod-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "current": {
+          "selected": false,
+          "text": "temp",
+          "value": "temp"
+        },
+        "hide": 0,
+        "name": "Text",
+        "options": [
+          {
+            "selected": true,
+            "text": "temp",
+            "value": "temp"
+          }
+        ],
+        "query": "temp",
+        "skipUrlSync": false,
+        "type": "textbox"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ]
+  },
+  "timezone": "",
+  "title": "Text options",
+  "uid": "WZ7AhQiVz",
+  "version": 1,
+  "weekStart": ""
+}

conf/provisioning/sample/dashboard-a.json

@@ -0,0 +1,36 @@
+{
+  "kind": "Dashboard",
+  "apiVersion": "dashboard.grafana.app/v0alpha1",
+  "metadata": {
+    "name": "ignored?",
+    "namespace": "default",
+    "uid": "ignored",
+    "resourceVersion": "also ignored",
+    "creationTimestamp": "2024-11-15T19:54:07Z",
+    "annotations": {
+      "hello": "world:"
+    },
+    "labels": {
+      "region": "west"
+    }
+  },
+  "spec": {
+    "title": "Green dashboard from provisioning",
+    "panels": [
+      {
+        "gridPos": {
+          "h": 13,
+          "w": 24,
+          "x": 0,
+          "y": 0
+        },
+        "options": {
+          "content": "<div style=\"background:#1cac78; color:#000; padding:50px; height:1000px\"><h1>This dashboard was loaded from provisioning 🎉🎉🎉🎉🎉🎉</h1></div>",
+          "mode": "html"
+        },
+        "transparent": true,
+        "type": "text"
+      }
+    ]
+  }
+}

conf/provisioning/sample/provisioning-s3.yaml

@@ -0,0 +1,9 @@
+apiVersion: provisioning.grafana.app/v0alpha1
+kind: Repository
+spec:
+  title: Provisioning... inception
+  description: provision a repository from provisioning
+  type: s3 
+  s3:
+    region: us-west-1
+    bucket: my-bucket

conf/provisioning/sample/sample-playlist.json

@@ -0,0 +1,25 @@
+{
+  "kind": "Playlist",
+  "apiVersion": "playlist.grafana.app/v0alpha1",
+  "metadata": {
+    "name": "adcmdrw",
+    "namespace": "default",
+    "uid": "8a2b984d-4663-4182-861b-edec2f987dff",
+    "resourceVersion": "1731336353828005",
+    "creationTimestamp": "2024-11-11T14:45:53Z"
+  },
+  "spec": {
+    "title": "Playlist from provisioning",
+    "interval": "5m",
+    "items": [
+      {
+        "type": "dashboard_by_uid",
+        "value": "xCmMwXdVz"
+      },
+      {
+        "type": "dashboard_by_tag",
+        "value": "panel-tests"
+      }
+    ]
+  }
+}

pkg/apis/provisioning/v0alpha1/types.go

@@ -126,6 +126,9 @@ type HelloWorld struct {
 type ResourceWrapper struct {
 	metav1.TypeMeta `json:",inline"`
 
+	// Path to the remote file
+	Path string `json:"path,omitempty"`
+
 	// The commit hash (if exists)
 	Commit string `json:"commit,omitempty"`
 

pkg/apis/provisioning/v0alpha1/zz_generated.openapi.go

@@ -329,6 +329,13 @@ func schema_pkg_apis_provisioning_v0alpha1_ResourceWrapper(ref common.ReferenceC
 							Format:      "",
 						},
 					},
+					"path": {
+						SchemaProps: spec.SchemaProps{
+							Description: "Path to the remote file",
+							Type:        []string{"string"},
+							Format:      "",
+						},
+					},
 					"commit": {
 						SchemaProps: spec.SchemaProps{
 							Description: "The commit hash (if exists)",

pkg/registry/apis/provisioning/github.go

@@ -2,7 +2,6 @@ package provisioning
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -14,7 +13,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 
-	common "github.com/grafana/grafana/pkg/apimachinery/apis/common/v0alpha1"
 	provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
 )
 
@@ -82,7 +80,7 @@ func extractOwnerAndRepo(repoURL string) (string, string, error) {
 }
 
 // ReadResource implements provisioning.Repository.
-func (r *githubRepository) ReadResource(ctx context.Context, filePath string, commit string) (*provisioning.ResourceWrapper, error) {
+func (r *githubRepository) Read(ctx context.Context, filePath string, commit string) ([]byte, error) {
 	tokenSrc := oauth2.StaticTokenSource(
 		&oauth2.Token{AccessToken: r.config.Spec.GitHub.Token},
 	)
@@ -106,23 +104,7 @@ func (r *githubRepository) ReadResource(ctx context.Context, filePath string, co
 	if err != nil {
 		return nil, err
 	}
-
-	var dashboardJSON map[string]interface{}
-	if err := json.Unmarshal([]byte(data), &dashboardJSON); err != nil {
-		// TODO: return bad request status code
-		return nil, fmt.Errorf("failed to unmarshal content: %w", err)
-	}
-
-	// TODO: validate the object is a valid dashboard
-	// Return the wrapped response
-	// Note we can not return it directly (using responder) because that will make sure the
-	// top level apiVersion is provisioning, not the remote resource
-	return &provisioning.ResourceWrapper{
-		Commit: commit,
-		Resource: common.Unstructured{
-			Object: dashboardJSON,
-		},
-	}, nil
+	return []byte(data), nil
 }
 
 // Webhook implements provisioning.Repository.

pkg/registry/apis/provisioning/local.go

@@ -3,6 +3,7 @@ package provisioning
 import (
 	"context"
 	"net/http"
+	"os"
 	"path/filepath"
 	"strings"
 
@@ -112,13 +113,21 @@ func (r *localRepository) Test(ctx context.Context) error {
 }
 
 // ReadResource implements provisioning.Repository.
-func (r *localRepository) ReadResource(ctx context.Context, path string, commit string) (*provisioning.ResourceWrapper, error) {
-	return nil, &errors.StatusError{
-		ErrStatus: v1.Status{
-			Message: "read resource is not yet implemented",
-			Code:    http.StatusNotImplemented,
-		},
+func (r *localRepository) Read(ctx context.Context, path string, commit string) ([]byte, error) {
+	if commit != "" {
+		return nil, errors.NewBadRequest("local repository does not support commits")
 	}
+	if r.path == "" {
+		return nil, &errors.StatusError{
+			ErrStatus: v1.Status{
+				Message: "the service is missing a root path",
+				Code:    http.StatusFailedDependency,
+			},
+		}
+	}
+
+	//nolint:gosec
+	return os.ReadFile(filepath.Join(r.path, path))
 }
 
 // Webhook implements provisioning.Repository.

pkg/registry/apis/provisioning/read.go

@@ -1,7 +1,9 @@
 package provisioning
 
 import (
+	"bytes"
 	"context"
+	"fmt"
 	"net/http"
 	"strings"
 
@@ -9,6 +11,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apiserver/pkg/registry/rest"
 
+	"github.com/grafana/grafana/pkg/apimachinery/apis/common/v0alpha1"
 	provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
 )
 
@@ -55,19 +58,38 @@ func (s *readConnector) Connect(ctx context.Context, name string, opts runtime.O
 
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		idx := strings.Index(r.URL.Path, "/"+name+"/read")
-		filePath := strings.TrimLeft(r.URL.Path[idx+len(name+"/read")+2:], "/")
+		filePath := strings.TrimLeft(r.URL.Path[idx+len(name+"/read")+1:], "/")
 		if filePath == "" {
 			responder.Error(errors.NewBadRequest("missing path"))
 			return
 		}
-		commit := r.URL.Query().Get("commit")
 
-		rsp, err := repo.ReadResource(ctx, filePath, commit)
+		commit := r.URL.Query().Get("commit")
+		data, err := repo.Read(ctx, filePath, commit)
 		if err != nil {
 			responder.Error(err)
-		} else {
-			responder.Object(200, rsp)
+			return
 		}
+
+		obj, gvk, err := LoadYAMLOrJSON(bytes.NewBuffer(data))
+		if err != nil {
+			obj, gvk, err = FallbackResourceLoader(data)
+			if err != nil {
+				responder.Error(err)
+				return
+			}
+		}
+
+		// TODO? dry run??? and add validation errors/warnings?
+		fmt.Printf("TODO, dryrun... %+v\n", gvk)
+
+		responder.Object(200, &provisioning.ResourceWrapper{
+			Path:   filePath,
+			Commit: commit,
+			Resource: v0alpha1.Unstructured{
+				Object: obj.Object,
+			},
+		})
 	}), nil
 }
 

pkg/registry/apis/provisioning/s3.go

@@ -50,7 +50,7 @@ func (r *s3Repository) Test(ctx context.Context) error {
 }
 
 // ReadResource implements provisioning.Repository.
-func (r *s3Repository) ReadResource(ctx context.Context, path string, commit string) (*provisioning.ResourceWrapper, error) {
+func (r *s3Repository) Read(ctx context.Context, path string, commit string) ([]byte, error) {
 	return nil, &errors.StatusError{
 		ErrStatus: v1.Status{
 			Message: "read resource is not yet implemented",

pkg/registry/apis/provisioning/types.go

@@ -27,8 +27,8 @@ type Repository interface {
 	// Called to check if all connection information actually works
 	Test(ctx context.Context) error
 
-	// Read a resource from settings
-	ReadResource(ctx context.Context, path string, commit string) (*provisioning.ResourceWrapper, error)
+	// Read a file from the resource
+	Read(ctx context.Context, path string, commit string) ([]byte, error)
 
 	// For repositories that support webhooks
 	Webhook() http.HandlerFunc
@@ -67,7 +67,7 @@ func (r *unknownRepository) Test(ctx context.Context) error {
 }
 
 // ReadResource implements provisioning.Repository.
-func (r *unknownRepository) ReadResource(ctx context.Context, path string, commit string) (*provisioning.ResourceWrapper, error) {
+func (r *unknownRepository) Read(ctx context.Context, path string, commit string) ([]byte, error) {
 	return nil, &errors.StatusError{
 		ErrStatus: v1.Status{
 			Message: "read resource is not yet implemented",

pkg/registry/apis/provisioning/utils.go

@@ -0,0 +1,82 @@
+package provisioning
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/runtime/serializer/yaml"
+	yamlutil "k8s.io/apimachinery/pkg/util/yaml"
+)
+
+func FallbackResourceLoader(data []byte) (*unstructured.Unstructured, *schema.GroupVersionKind, error) {
+	// Try parsing as JSON
+	if data[0] == '{' {
+		var value map[string]any
+		err := json.Unmarshal(data, &value)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		// regular version headers exist
+		if value["apiVersion"] != nil && value["Kind"] != nil {
+			gv, err := schema.ParseGroupVersion(value["apiVersion"].(string))
+			if err != nil {
+				return nil, nil, fmt.Errorf("invalid apiVersion")
+			}
+			gvk := gv.WithKind(value["Kind"].(string))
+			return &unstructured.Unstructured{Object: value}, &gvk, nil
+		}
+
+		// If this is a dashboard, convert it
+		if value["panels"] != nil &&
+			value["schemaVersion"] != nil &&
+			value["tags"] != nil {
+			gvk := &schema.GroupVersionKind{
+				Group:   "dashboards.grafana.app",
+				Version: "v0alpha1",
+				Kind:    "Dashboard"}
+			return &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": gvk.GroupVersion().String(),
+					"metadata": map[string]any{
+						"name": value["uid"],
+					},
+					"spec": value,
+				},
+			}, gvk, nil
+		}
+	}
+
+	return nil, nil, fmt.Errorf("unable to convert")
+}
+
+func LoadYAMLOrJSON(input io.Reader) (*unstructured.Unstructured, *schema.GroupVersionKind, error) {
+	decoder := yamlutil.NewYAMLOrJSONDecoder(input, 1024)
+	var rawObj runtime.RawExtension
+	err := decoder.Decode(&rawObj)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	obj, gvk, err := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme).
+		Decode(rawObj.Raw, nil, nil)
+	if err != nil {
+		return nil, gvk, err
+	}
+
+	// The decoder should put it directly into an unstructured object
+	val, ok := obj.(*unstructured.Unstructured)
+	if ok {
+		return val, gvk, err
+	}
+
+	unstructuredMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj)
+	if err != nil {
+		return nil, gvk, err
+	}
+	return &unstructured.Unstructured{Object: unstructuredMap}, gvk, err
+}

pkg/registry/apis/provisioning/utils_test.go

@@ -0,0 +1,68 @@
+package provisioning
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+func TestUtils(t *testing.T) {
+	t.Run("load playlist json", func(t *testing.T) {
+		obj, gvk, err := LoadYAMLOrJSON(bytes.NewReader([]byte(`{
+			"kind": "Playlist",
+			"apiVersion": "playlist.grafana.app/v0alpha1",
+			"metadata": {
+				"name": "hello"
+			},
+			"spec": {
+				"title": "Playlist from provisioning"
+			}
+		}`)))
+
+		require.NoError(t, err)
+		require.Equal(t, &schema.GroupVersionKind{
+			Group:   "playlist.grafana.app",
+			Version: "v0alpha1",
+			Kind:    "Playlist",
+		}, gvk)
+		require.NotNil(t, obj)
+	})
+
+	t.Run("load playlist yaml", func(t *testing.T) {
+		obj, gvk, err := LoadYAMLOrJSON(bytes.NewReader([]byte(`
+apiVersion: playlist.grafana.app/v0alpha1
+kind: Playlist
+metadata:
+	name: hello
+spec:
+	title: a title
+`)))
+
+		require.NoError(t, err)
+		require.Equal(t, &schema.GroupVersionKind{
+			Group:   "playlist.grafana.app",
+			Version: "v0alpha1",
+			Kind:    "Playlist",
+		}, gvk)
+		require.NotNil(t, obj)
+	})
+
+	t.Run("load dashboard json", func(t *testing.T) {
+		// Support dashboard conversion
+		obj, gvk, err := FallbackResourceLoader([]byte(`{
+			"schemaVersion": 7,
+			"panels": [],
+			"tags": []
+		}`))
+
+		require.NoError(t, err)
+		require.Equal(t, &schema.GroupVersionKind{
+			Group:   "dashboards.grafana.app",
+			Version: "v0alpha1",
+			Kind:    "Dashboard",
+		}, gvk)
+		require.NotNil(t, obj)
+	})
+}