diff --git a/.gitignore b/.gitignore index 3b2d8f1d5d7..8918202646b 100644 --- a/.gitignore +++ b/.gitignore @@ -94,6 +94,9 @@ conf/custom.ini /conf/provisioning/**/*.yaml !/conf/provisioning/**/sample.yaml +# sample yaml files +!/conf/provisioning/sample/*.yaml + /conf/ldap_dev.toml /conf/ldap_freeipa.toml profile.cov diff --git a/conf/provisioning/sample/classic-dashboard.json b/conf/provisioning/sample/classic-dashboard.json new file mode 100644 index 00000000000..1d9818ac50d --- /dev/null +++ b/conf/provisioning/sample/classic-dashboard.json @@ -0,0 +1,321 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1348, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "testdata", + "uid": "PD8C576611E62080A" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 4, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "## Data center = $datacenter\n\n### server = $server\n\n#### pod = $pod\n\n---\ntext = $Text", + "mode": "markdown" + }, + "pluginVersion": "9.2.0-pre", + "targets": [ + { + "datasource": { + "type": "testdata", + "uid": "PD8C576611E62080A" + }, + "refId": "A", + "scenarioId": "random_walk" + } + ], + "title": "Markdown (with variables)", + "type": "text" + }, + { + "datasource": { + "type": "testdata", + "uid": "PD8C576611E62080A" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 5, + "options": { + "code": { + "language": "json", + "showLineNumbers": true, + "showMiniMap": false + }, + "content": "{\n \"datacenter\": $datacenter,\n \"server\": $server,\n \"pod\": $pod\n \"text\": $Text\n}\n", + "mode": "code" + }, + "pluginVersion": "9.2.0-pre", + "targets": [ + { + "datasource": { + "type": "testdata", + "uid": "PD8C576611E62080A" + }, + "refId": "A", + "scenarioId": "random_walk" + } + ], + "title": "JSON (with variables)", + "type": "text" + }, + { + "datasource": { + "type": "testdata", + "uid": "PD8C576611E62080A" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 6, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "

Data center

\n

$datacenter

\n\n

server

\n

$server

\n\n

pod

\n

$pod

\n\n

Text

\n

$Text

", + "mode": "html" + }, + "pluginVersion": "9.2.0-pre", + "targets": [ + { + "datasource": { + "type": "testdata", + "uid": "PD8C576611E62080A" + }, + "refId": "A", + "scenarioId": "random_walk" + } + ], + "title": "HTML (with variables)", + "type": "text" + }, + { + "datasource": { + "type": "testdata", + "uid": "PD8C576611E62080A" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 7, + "options": { + "code": { + "language": "markdown", + "showLineNumbers": true, + "showMiniMap": true + }, + "content": "## Data center\n$datacenter\n\n### server\n$server\n\n#### pod = \n$pod\n", + "mode": "code" + }, + "pluginVersion": "9.2.0-pre", + "targets": [ + { + "datasource": { + "type": "testdata", + "uid": "PD8C576611E62080A" + }, + "refId": "A", + "scenarioId": "random_walk" + } + ], + "title": "Markdown (code w/ with variables)", + "type": "text" + } + ], + "refresh": false, + "schemaVersion": 37, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "testdata", + "uid": "PD8C576611E62080A" + }, + "definition": "*", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "query": "*", + "refId": "gdev-testdata-datacenter-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "testdata", + "uid": "PD8C576611E62080A" + }, + "definition": "$datacenter.*", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "server", + "options": [], + "query": { + "query": "$datacenter.*", + "refId": "gdev-testdata-server-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": [ + "AAA", + "ACB" + ], + "value": [ + "AAA", + "ACB" + ] + }, + "datasource": { + "type": "testdata", + "uid": "PD8C576611E62080A" + }, + "definition": "$datacenter.$server.*", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "pod", + "options": [], + "query": { + "query": "$datacenter.$server.*", + "refId": "gdev-testdata-pod-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "temp", + "value": "temp" + }, + "hide": 0, + "name": "Text", + "options": [ + { + "selected": true, + "text": "temp", + "value": "temp" + } + ], + "query": "temp", + "skipUrlSync": false, + "type": "textbox" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Text options", + "uid": "WZ7AhQiVz", + "version": 1, + "weekStart": "" +} diff --git a/conf/provisioning/sample/dashboard-a.json b/conf/provisioning/sample/dashboard-a.json new file mode 100644 index 00000000000..e6a04bb3e35 --- /dev/null +++ b/conf/provisioning/sample/dashboard-a.json @@ -0,0 +1,36 @@ +{ + "kind": "Dashboard", + "apiVersion": "dashboard.grafana.app/v0alpha1", + "metadata": { + "name": "ignored?", + "namespace": "default", + "uid": "ignored", + "resourceVersion": "also ignored", + "creationTimestamp": "2024-11-15T19:54:07Z", + "annotations": { + "hello": "world:" + }, + "labels": { + "region": "west" + } + }, + "spec": { + "title": "Green dashboard from provisioning", + "panels": [ + { + "gridPos": { + "h": 13, + "w": 24, + "x": 0, + "y": 0 + }, + "options": { + "content": "

This dashboard was loaded from provisioning 🎉🎉🎉🎉🎉🎉

", + "mode": "html" + }, + "transparent": true, + "type": "text" + } + ] + } +} \ No newline at end of file diff --git a/conf/provisioning/sample/provisioning-s3.yaml b/conf/provisioning/sample/provisioning-s3.yaml new file mode 100644 index 00000000000..0cc5a739e91 --- /dev/null +++ b/conf/provisioning/sample/provisioning-s3.yaml @@ -0,0 +1,9 @@ +apiVersion: provisioning.grafana.app/v0alpha1 +kind: Repository +spec: + title: Provisioning... inception + description: provision a repository from provisioning + type: s3 + s3: + region: us-west-1 + bucket: my-bucket diff --git a/conf/provisioning/sample/sample-playlist.json b/conf/provisioning/sample/sample-playlist.json new file mode 100644 index 00000000000..1b72ccdad59 --- /dev/null +++ b/conf/provisioning/sample/sample-playlist.json @@ -0,0 +1,25 @@ +{ + "kind": "Playlist", + "apiVersion": "playlist.grafana.app/v0alpha1", + "metadata": { + "name": "adcmdrw", + "namespace": "default", + "uid": "8a2b984d-4663-4182-861b-edec2f987dff", + "resourceVersion": "1731336353828005", + "creationTimestamp": "2024-11-11T14:45:53Z" + }, + "spec": { + "title": "Playlist from provisioning", + "interval": "5m", + "items": [ + { + "type": "dashboard_by_uid", + "value": "xCmMwXdVz" + }, + { + "type": "dashboard_by_tag", + "value": "panel-tests" + } + ] + } +} \ No newline at end of file diff --git a/pkg/apis/provisioning/v0alpha1/types.go b/pkg/apis/provisioning/v0alpha1/types.go index 8b764f6b744..1cd3f6f915b 100644 --- a/pkg/apis/provisioning/v0alpha1/types.go +++ b/pkg/apis/provisioning/v0alpha1/types.go @@ -126,6 +126,9 @@ type HelloWorld struct { type ResourceWrapper struct { metav1.TypeMeta `json:",inline"` + // Path to the remote file + Path string `json:"path,omitempty"` + // The commit hash (if exists) Commit string `json:"commit,omitempty"` diff --git a/pkg/apis/provisioning/v0alpha1/zz_generated.openapi.go b/pkg/apis/provisioning/v0alpha1/zz_generated.openapi.go index c741cb068da..104bd4d0c9a 100644 --- a/pkg/apis/provisioning/v0alpha1/zz_generated.openapi.go +++ b/pkg/apis/provisioning/v0alpha1/zz_generated.openapi.go @@ -329,6 +329,13 @@ func schema_pkg_apis_provisioning_v0alpha1_ResourceWrapper(ref common.ReferenceC Format: "", }, }, + "path": { + SchemaProps: spec.SchemaProps{ + Description: "Path to the remote file", + Type: []string{"string"}, + Format: "", + }, + }, "commit": { SchemaProps: spec.SchemaProps{ Description: "The commit hash (if exists)", diff --git a/pkg/registry/apis/provisioning/github.go b/pkg/registry/apis/provisioning/github.go index 3b94eae4865..f4b6d68f53e 100644 --- a/pkg/registry/apis/provisioning/github.go +++ b/pkg/registry/apis/provisioning/github.go @@ -2,7 +2,6 @@ package provisioning import ( "context" - "encoding/json" "fmt" "net/http" "net/url" @@ -14,7 +13,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation/field" - common "github.com/grafana/grafana/pkg/apimachinery/apis/common/v0alpha1" provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" ) @@ -82,7 +80,7 @@ func extractOwnerAndRepo(repoURL string) (string, string, error) { } // ReadResource implements provisioning.Repository. -func (r *githubRepository) ReadResource(ctx context.Context, filePath string, commit string) (*provisioning.ResourceWrapper, error) { +func (r *githubRepository) Read(ctx context.Context, filePath string, commit string) ([]byte, error) { tokenSrc := oauth2.StaticTokenSource( &oauth2.Token{AccessToken: r.config.Spec.GitHub.Token}, ) @@ -106,23 +104,7 @@ func (r *githubRepository) ReadResource(ctx context.Context, filePath string, co if err != nil { return nil, err } - - var dashboardJSON map[string]interface{} - if err := json.Unmarshal([]byte(data), &dashboardJSON); err != nil { - // TODO: return bad request status code - return nil, fmt.Errorf("failed to unmarshal content: %w", err) - } - - // TODO: validate the object is a valid dashboard - // Return the wrapped response - // Note we can not return it directly (using responder) because that will make sure the - // top level apiVersion is provisioning, not the remote resource - return &provisioning.ResourceWrapper{ - Commit: commit, - Resource: common.Unstructured{ - Object: dashboardJSON, - }, - }, nil + return []byte(data), nil } // Webhook implements provisioning.Repository. diff --git a/pkg/registry/apis/provisioning/local.go b/pkg/registry/apis/provisioning/local.go index 659557caeac..7af61064ada 100644 --- a/pkg/registry/apis/provisioning/local.go +++ b/pkg/registry/apis/provisioning/local.go @@ -3,6 +3,7 @@ package provisioning import ( "context" "net/http" + "os" "path/filepath" "strings" @@ -112,13 +113,21 @@ func (r *localRepository) Test(ctx context.Context) error { } // ReadResource implements provisioning.Repository. -func (r *localRepository) ReadResource(ctx context.Context, path string, commit string) (*provisioning.ResourceWrapper, error) { - return nil, &errors.StatusError{ - ErrStatus: v1.Status{ - Message: "read resource is not yet implemented", - Code: http.StatusNotImplemented, - }, +func (r *localRepository) Read(ctx context.Context, path string, commit string) ([]byte, error) { + if commit != "" { + return nil, errors.NewBadRequest("local repository does not support commits") } + if r.path == "" { + return nil, &errors.StatusError{ + ErrStatus: v1.Status{ + Message: "the service is missing a root path", + Code: http.StatusFailedDependency, + }, + } + } + + //nolint:gosec + return os.ReadFile(filepath.Join(r.path, path)) } // Webhook implements provisioning.Repository. diff --git a/pkg/registry/apis/provisioning/read.go b/pkg/registry/apis/provisioning/read.go index 69b4b35fb8c..0ccb849f783 100644 --- a/pkg/registry/apis/provisioning/read.go +++ b/pkg/registry/apis/provisioning/read.go @@ -1,7 +1,9 @@ package provisioning import ( + "bytes" "context" + "fmt" "net/http" "strings" @@ -9,6 +11,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/registry/rest" + "github.com/grafana/grafana/pkg/apimachinery/apis/common/v0alpha1" provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" ) @@ -55,19 +58,38 @@ func (s *readConnector) Connect(ctx context.Context, name string, opts runtime.O return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { idx := strings.Index(r.URL.Path, "/"+name+"/read") - filePath := strings.TrimLeft(r.URL.Path[idx+len(name+"/read")+2:], "/") + filePath := strings.TrimLeft(r.URL.Path[idx+len(name+"/read")+1:], "/") if filePath == "" { responder.Error(errors.NewBadRequest("missing path")) return } - commit := r.URL.Query().Get("commit") - rsp, err := repo.ReadResource(ctx, filePath, commit) + commit := r.URL.Query().Get("commit") + data, err := repo.Read(ctx, filePath, commit) if err != nil { responder.Error(err) - } else { - responder.Object(200, rsp) + return } + + obj, gvk, err := LoadYAMLOrJSON(bytes.NewBuffer(data)) + if err != nil { + obj, gvk, err = FallbackResourceLoader(data) + if err != nil { + responder.Error(err) + return + } + } + + // TODO? dry run??? and add validation errors/warnings? + fmt.Printf("TODO, dryrun... %+v\n", gvk) + + responder.Object(200, &provisioning.ResourceWrapper{ + Path: filePath, + Commit: commit, + Resource: v0alpha1.Unstructured{ + Object: obj.Object, + }, + }) }), nil } diff --git a/pkg/registry/apis/provisioning/s3.go b/pkg/registry/apis/provisioning/s3.go index edba876eced..9e4d1a4da19 100644 --- a/pkg/registry/apis/provisioning/s3.go +++ b/pkg/registry/apis/provisioning/s3.go @@ -50,7 +50,7 @@ func (r *s3Repository) Test(ctx context.Context) error { } // ReadResource implements provisioning.Repository. -func (r *s3Repository) ReadResource(ctx context.Context, path string, commit string) (*provisioning.ResourceWrapper, error) { +func (r *s3Repository) Read(ctx context.Context, path string, commit string) ([]byte, error) { return nil, &errors.StatusError{ ErrStatus: v1.Status{ Message: "read resource is not yet implemented", diff --git a/pkg/registry/apis/provisioning/types.go b/pkg/registry/apis/provisioning/types.go index 1b579c3064b..97bf13162ae 100644 --- a/pkg/registry/apis/provisioning/types.go +++ b/pkg/registry/apis/provisioning/types.go @@ -27,8 +27,8 @@ type Repository interface { // Called to check if all connection information actually works Test(ctx context.Context) error - // Read a resource from settings - ReadResource(ctx context.Context, path string, commit string) (*provisioning.ResourceWrapper, error) + // Read a file from the resource + Read(ctx context.Context, path string, commit string) ([]byte, error) // For repositories that support webhooks Webhook() http.HandlerFunc @@ -67,7 +67,7 @@ func (r *unknownRepository) Test(ctx context.Context) error { } // ReadResource implements provisioning.Repository. -func (r *unknownRepository) ReadResource(ctx context.Context, path string, commit string) (*provisioning.ResourceWrapper, error) { +func (r *unknownRepository) Read(ctx context.Context, path string, commit string) ([]byte, error) { return nil, &errors.StatusError{ ErrStatus: v1.Status{ Message: "read resource is not yet implemented", diff --git a/pkg/registry/apis/provisioning/utils.go b/pkg/registry/apis/provisioning/utils.go new file mode 100644 index 00000000000..a46c7d5f8bf --- /dev/null +++ b/pkg/registry/apis/provisioning/utils.go @@ -0,0 +1,82 @@ +package provisioning + +import ( + "encoding/json" + "fmt" + "io" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer/yaml" + yamlutil "k8s.io/apimachinery/pkg/util/yaml" +) + +func FallbackResourceLoader(data []byte) (*unstructured.Unstructured, *schema.GroupVersionKind, error) { + // Try parsing as JSON + if data[0] == '{' { + var value map[string]any + err := json.Unmarshal(data, &value) + if err != nil { + return nil, nil, err + } + + // regular version headers exist + if value["apiVersion"] != nil && value["Kind"] != nil { + gv, err := schema.ParseGroupVersion(value["apiVersion"].(string)) + if err != nil { + return nil, nil, fmt.Errorf("invalid apiVersion") + } + gvk := gv.WithKind(value["Kind"].(string)) + return &unstructured.Unstructured{Object: value}, &gvk, nil + } + + // If this is a dashboard, convert it + if value["panels"] != nil && + value["schemaVersion"] != nil && + value["tags"] != nil { + gvk := &schema.GroupVersionKind{ + Group: "dashboards.grafana.app", + Version: "v0alpha1", + Kind: "Dashboard"} + return &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": gvk.GroupVersion().String(), + "metadata": map[string]any{ + "name": value["uid"], + }, + "spec": value, + }, + }, gvk, nil + } + } + + return nil, nil, fmt.Errorf("unable to convert") +} + +func LoadYAMLOrJSON(input io.Reader) (*unstructured.Unstructured, *schema.GroupVersionKind, error) { + decoder := yamlutil.NewYAMLOrJSONDecoder(input, 1024) + var rawObj runtime.RawExtension + err := decoder.Decode(&rawObj) + if err != nil { + return nil, nil, err + } + + obj, gvk, err := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme). + Decode(rawObj.Raw, nil, nil) + if err != nil { + return nil, gvk, err + } + + // The decoder should put it directly into an unstructured object + val, ok := obj.(*unstructured.Unstructured) + if ok { + return val, gvk, err + } + + unstructuredMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) + if err != nil { + return nil, gvk, err + } + return &unstructured.Unstructured{Object: unstructuredMap}, gvk, err +} diff --git a/pkg/registry/apis/provisioning/utils_test.go b/pkg/registry/apis/provisioning/utils_test.go new file mode 100644 index 00000000000..bec1da9d2c8 --- /dev/null +++ b/pkg/registry/apis/provisioning/utils_test.go @@ -0,0 +1,68 @@ +package provisioning + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +func TestUtils(t *testing.T) { + t.Run("load playlist json", func(t *testing.T) { + obj, gvk, err := LoadYAMLOrJSON(bytes.NewReader([]byte(`{ + "kind": "Playlist", + "apiVersion": "playlist.grafana.app/v0alpha1", + "metadata": { + "name": "hello" + }, + "spec": { + "title": "Playlist from provisioning" + } + }`))) + + require.NoError(t, err) + require.Equal(t, &schema.GroupVersionKind{ + Group: "playlist.grafana.app", + Version: "v0alpha1", + Kind: "Playlist", + }, gvk) + require.NotNil(t, obj) + }) + + t.Run("load playlist yaml", func(t *testing.T) { + obj, gvk, err := LoadYAMLOrJSON(bytes.NewReader([]byte(` +apiVersion: playlist.grafana.app/v0alpha1 +kind: Playlist +metadata: + name: hello +spec: + title: a title +`))) + + require.NoError(t, err) + require.Equal(t, &schema.GroupVersionKind{ + Group: "playlist.grafana.app", + Version: "v0alpha1", + Kind: "Playlist", + }, gvk) + require.NotNil(t, obj) + }) + + t.Run("load dashboard json", func(t *testing.T) { + // Support dashboard conversion + obj, gvk, err := FallbackResourceLoader([]byte(`{ + "schemaVersion": 7, + "panels": [], + "tags": [] + }`)) + + require.NoError(t, err) + require.Equal(t, &schema.GroupVersionKind{ + Group: "dashboards.grafana.app", + Version: "v0alpha1", + Kind: "Dashboard", + }, gvk) + require.NotNil(t, obj) + }) +}