From 2554c482eaed570432447abe6dcefcb5aa3f590a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Agn=C3=A8s=20Toulet?= <35176601+AgnesToulet@users.noreply.github.com> Date: Tue, 13 Jun 2023 14:01:29 +0200 Subject: [PATCH] Kinds: publish kinds to kind registry (#67515) * Start local schema registry * Try to use latest version * Revert "Try to use latest version" This reverts commit 682385c0e4cef8f623d88fe9a2e83cdb412036a7. * update schema registry jenny to validate new lineages * save kind instead of lineage * handle plugins * get published schemas from GH + fix plugins * handle kind not yet published * Add script to use in workflow + handle maturity * first pass on publish-kinds GH workflow * fix script path * remove unused script * use make gen-cue instead of script * trigger publish-kinds on every commit (for test) * temporary update to use specific thema commit * wrapping errors * remove GH token & refactor for rate limit * Update publish-kinds.yml * Update publish-kinds.yml * revert go.mod and go.sum updates * use Thema specific commit * Kind registry v2 * fix script path * fix second script path * update go.mod * update schema registry source * test checks * add GITHUB_TOKEN * revert test checks * actually write next files when publishing * Add kind set arg * Add comments * clean up workflows * update Thema * Update .betterer.results * few fixes after lineage flattening * Update publish-kinds-next.yml * add codeowners for new files * update thema * apply review feedback * update go version in workflows * clean up workflows * Add step to generate token and test * Update publish-kinds-next.yml * fix script * try with the app name * Update publish-kinds-next.yml * clean up and update release workflow * add comment * publish kinds only on cue updates --- .github/CODEOWNERS | 5 + .github/workflows/publish-kinds-next.yml | 49 +++ .github/workflows/publish-kinds-release.yml | 70 +++ .../workflows/scripts/kinds/verify-kinds.go | 414 ++++++++++++++++++ .github/workflows/verify-kinds.yml | 25 ++ go.mod | 8 +- go.sum | 8 +- pkg/cuectx/load.go | 3 + pkg/plugins/pfs/pfs.go | 43 ++ 9 files changed, 617 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/publish-kinds-next.yml create mode 100644 .github/workflows/publish-kinds-release.yml create mode 100644 .github/workflows/scripts/kinds/verify-kinds.go create mode 100644 .github/workflows/verify-kinds.yml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4ab61bec336..16a876aaf4d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -619,6 +619,11 @@ embed.go @grafana/grafana-as-code /.github/workflows/stale.yml @grafana/grafana-frontend-platform /.github/workflows/update-changelog.yml @grafana/grafana-delivery /.github/workflows/snyk.yml @grafana/security-team +/.github/workflows/scripts/kinds/verify-kinds.go @grafana/grafana-as-code +/.github/workflows/publish-kinds-next.yml @grafana/grafana-as-code +/.github/workflows/publish-kinds-release.yml @grafana/grafana-as-code +/.github/workflows/verify-kinds.yml @grafana/grafana-as-code + # Generated files not requiring owner approval /packages/grafana-data/src/types/featureToggles.gen.ts @grafanabot diff --git a/.github/workflows/publish-kinds-next.yml b/.github/workflows/publish-kinds-next.yml new file mode 100644 index 00000000000..7f1bbde3962 --- /dev/null +++ b/.github/workflows/publish-kinds-next.yml @@ -0,0 +1,49 @@ +name: "publish-kinds-next" + +on: + push: + branches: + - "main" + paths: + - '**/*.cue' + workflow_dispatch: + +jobs: + main: + runs-on: "ubuntu-latest" + steps: + - name: "Checkout Grafana repo" + uses: "actions/checkout@v3" + with: + fetch-depth: 0 + + - name: "Setup Go" + uses: "actions/setup-go@v4" + with: + go-version: '1.20.4' + + - name: "Verify kinds" + run: go run .github/workflows/scripts/kinds/verify-kinds.go + env: + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + + - name: "Generate token" + id: generate_token + uses: tibdex/github-app-token@b62528385c34dbc9f38e5f4225ac829252d1ea92 + with: + app_id: ${{ secrets.GRAFANA_DELIVERY_BOT_APP_ID }} + private_key: ${{ secrets.GRAFANA_DELIVERY_BOT_APP_PEM }} + + - name: "Clone website-sync Action" + run: "git clone --single-branch --no-tags --depth 1 -b master https://grafana-delivery-bot:${{ steps.generate_token.outputs.token }}@github.com/grafana/website-sync ./.github/actions/website-sync" + + - name: "Publish to kind registry (next)" + uses: "./.github/actions/website-sync" + id: "publish-next" + with: + repository: "grafana/kind-registry" + branch: "main" + host: "github.com" + github_pat: "grafana-delivery-bot:${{ steps.generate_token.outputs.token }}" + source_folder: ".github/workflows/scripts/kinds/next" + target_folder: "grafana/next" diff --git a/.github/workflows/publish-kinds-release.yml b/.github/workflows/publish-kinds-release.yml new file mode 100644 index 00000000000..345eae3d44c --- /dev/null +++ b/.github/workflows/publish-kinds-release.yml @@ -0,0 +1,70 @@ +name: "publish-kinds-release" + +on: + push: + branches: + - v[0-9]+.[0-9]+.x + tags: + - v[0-9]+.[0-9]+.[0-9]+ + paths: + - '**/*.cue' + workflow_dispatch: + +jobs: + main: + runs-on: "ubuntu-latest" + steps: + - name: "Checkout Grafana repo" + uses: "actions/checkout@v3" + with: + fetch-depth: 0 + + - name: "Setup Go" + uses: "actions/setup-go@v4" + with: + go-version: '1.20.4' + + - name: "Verify kinds" + run: go run .github/workflows/scripts/kinds/verify-kinds.go + env: + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + + - name: "Checkout Actions library" + uses: "actions/checkout@v3" + with: + repository: "grafana/grafana-github-actions" + path: "./actions" + + - name: "Install Actions from library" + run: "npm install --production --prefix ./actions" + + - name: "Determine if there is a matching release tag" + id: "has-matching-release-tag" + uses: "./actions/has-matching-release-tag" + with: + ref_name: "${{ github.ref_name }}" + release_tag_regexp: "^v(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)$" + release_branch_regexp: "^v(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.x$" + + - name: "Generate token" + id: generate_token + uses: tibdex/github-app-token@b62528385c34dbc9f38e5f4225ac829252d1ea92 + with: + app_id: ${{ secrets.GRAFANA_DELIVERY_BOT_APP_ID }} + private_key: ${{ secrets.GRAFANA_DELIVERY_BOT_APP_PEM }} + + - name: "Clone website-sync Action" + if: "steps.has-matching-release-tag.outputs.bool == 'true'" + run: "git clone --single-branch --no-tags --depth 1 -b master https://grafana-delivery-bot:${{ steps.generate_token.outputs.token }}@github.com/grafana/website-sync ./.github/actions/website-sync" + + - name: "Publish to kind registry (release)" + if: "steps.has-matching-release-tag.outputs.bool == 'true'" + uses: "./.github/actions/website-sync" + id: "publish-release" + with: + repository: "grafana/kind-registry" + branch: "main" + host: "github.com" + github_pat: "grafana-delivery-bot:${{ steps.generate_token.outputs.token }}" + source_folder: ".github/workflows/scripts/kinds/next" + target_folder: "grafana/${{ github.ref_name }}" diff --git a/.github/workflows/scripts/kinds/verify-kinds.go b/.github/workflows/scripts/kinds/verify-kinds.go new file mode 100644 index 00000000000..cf55c1c369e --- /dev/null +++ b/.github/workflows/scripts/kinds/verify-kinds.go @@ -0,0 +1,414 @@ +package main + +import ( + "context" + "fmt" + "net/http" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "testing/fstest" + + "cuelang.org/go/cue" + cueformat "cuelang.org/go/cue/format" + "github.com/google/go-github/github" + "github.com/grafana/codejen" + "github.com/grafana/grafana/pkg/codegen" + "github.com/grafana/grafana/pkg/cuectx" + "github.com/grafana/grafana/pkg/plugins/pfs" + "github.com/grafana/grafana/pkg/plugins/pfs/corelist" + "github.com/grafana/grafana/pkg/registry/corekind" + "github.com/grafana/kindsys" + "github.com/grafana/thema" + "golang.org/x/oauth2" +) + +const ( + GITHUB_OWNER = "grafana" + GITHUB_REPO = "kind-registry" +) + +// main This script verifies that stable kinds are not updated once published (new schemas +// can be added but existing ones cannot be updated). +// If the env variable CODEGEN_VERIFY is not present, this also generates kind files into a +// local "next" folder, ready to be published in the kind-registry repo. +// If kind names are given as parameters, the script will make the above actions only for the +// given kinds. +func main() { + var kindArgs []string + if len(os.Args) > 1 { + kindArgs = os.Args[1:] + } + + var corek []kindsys.Kind + var compok []kindsys.Composable + + // This script will reach the GH API rate limit if ran for all kinds without token. + // If you don't have a GH token, run the script with kind names as parameters to run + // it only for a limited set of kinds. + var ts oauth2.TokenSource + token, ok := os.LookupEnv("GITHUB_TOKEN") + if ok { + ts = oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: token}, + ) + } + + ctx := context.Background() + tc := oauth2.NewClient(ctx, ts) + client := github.NewClient(tc) + + // Search for the latest version directory present in the kind-registry repo + latestRegistryDir, err := findLatestDir(ctx, client) + if err != nil { + die(fmt.Errorf("failed to get latest directory for published kinds: %s", err)) + } + + errs := make([]error, 0) + + // Kind verification + for _, kind := range corekind.NewBase(nil).All() { + if len(kindArgs) > 0 && !contains(kindArgs, kind.Name()) { + continue + } + + name := kind.Props().Common().MachineName + err := verifyKind(kind, name, "core", latestRegistryDir) + if err != nil { + errs = append(errs, err) + continue + } + + corek = append(corek, kind) + } + + for _, pp := range corelist.New(nil) { + // ElasticSearch composable kind causes the CUE evaluator to hand + // see https://github.com/grafana/grafana/pull/68034#discussion_r1187800059 + if pp.Properties.Id == "elasticsearch" { + continue + } + + for _, kind := range pp.ComposableKinds { + if len(kindArgs) > 0 && !contains(kindArgs, kind.Name()) { + continue + } + + si, err := kindsys.FindSchemaInterface(kind.Def().Properties.SchemaInterface) + if err != nil { + errs = append(errs, err) + continue + } + + name := strings.ToLower(fmt.Sprintf("%s/%s", strings.TrimSuffix(kind.Lineage().Name(), si.Name()), si.Name())) + err = verifyKind(kind, name, "composable", latestRegistryDir) + if err != nil { + errs = append(errs, err) + continue + } + + compok = append(compok, kind) + } + } + + die(errs...) + + if _, set := os.LookupEnv("CODEGEN_VERIFY"); set { + os.Exit(0) + } + + // File generation + jfs := codejen.NewFS() + registryPath := filepath.Join(".github", "workflows", "scripts", "kinds") + + coreJennies := codejen.JennyList[kindsys.Kind]{} + coreJennies.Append( + KindRegistryJenny(registryPath, kindArgs), + ) + corefs, err := coreJennies.GenerateFS(corek...) + die(err) + die(jfs.Merge(corefs)) + + composableJennies := codejen.JennyList[kindsys.Composable]{} + composableJennies.Append( + ComposableKindRegistryJenny(registryPath, kindArgs), + ) + composablefs, err := composableJennies.GenerateFS(compok...) + die(err) + die(jfs.Merge(composablefs)) + + if err = jfs.Write(context.Background(), ""); err != nil { + die(fmt.Errorf("error while writing generated code to disk:\n%s", err)) + } +} + +func die(errs ...error) { + if len(errs) > 0 && errs[0] != nil { + for _, err := range errs { + fmt.Fprint(os.Stderr, err, "\n") + } + fmt.Println("Run `go run verify-kinds.go ` to run the script on a limited set of kinds.") + os.Exit(1) + } +} + +// verifyKind verifies that stable kinds are not updated once published (new schemas +// can be added but existing ones cannot be updated) +func verifyKind(kind kindsys.Kind, name string, category string, latestRegistryDir string) error { + oldKindString, err := getPublishedKind(name, category, latestRegistryDir) + if err != nil { + return err + } + + var oldKind kindsys.Kind + if oldKindString != "" { + switch category { + case "core": + oldKind, err = loadCoreKind(name, oldKindString) + case "composable": + oldKind, err = loadComposableKind(name, oldKindString) + default: + return fmt.Errorf("kind can only be core or composable") + } + } + if err != nil { + return err + } + + // Kind is new - no need to compare it + if oldKind == nil { + return nil + } + + // Check that maturity isn't downgraded + if kind.Maturity().Less(oldKind.Maturity()) { + return fmt.Errorf("kind maturity can't be downgraded once a kind is published") + } + + if oldKind.Maturity().Less(kindsys.MaturityStable) { + return nil + } + + // Check that old schemas do not contain updates + err = thema.IsAppendOnly(oldKind.Lineage(), kind.Lineage()) + if err != nil { + return fmt.Errorf("existing schemas in lineage %s cannot be modified: %w", name, err) + } + + return nil +} + +// getPublishedKind retrieves the latest published kind from the kind registry +func getPublishedKind(name string, category string, latestRegistryDir string) (string, error) { + var ts oauth2.TokenSource + token, ok := os.LookupEnv("GITHUB_TOKEN") + if ok { + ts = oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: token}, + ) + } + + ctx := context.Background() + tc := oauth2.NewClient(ctx, ts) + client := github.NewClient(tc) + + if latestRegistryDir == "" { + return "", nil + } + + file, _, resp, err := client.Repositories.GetContents(ctx, GITHUB_OWNER, GITHUB_REPO, fmt.Sprintf("grafana/%s/%s/%s.cue", latestRegistryDir, category, name), nil) + if err != nil { + if resp.StatusCode == http.StatusNotFound { + return "", nil + } + return "", fmt.Errorf("error retrieving published kind from GH, %d: %w", resp.StatusCode, err) + } + content, err := file.GetContent() + if err != nil { + return "", fmt.Errorf("error decoding published kind content: %w", err) + } + + return content, nil +} + +// findLatestDir get the latest version directory published in the kind registry +func findLatestDir(ctx context.Context, client *github.Client) (string, error) { + re := regexp.MustCompile(`([0-9]+)\.([0-9]+)\.([0-9]+)`) + latestVersion := []uint64{0, 0, 0} + latestDir := "" + + _, dir, resp, err := client.Repositories.GetContents(ctx, GITHUB_OWNER, GITHUB_REPO, "grafana", nil) + if err != nil { + if resp.StatusCode == http.StatusNotFound { + return "", nil + } + return "", err + } + + for _, content := range dir { + if content.GetType() != "dir" { + continue + } + + parts := re.FindStringSubmatch(content.GetName()) + if parts == nil || len(parts) < 4 { + continue + } + + version := make([]uint64, len(parts)-1) + for i := 1; i < len(parts); i++ { + version[i-1], _ = strconv.ParseUint(parts[i], 10, 32) + } + + if isLess(latestVersion, version) { + latestVersion = version + latestDir = content.GetName() + } + } + + return latestDir, nil +} + +func isLess(v1 []uint64, v2 []uint64) bool { + if len(v1) == 1 || len(v2) == 1 { + return v1[0] < v2[0] + } + + return v1[0] < v2[0] || (v1[0] == v2[0] && isLess(v1[2:], v2[2:])) +} + +func loadCoreKind(name string, kind string) (kindsys.Kind, error) { + fs := fstest.MapFS{ + fmt.Sprintf("%s.cue", name): &fstest.MapFile{ + Data: []byte(kind), + }, + } + + rt := cuectx.GrafanaThemaRuntime() + + def, err := cuectx.LoadCoreKindDef(fmt.Sprintf("%s.cue", name), rt.Context(), fs) + if err != nil { + return nil, fmt.Errorf("%s is not a valid kind: %w", name, err) + } + + return kindsys.BindCore(rt, def) +} + +func loadComposableKind(name string, kind string) (kindsys.Kind, error) { + parts := strings.Split(name, "/") + if len(parts) > 1 { + name = parts[1] + } + + fs := fstest.MapFS{ + fmt.Sprintf("%s.cue", name): &fstest.MapFile{ + Data: []byte("package grafanaplugin\n" + kind), + }, + } + + rt := cuectx.GrafanaThemaRuntime() + + def, err := pfs.LoadComposableKindDef(fs, rt, fmt.Sprintf("%s.cue", name)) + if err != nil { + return nil, fmt.Errorf("%s is not a valid kind: %w", name, err) + } + + return kindsys.BindComposable(rt, def) +} + +// KindRegistryJenny generates kind files into the "next" folder of the local kind registry. +func KindRegistryJenny(path string, kindSet []string) codegen.OneToOne { + return &kindregjenny{ + path: path, + kindSet: kindSet, + } +} + +type kindregjenny struct { + path string + kindSet []string +} + +func (j *kindregjenny) JennyName() string { + return "KindRegistryJenny" +} + +func (j *kindregjenny) Generate(kind kindsys.Kind) (*codejen.File, error) { + if len(j.kindSet) > 0 && !contains(j.kindSet, kind.Name()) { + return nil, nil + } + + name := kind.Props().Common().MachineName + core, ok := kind.(kindsys.Core) + if !ok { + return nil, fmt.Errorf("kind sent to KindRegistryJenny must be a core kind") + } + + newKindBytes, err := kindToBytes(core.Def().V) + if err != nil { + return nil, err + } + + path := filepath.Join(j.path, "next", "core", name+".cue") + return codejen.NewFile(path, newKindBytes, j), nil +} + +// kindToBytes converts a kind cue value to a .cue file content +func kindToBytes(kind cue.Value) ([]byte, error) { + node := kind.Syntax( + cue.All(), + cue.Schema(), + cue.Docs(true), + ) + + return cueformat.Node(node) +} + +// ComposableKindRegistryJenny generates kind files into the "next" folder of the local kind registry. +func ComposableKindRegistryJenny(path string, kindSet []string) codejen.OneToOne[kindsys.Composable] { + return &ckrJenny{ + path: path, + kindSet: kindSet, + } +} + +type ckrJenny struct { + path string + kindSet []string +} + +func (j *ckrJenny) JennyName() string { + return "ComposableKindRegistryJenny" +} + +func (j *ckrJenny) Generate(k kindsys.Composable) (*codejen.File, error) { + if len(j.kindSet) > 0 && !contains(j.kindSet, k.Name()) { + return nil, nil + } + + si, err := kindsys.FindSchemaInterface(k.Def().Properties.SchemaInterface) + if err != nil { + panic(err) + } + + name := strings.ToLower(fmt.Sprintf("%s/%s", strings.TrimSuffix(k.Lineage().Name(), si.Name()), si.Name())) + + newKindBytes, err := kindToBytes(k.Def().V) + if err != nil { + return nil, err + } + + return codejen.NewFile(filepath.Join(j.path, "next", "composable", name+".cue"), newKindBytes, j), nil +} + +func contains(array []string, value string) bool { + for _, v := range array { + if v == value { + return true + } + } + + return false +} diff --git a/.github/workflows/verify-kinds.yml b/.github/workflows/verify-kinds.yml new file mode 100644 index 00000000000..60f13dbdb2c --- /dev/null +++ b/.github/workflows/verify-kinds.yml @@ -0,0 +1,25 @@ +name: "verify-kinds" + +on: + pull_request: + branches: [ main ] + +jobs: + main: + runs-on: "ubuntu-latest" + steps: + - name: "Checkout Grafana repo" + uses: "actions/checkout@v3" + with: + fetch-depth: 0 + + - name: "Setup Go" + uses: "actions/setup-go@v4" + with: + go-version: '1.20.4' + + - name: "Verify kinds" + run: go run .github/workflows/scripts/kinds/verify-kinds.go + env: + CODEGEN_VERIFY: 1 + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} diff --git a/go.mod b/go.mod index 090bb7d725d..7973077ab80 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ replace github.com/prometheus/prometheus => github.com/prometheus/prometheus v0. require ( cloud.google.com/go/storage v1.28.1 - cuelang.org/go v0.5.0 + cuelang.org/go v0.6.0-0.dev github.com/Azure/azure-sdk-for-go v65.0.0+incompatible github.com/Azure/go-autorest/autorest v0.11.28 github.com/BurntSushi/toml v1.2.1 @@ -122,7 +122,7 @@ require ( gopkg.in/mail.v2 v2.3.1 gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 - xorm.io/builder v0.3.6 // indirect + xorm.io/builder v0.3.6 xorm.io/core v0.7.3 xorm.io/xorm v0.8.2 ) @@ -205,7 +205,7 @@ require ( github.com/rs/cors v1.9.0 // indirect github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect github.com/segmentio/encoding v0.3.6 // indirect - github.com/sergi/go-diff v1.2.0 // indirect + github.com/sergi/go-diff v1.3.1 // indirect github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 // indirect github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 // indirect github.com/sirupsen/logrus v1.9.0 // indirect @@ -265,7 +265,7 @@ require ( github.com/grafana/dataplane/sdata v0.0.6 github.com/grafana/go-mssqldb v0.9.1 github.com/grafana/kindsys v0.0.0-20230508162304-452481b63482 - github.com/grafana/thema v0.0.0-20230524160113-4e9d6e28a640 + github.com/grafana/thema v0.0.0-20230601172625-e3eaca4d36bd github.com/ory/fosite v0.44.1-0.20230317114349-45a6785cc54f github.com/redis/go-redis/v9 v9.0.2 github.com/weaveworks/common v0.0.0-20230208133027-16871410fca4 diff --git a/go.sum b/go.sum index d585689210c..7f7c5193ea2 100644 --- a/go.sum +++ b/go.sum @@ -1394,8 +1394,8 @@ github.com/grafana/saml v0.4.13-0.20230331080031-67cbfa09c7b6 h1:oHn/OOUkECNX06D github.com/grafana/saml v0.4.13-0.20230331080031-67cbfa09c7b6/go.mod h1:igEejV+fihTIlHXYP8zOec3V5A8y3lws5bQBFsTm4gA= github.com/grafana/sqlds/v2 v2.3.10 h1:HWKhE0vR6LoEiE+Is8CSZOgaB//D1yqb2ntkass9Fd4= github.com/grafana/sqlds/v2 v2.3.10/go.mod h1:c6ibxnxRVGxV/0YkEgvy7QpQH/lyifFyV7K/14xvdIs= -github.com/grafana/thema v0.0.0-20230524160113-4e9d6e28a640 h1:kFCq4pmBB61xzTHen/TNttd5hNlkeqGfqU9usmCc81U= -github.com/grafana/thema v0.0.0-20230524160113-4e9d6e28a640/go.mod h1:Pn9nfzCk7nV0mvNgwusgCjCROZP6nm4GpwTnmEhLT24= +github.com/grafana/thema v0.0.0-20230601172625-e3eaca4d36bd h1:gK5LMNi8AUp8xcrOSNfbvE5g3dT46Qgy+/pCon1Z9jc= +github.com/grafana/thema v0.0.0-20230601172625-e3eaca4d36bd/go.mod h1:KWAKeFXxQYiJ/kBVbijBLRVq9atxkfkeeFIvmj4clEA= github.com/grafana/xorm v0.8.3-0.20220614223926-2fcda7565af6 h1:I9dh1MXGX0wGyxdV/Sl7+ugnki4Dfsy8lv2s5Yf887o= github.com/grafana/xorm v0.8.3-0.20220614223926-2fcda7565af6/go.mod h1:ZkJLEYLoVyg7amJK/5r779bHyzs2AU8f8VMiP6BM7uY= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= @@ -2265,8 +2265,8 @@ github.com/sercand/kuberesolver v2.4.0+incompatible/go.mod h1:lWF3GL0xptCB/vCiJP github.com/serenize/snaker v0.0.0-20171204205717-a683aaf2d516/go.mod h1:Yow6lPLSAXx2ifx470yD/nUe22Dv5vBvxK/UK9UUTVs= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= -github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= +github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= github.com/shirou/gopsutil v3.21.6+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shoenig/test v0.6.2/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= diff --git a/pkg/cuectx/load.go b/pkg/cuectx/load.go index 730cea4b355..e42af45c434 100644 --- a/pkg/cuectx/load.go +++ b/pkg/cuectx/load.go @@ -136,6 +136,9 @@ func LoadGrafanaInstance(relpath string, pkg string, overlay fs.FS) (*build.Inst if err != nil { return nil, err } + + // merge the kindsys filesystem with ours for the kind categories + f = merged_fs.NewMergedFS(kindsys.CueSchemaFS, f) } if pkg != "" { diff --git a/pkg/plugins/pfs/pfs.go b/pkg/plugins/pfs/pfs.go index 1aef21090e4..4aeb77798de 100644 --- a/pkg/plugins/pfs/pfs.go +++ b/pkg/plugins/pfs/pfs.go @@ -232,6 +232,49 @@ func ParsePluginFS(fsys fs.FS, rt *thema.Runtime) (ParsedPlugin, error) { return pp, nil } +// LoadComposableKindDef loads and validates a composable kind definition. +// On success, it returns a [Def] which contains the entire contents of the kind definition. +// +// defpath is the path to the directory containing the composable kind definition, +// relative to the root of the caller's repository. +// +// NOTE This function will be deprecated in favor of a more generic loader when kind +// providers will be implemented. +func LoadComposableKindDef(fsys fs.FS, rt *thema.Runtime, defpath string) (kindsys.Def[kindsys.ComposableProperties], error) { + pp := ParsedPlugin{ + ComposableKinds: make(map[string]kindsys.Composable), + Properties: plugindef.PluginDef{ + Id: defpath, + }, + } + + fsys, err := ensureCueMod(fsys, pp.Properties) + if err != nil { + return kindsys.Def[kindsys.ComposableProperties]{}, fmt.Errorf("%s has invalid cue.mod: %w", pp.Properties.Id, err) + } + + bi, err := cuectx.LoadInstanceWithGrafana(fsys, "", load.Package(PackageName)) + if err != nil { + return kindsys.Def[kindsys.ComposableProperties]{}, err + } + + ctx := rt.Context() + v := ctx.BuildInstance(bi) + if v.Err() != nil { + return kindsys.Def[kindsys.ComposableProperties]{}, fmt.Errorf("%s not a valid CUE instance: %w", defpath, v.Err()) + } + + props, err := kindsys.ToKindProps[kindsys.ComposableProperties](v) + if err != nil { + return kindsys.Def[kindsys.ComposableProperties]{}, err + } + + return kindsys.Def[kindsys.ComposableProperties]{ + V: v, + Properties: props, + }, nil +} + func ensureCueMod(fsys fs.FS, pdef plugindef.PluginDef) (fs.FS, error) { if modf, err := fs.ReadFile(fsys, "cue.mod/module.cue"); err != nil { if !errors.Is(err, fs.ErrNotExist) {