diff --git a/go.mod b/go.mod index 88c5f21caf5..4850f341345 100644 --- a/go.mod +++ b/go.mod @@ -37,6 +37,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/ec2 v1.200.0 // @grafana/aws-datasources github.com/aws/aws-sdk-go-v2/service/oam v1.15.13 // @grafana/aws-datasources github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.25.13 // @grafana/aws-datasources + github.com/aws/smithy-go v1.22.2 // @grafana/aws-datasources github.com/beevik/etree v1.4.1 // @grafana/grafana-backend-group github.com/benbjohnson/clock v1.3.5 // @grafana/alerting-backend github.com/blang/semver/v4 v4.0.0 // indirect; @grafana/grafana-developer-enablement-squad @@ -235,8 +236,6 @@ require ( github.com/grafana/grafana/pkg/apis/folder v0.0.0-20250327091030-e216c2f29dcc // @grafana/grafana-app-platform-squad ) -require github.com/aws/smithy-go v1.22.2 // @grafana/aws-datasources - require ( cel.dev/expr v0.19.1 // indirect cloud.google.com/go v0.118.2 // indirect @@ -582,8 +581,6 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) -require github.com/grafana/grafana/pkg/build v0.0.0-20250401142320-991b016d6319 // @grafana/grafana-developer-enablement-squad - // Use fork of crewjam/saml with fixes for some issues until changes get merged into upstream replace github.com/crewjam/saml => github.com/grafana/saml v0.4.15-0.20240917091248-ae3bbdad8a56 diff --git a/go.sum b/go.sum index d39aba37901..724d8197b32 100644 --- a/go.sum +++ b/go.sum @@ -1626,8 +1626,6 @@ github.com/grafana/grafana/pkg/apis/secret v0.0.0-20250319110241-5a004939da2a h1 github.com/grafana/grafana/pkg/apis/secret v0.0.0-20250319110241-5a004939da2a/go.mod h1:K/fP4kODJmABug5b90PhACUZD6Xh/veEz2b1VRKNyuA= github.com/grafana/grafana/pkg/apiserver v0.0.0-20250325075903-77fa2271be7a h1:NN0j9zdqYpfliR0P+au/PAJ5lqP7IZPNe8tAX5eaQNE= github.com/grafana/grafana/pkg/apiserver v0.0.0-20250325075903-77fa2271be7a/go.mod h1:3Z958XEs20R6Wf5y4TFD07PGuGld6grB+wZ1qP/iyqg= -github.com/grafana/grafana/pkg/build v0.0.0-20250401142320-991b016d6319 h1:Yh/pW6dY9fb9473zeL706kuqZlNG4jkMkiwljM665zI= -github.com/grafana/grafana/pkg/build v0.0.0-20250401142320-991b016d6319/go.mod h1:xgwzIkkrPbAIY6PXa2fuCBzad5cFM4hdHKW7/V32szE= github.com/grafana/grafana/pkg/promlib v0.0.8 h1:VUWsqttdf0wMI4j9OX9oNrykguQpZcruudDAFpJJVw0= github.com/grafana/grafana/pkg/promlib v0.0.8/go.mod h1:U1ezG/MGaEPoThqsr3lymMPN5yIPdVTJnDZ+wcXT+ao= github.com/grafana/grafana/pkg/semconv v0.0.0-20250220164708-c8d4ff28a450 h1:wSqgLKFwI7fyeqf3djRXGClBLb/UPjZ4XPm/UsKFDB0= diff --git a/pkg/registry/apis/provisioning/files.go b/pkg/registry/apis/provisioning/files.go index 8dc2cafb378..53be0555ddf 100644 --- a/pkg/registry/apis/provisioning/files.go +++ b/pkg/registry/apis/provisioning/files.go @@ -107,7 +107,7 @@ func (s *filesConnector) Connect(ctx context.Context, name string, opts runtime. } if filePath == "" { - responder.Error(apierrors.NewBadRequest("path is required")) + responder.Error(apierrors.NewBadRequest("missing request path")) return } diff --git a/pkg/registry/apis/provisioning/jobs/driver.go b/pkg/registry/apis/provisioning/jobs/driver.go index f158dfd83e4..e599792d28b 100644 --- a/pkg/registry/apis/provisioning/jobs/driver.go +++ b/pkg/registry/apis/provisioning/jobs/driver.go @@ -5,11 +5,12 @@ import ( "errors" "time" + "k8s.io/apiserver/pkg/endpoints/request" + "github.com/grafana/grafana-app-sdk/logging" "github.com/grafana/grafana/pkg/apimachinery/identity" provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" "github.com/grafana/grafana/pkg/registry/apis/provisioning/apifmt" - "k8s.io/apiserver/pkg/endpoints/request" ) // Store is an abstraction for the storage API. @@ -152,12 +153,21 @@ func (d *jobDriver) drive(ctx context.Context) error { // Process the job. start := time.Now() + job.Status.Started = start.UnixMilli() err = d.processJob(ctx, job) // NOTE: We pass in a pointer here such that the job status can be kept in Complete without re-fetching. end := time.Now() logger.Debug("job processed", "duration", end.Sub(start), "error", err) + // Mark the job as failed and remove from queue if err != nil { - return apifmt.Errorf("failed to process job '%s' in '%s': %w", job.GetName(), job.GetNamespace(), err) + job.Status.State = provisioning.JobStateError + job.Status.Errors = append(job.Status.Errors, err.Error()) + } + + job.Status.Progress = 0 // clear progressbar + job.Status.Finished = end.UnixMilli() + if !job.Status.State.Finished() { + job.Status.State = provisioning.JobStateSuccess // no error } // Mark the job as completed. diff --git a/pkg/registry/apis/provisioning/jobs/migrate/resources.go b/pkg/registry/apis/provisioning/jobs/migrate/resources.go index 5bd0b203659..0e01c1b251d 100644 --- a/pkg/registry/apis/provisioning/jobs/migrate/resources.go +++ b/pkg/registry/apis/provisioning/jobs/migrate/resources.go @@ -4,6 +4,8 @@ import ( "context" "fmt" + "k8s.io/apimachinery/pkg/runtime/schema" + "github.com/grafana/grafana/pkg/apimachinery/utils" provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" "github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy" @@ -12,7 +14,6 @@ import ( "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" "github.com/grafana/grafana/pkg/storage/unified/parquet" "github.com/grafana/grafana/pkg/storage/unified/resource" - "k8s.io/apimachinery/pkg/runtime/schema" ) var _ resource.BulkResourceWriter = (*legacyResourceResourceMigrator)(nil) @@ -117,7 +118,7 @@ func (r *legacyResourceResourceMigrator) Migrate(ctx context.Context) error { opts.OnlyCount = false // this time actually write _, err = r.legacy.Migrate(ctx, opts) if err != nil { - return fmt.Errorf("error running legacy migrate %s %w", r.kind.Resource, err) + return fmt.Errorf("error running legacy migrate (%s) %w", r.kind.Resource, err) } return nil diff --git a/pkg/registry/apis/provisioning/jobs/migrate/worker.go b/pkg/registry/apis/provisioning/jobs/migrate/worker.go index a841fa5abdf..f1e64f19158 100644 --- a/pkg/registry/apis/provisioning/jobs/migrate/worker.go +++ b/pkg/registry/apis/provisioning/jobs/migrate/worker.go @@ -8,6 +8,10 @@ import ( "io" "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/dynamic" + "github.com/grafana/grafana-app-sdk/logging" provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" "github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy" @@ -18,9 +22,6 @@ import ( "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" "github.com/grafana/grafana/pkg/storage/legacysql/dualwrite" "github.com/grafana/grafana/pkg/storage/unified/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/client-go/dynamic" ) type MigrationWorker struct { diff --git a/pkg/registry/apis/provisioning/repository/local.go b/pkg/registry/apis/provisioning/repository/local.go index 42f568311d6..c68da67d80f 100644 --- a/pkg/registry/apis/provisioning/repository/local.go +++ b/pkg/registry/apis/provisioning/repository/local.go @@ -269,6 +269,10 @@ func (r *localRepository) ReadTree(ctx context.Context, ref string) ([]FileTreeE Path: strings.TrimLeft(path[rootlen:], "/"), Size: info.Size(), } + if entry.Path == "" { + return nil // skip the root file + } + if !info.IsDir() { entry.Blob = true entry.Hash, _, err = r.calculateFileHash(path) diff --git a/pkg/registry/apis/provisioning/repository/local_test.go b/pkg/registry/apis/provisioning/repository/local_test.go index 0052582856c..42e09e725f4 100644 --- a/pkg/registry/apis/provisioning/repository/local_test.go +++ b/pkg/registry/apis/provisioning/repository/local_test.go @@ -1,6 +1,8 @@ package repository import ( + "context" + "io/fs" "testing" "github.com/stretchr/testify/assert" @@ -12,16 +14,62 @@ import ( func TestLocalResolver(t *testing.T) { resolver := &LocalFolderResolver{ PermittedPrefixes: []string{ - "/github/testdata", + "github", }, - HomePath: "/", + HomePath: "./", } - _, err := resolver.LocalPath("github/testdata") + fullpath, err := resolver.LocalPath("github/testdata") require.NoError(t, err) + require.Equal(t, "github/testdata", fullpath) _, err = resolver.LocalPath("something") require.Error(t, err) + + // Check valid errors + r := NewLocal(&v0alpha1.Repository{ + Spec: v0alpha1.RepositorySpec{ + Local: &v0alpha1.LocalRepositoryConfig{ + Path: "github", + }, + }, + }, resolver) + + // Full tree + tree, err := r.ReadTree(context.Background(), "") + require.NoError(t, err) + names := []string{} + for _, v := range tree { + names = append(names, v.Path) + } + require.Equal(t, []string{ + "client.go", + "factory.go", + "impl.go", + "testdata", + "testdata/webhook-issue_comment-created.json", + "testdata/webhook-ping-check.json", + "testdata/webhook-pull_request-opened.json", + "testdata/webhook-push-different_branch.json", + "testdata/webhook-push-nested.json", + "testdata/webhook-push-nothing_relevant.json", + }, names) + + v, err := r.Read(context.Background(), "testdata", "") + require.NoError(t, err) + require.Equal(t, "testdata", v.Path) + require.Nil(t, v.Data) + + v, err = r.Read(context.Background(), "testdata/webhook-push-nested.json", "") + require.NoError(t, err) + require.Equal(t, "4eb879daca9942a887862b3d76fe9f24528d0408", v.Hash) + + // read unknown file + _, err = r.Read(context.Background(), "testdata/missing", "") + require.ErrorIs(t, err, fs.ErrNotExist) + + _, err = r.Read(context.Background(), "testdata/webhook-push-nested.json/", "") + require.Error(t, err) // not a directory } func TestLocal(t *testing.T) { diff --git a/pkg/registry/apis/provisioning/resources/folders.go b/pkg/registry/apis/provisioning/resources/folders.go index 12150403e3c..976eda9e52f 100644 --- a/pkg/registry/apis/provisioning/resources/folders.go +++ b/pkg/registry/apis/provisioning/resources/folders.go @@ -153,6 +153,9 @@ func (fm *FolderManager) EnsureTreeExists(ctx context.Context, ref, path string, if path != "" { p = safepath.Join(path, p) } + if !safepath.IsDir(p) { + p = p + "/" // trailing slash indicates folder + } _, err := fm.repo.Read(ctx, p, ref) if err != nil && !(errors.Is(err, repository.ErrFileNotFound) || apierrors.IsNotFound(err)) { diff --git a/pkg/registry/apis/provisioning/resources/parser.go b/pkg/registry/apis/provisioning/resources/parser.go index a81dc9ccf65..e37227bc4a1 100644 --- a/pkg/registry/apis/provisioning/resources/parser.go +++ b/pkg/registry/apis/provisioning/resources/parser.go @@ -121,10 +121,6 @@ func (r *Parser) Parse(ctx context.Context, info *repository.FileInfo, validate return parsed, err } - if info.Path == "" { - return parsed, errors.New("path is required") - } - parsed.Obj, parsed.GVK, err = DecodeYAMLObject(bytes.NewBuffer(info.Data)) if err != nil { logger.Debug("failed to find GVK of the input data", "error", err)