public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Provisioning: Support migrate to local (#103247)

Browse Source
This commit is contained in:
Ryan McKinley
2025-04-02 11:25:06 +03:00
committed by GitHub
parent 6781612335
commit 34192e3ad0
10 changed files with 79 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go.mod
+1 -4
View File
@@ -37,6 +37,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/ec2 v1.200.0 // @grafana/aws-datasources
github.com/aws/aws-sdk-go-v2/service/oam v1.15.13 // @grafana/aws-datasources
github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.25.13 // @grafana/aws-datasources
github.com/aws/smithy-go v1.22.2 // @grafana/aws-datasources
github.com/beevik/etree v1.4.1 // @grafana/grafana-backend-group
github.com/benbjohnson/clock v1.3.5 // @grafana/alerting-backend
github.com/blang/semver/v4 v4.0.0 // indirect; @grafana/grafana-developer-enablement-squad
@@ -235,8 +236,6 @@ require (
github.com/grafana/grafana/pkg/apis/folder v0.0.0-20250327091030-e216c2f29dcc // @grafana/grafana-app-platform-squad
)
require github.com/aws/smithy-go v1.22.2 // @grafana/aws-datasources
require (
cel.dev/expr v0.19.1 // indirect
cloud.google.com/go v0.118.2 // indirect
@@ -582,8 +581,6 @@ require (
sigs.k8s.io/yaml v1.4.0 // indirect
)
require github.com/grafana/grafana/pkg/build v0.0.0-20250401142320-991b016d6319 // @grafana/grafana-developer-enablement-squad
// Use fork of crewjam/saml with fixes for some issues until changes get merged into upstream
replace github.com/crewjam/saml => github.com/grafana/saml v0.4.15-0.20240917091248-ae3bbdad8a56
go.sum
-2
View File
@@ -1626,8 +1626,6 @@ github.com/grafana/grafana/pkg/apis/secret v0.0.0-20250319110241-5a004939da2a h1
github.com/grafana/grafana/pkg/apis/secret v0.0.0-20250319110241-5a004939da2a/go.mod h1:K/fP4kODJmABug5b90PhACUZD6Xh/veEz2b1VRKNyuA=
github.com/grafana/grafana/pkg/apiserver v0.0.0-20250325075903-77fa2271be7a h1:NN0j9zdqYpfliR0P+au/PAJ5lqP7IZPNe8tAX5eaQNE=
github.com/grafana/grafana/pkg/apiserver v0.0.0-20250325075903-77fa2271be7a/go.mod h1:3Z958XEs20R6Wf5y4TFD07PGuGld6grB+wZ1qP/iyqg=
github.com/grafana/grafana/pkg/build v0.0.0-20250401142320-991b016d6319 h1:Yh/pW6dY9fb9473zeL706kuqZlNG4jkMkiwljM665zI=
github.com/grafana/grafana/pkg/build v0.0.0-20250401142320-991b016d6319/go.mod h1:xgwzIkkrPbAIY6PXa2fuCBzad5cFM4hdHKW7/V32szE=
github.com/grafana/grafana/pkg/promlib v0.0.8 h1:VUWsqttdf0wMI4j9OX9oNrykguQpZcruudDAFpJJVw0=
github.com/grafana/grafana/pkg/promlib v0.0.8/go.mod h1:U1ezG/MGaEPoThqsr3lymMPN5yIPdVTJnDZ+wcXT+ao=
github.com/grafana/grafana/pkg/semconv v0.0.0-20250220164708-c8d4ff28a450 h1:wSqgLKFwI7fyeqf3djRXGClBLb/UPjZ4XPm/UsKFDB0=
pkg/registry/apis/provisioning/files.go
+1 -1
View File
@@ -107,7 +107,7 @@ func (s *filesConnector) Connect(ctx context.Context, name string, opts runtime.
}
if filePath == "" {
responder.Error(apierrors.NewBadRequest("path is required"))
responder.Error(apierrors.NewBadRequest("missing request path"))
return
}
pkg/registry/apis/provisioning/jobs/driver.go
+12 -2
View File
@@ -5,11 +5,12 @@ import (
"errors"
"time"
"k8s.io/apiserver/pkg/endpoints/request"
"github.com/grafana/grafana-app-sdk/logging"
"github.com/grafana/grafana/pkg/apimachinery/identity"
provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/apifmt"
"k8s.io/apiserver/pkg/endpoints/request"
)
// Store is an abstraction for the storage API.
@@ -152,12 +153,21 @@ func (d *jobDriver) drive(ctx context.Context) error {
// Process the job.
start := time.Now()
job.Status.Started = start.UnixMilli()
err = d.processJob(ctx, job) // NOTE: We pass in a pointer here such that the job status can be kept in Complete without re-fetching.
end := time.Now()
logger.Debug("job processed", "duration", end.Sub(start), "error", err)
// Mark the job as failed and remove from queue
if err != nil {
return apifmt.Errorf("failed to process job '%s' in '%s': %w", job.GetName(), job.GetNamespace(), err)
job.Status.State = provisioning.JobStateError
job.Status.Errors = append(job.Status.Errors, err.Error())
}
job.Status.Progress = 0 // clear progressbar
job.Status.Finished = end.UnixMilli()
if !job.Status.State.Finished() {
job.Status.State = provisioning.JobStateSuccess // no error
}
// Mark the job as completed.
pkg/registry/apis/provisioning/jobs/migrate/resources.go
+3 -2
View File
@@ -4,6 +4,8 @@ import (
"context"
"fmt"
"k8s.io/apimachinery/pkg/runtime/schema"
"github.com/grafana/grafana/pkg/apimachinery/utils"
provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
"github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy"
@@ -12,7 +14,6 @@ import (
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
"github.com/grafana/grafana/pkg/storage/unified/parquet"
"github.com/grafana/grafana/pkg/storage/unified/resource"
"k8s.io/apimachinery/pkg/runtime/schema"
)
var _ resource.BulkResourceWriter = (*legacyResourceResourceMigrator)(nil)
@@ -117,7 +118,7 @@ func (r *legacyResourceResourceMigrator) Migrate(ctx context.Context) error {
opts.OnlyCount = false // this time actually write
_, err = r.legacy.Migrate(ctx, opts)
if err != nil {
return fmt.Errorf("error running legacy migrate %s %w", r.kind.Resource, err)
return fmt.Errorf("error running legacy migrate (%s) %w", r.kind.Resource, err)
}
return nil
pkg/registry/apis/provisioning/jobs/migrate/worker.go
+4 -3
View File
@@ -8,6 +8,10 @@ import (
"io"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/client-go/dynamic"
"github.com/grafana/grafana-app-sdk/logging"
provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
"github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy"
@@ -18,9 +22,6 @@ import (
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
"github.com/grafana/grafana/pkg/storage/legacysql/dualwrite"
"github.com/grafana/grafana/pkg/storage/unified/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/client-go/dynamic"
)
type MigrationWorker struct {
pkg/registry/apis/provisioning/repository/local.go
+4
View File
@@ -269,6 +269,10 @@ func (r *localRepository) ReadTree(ctx context.Context, ref string) ([]FileTreeE
Path: strings.TrimLeft(path[rootlen:], "/"),
Size: info.Size(),
}
if entry.Path == "" {
return nil // skip the root file
}
if !info.IsDir() {
entry.Blob = true
entry.Hash, _, err = r.calculateFileHash(path)
pkg/registry/apis/provisioning/repository/local_test.go
+51 -3
View File
@@ -1,6 +1,8 @@
package repository
import (
"context"
"io/fs"
"testing"
"github.com/stretchr/testify/assert"
@@ -12,16 +14,62 @@ import (
func TestLocalResolver(t *testing.T) {
resolver := &LocalFolderResolver{
PermittedPrefixes: []string{
"/github/testdata",
"github",
},
HomePath: "/",
HomePath: "./",
}
_, err := resolver.LocalPath("github/testdata")
fullpath, err := resolver.LocalPath("github/testdata")
require.NoError(t, err)
require.Equal(t, "github/testdata", fullpath)
_, err = resolver.LocalPath("something")
require.Error(t, err)
// Check valid errors
r := NewLocal(&v0alpha1.Repository{
Spec: v0alpha1.RepositorySpec{
Local: &v0alpha1.LocalRepositoryConfig{
Path: "github",
},
},
}, resolver)
// Full tree
tree, err := r.ReadTree(context.Background(), "")
require.NoError(t, err)
names := []string{}
for _, v := range tree {
names = append(names, v.Path)
}
require.Equal(t, []string{
"client.go",
"factory.go",
"impl.go",
"testdata",
"testdata/webhook-issue_comment-created.json",
"testdata/webhook-ping-check.json",
"testdata/webhook-pull_request-opened.json",
"testdata/webhook-push-different_branch.json",
"testdata/webhook-push-nested.json",
"testdata/webhook-push-nothing_relevant.json",
}, names)
v, err := r.Read(context.Background(), "testdata", "")
require.NoError(t, err)
require.Equal(t, "testdata", v.Path)
require.Nil(t, v.Data)
v, err = r.Read(context.Background(), "testdata/webhook-push-nested.json", "")
require.NoError(t, err)
require.Equal(t, "4eb879daca9942a887862b3d76fe9f24528d0408", v.Hash)
// read unknown file
_, err = r.Read(context.Background(), "testdata/missing", "")
require.ErrorIs(t, err, fs.ErrNotExist)
_, err = r.Read(context.Background(), "testdata/webhook-push-nested.json/", "")
require.Error(t, err) // not a directory
}
func TestLocal(t *testing.T) {
pkg/registry/apis/provisioning/resources/folders.go
+3
View File
@@ -153,6 +153,9 @@ func (fm *FolderManager) EnsureTreeExists(ctx context.Context, ref, path string,
if path != "" {
p = safepath.Join(path, p)
}
if !safepath.IsDir(p) {
p = p + "/" // trailing slash indicates folder
}
_, err := fm.repo.Read(ctx, p, ref)
if err != nil && !(errors.Is(err, repository.ErrFileNotFound) || apierrors.IsNotFound(err)) {
pkg/registry/apis/provisioning/resources/parser.go
-4
View File
@@ -121,10 +121,6 @@ func (r *Parser) Parse(ctx context.Context, info *repository.FileInfo, validate
return parsed, err
}
if info.Path == "" {
return parsed, errors.New("path is required")
}
parsed.Obj, parsed.GVK, err = DecodeYAMLObject(bytes.NewBuffer(info.Data))
if err != nil {
logger.Debug("failed to find GVK of the input data", "error", err)