From 197acd73c051c46ab2aea07d0d2b4ff53ad02a9e Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Thu, 28 Jul 2022 23:26:44 -0700 Subject: [PATCH] Storage: support git + github backed roots (#52192) --- .betterer.results | 13 +- go.mod | 4 +- go.sum | 5 +- pkg/api/api.go | 12 +- pkg/api/http_server.go | 4 +- pkg/server/wire.go | 1 - pkg/services/featuremgmt/registry.go | 7 +- pkg/services/store/config.go | 114 +++++- pkg/services/store/github_helper.go | 182 ++++++++ pkg/services/store/http.go | 105 +++-- pkg/services/store/sanitize.go | 2 +- pkg/services/store/service.go | 196 +++++++-- pkg/services/store/service_test.go | 36 +- pkg/services/store/storage_disk.go | 26 +- pkg/services/store/storage_git.go | 387 ++++++++++++++++++ pkg/services/store/storage_sql.go | 30 +- pkg/services/store/tree.go | 29 +- pkg/services/store/types.go | 58 +-- .../SaveDashboard/SaveDashboardDrawer.tsx | 23 +- .../SaveDashboard/forms/SaveDashboardForm.tsx | 4 +- .../SaveDashboard/forms/SaveToStorageForm.tsx | 222 ++++++++++ .../manage-dashboards/state/actions.ts | 5 - public/app/features/storage/RootView.tsx | 88 ++-- public/app/features/storage/StoragePage.tsx | 3 +- public/app/features/storage/storage.ts | 53 +-- public/app/features/storage/types.ts | 57 +++ 26 files changed, 1391 insertions(+), 275 deletions(-) create mode 100644 pkg/services/store/github_helper.go create mode 100644 pkg/services/store/storage_git.go create mode 100644 public/app/features/dashboard/components/SaveDashboard/forms/SaveToStorageForm.tsx diff --git a/.betterer.results b/.betterer.results index baa129aea9d..871afc69376 100644 --- a/.betterer.results +++ b/.betterer.results @@ -5694,12 +5694,9 @@ exports[`better eslint`] = { "public/app/features/storage/storage.ts:5381": [ [0, 0, 0, "Unexpected any. Specify a different type.", "0"], [0, 0, 0, "Unexpected any. Specify a different type.", "1"], - [0, 0, 0, "Unexpected any. Specify a different type.", "2"], + [0, 0, 0, "Do not use any type assertions.", "2"], [0, 0, 0, "Do not use any type assertions.", "3"], - [0, 0, 0, "Do not use any type assertions.", "4"], - [0, 0, 0, "Unexpected any. Specify a different type.", "5"], - [0, 0, 0, "Unexpected any. Specify a different type.", "6"], - [0, 0, 0, "Do not use any type assertions.", "7"] + [0, 0, 0, "Unexpected any. Specify a different type.", "4"] ], "public/app/features/teams/CreateTeam.test.tsx:5381": [ [0, 0, 0, "Unexpected any. Specify a different type.", "0"] @@ -8528,8 +8525,10 @@ exports[`better eslint`] = { [0, 0, 0, "Do not use any type assertions.", "0"] ], "public/app/plugins/panel/barchart/BarChartPanel.tsx:5381": [ - [0, 0, 0, "Do not use any type assertions.", "0"], - [0, 0, 0, "Do not use any type assertions.", "1"] + [0, 0, 0, "Unexpected any. Specify a different type.", "0"], + [0, 0, 0, "Unexpected any. Specify a different type.", "1"], + [0, 0, 0, "Do not use any type assertions.", "2"], + [0, 0, 0, "Do not use any type assertions.", "3"] ], "public/app/plugins/panel/barchart/TickSpacingEditor.tsx:5381": [ [0, 0, 0, "Unexpected any. Specify a different type.", "0"] diff --git a/go.mod b/go.mod index 55f35a6be91..7cd05dbc66a 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( github.com/go-sql-driver/mysql v1.6.0 github.com/go-stack/stack v1.8.0 github.com/gobwas/glob v0.2.3 - github.com/gofrs/uuid v4.0.0+incompatible + github.com/gofrs/uuid v4.0.0+incompatible // indirect github.com/gogo/protobuf v1.3.2 github.com/golang/mock v1.6.0 github.com/golang/snappy v0.0.4 @@ -245,6 +245,7 @@ require ( github.com/blugelabs/bluge_segment_api v0.2.0 github.com/getkin/kin-openapi v0.94.0 github.com/golang-migrate/migrate/v4 v4.7.0 + github.com/google/go-github/v45 v45.2.0 github.com/grafana/dskit v0.0.0-20211011144203-3a88ec0b675f github.com/grafana/thema v0.0.0-20220726124731-b8017e278cc1 go.etcd.io/etcd/api/v3 v3.5.4 @@ -258,6 +259,7 @@ require ( cloud.google.com/go v0.100.2 // indirect github.com/armon/go-metrics v0.3.10 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/gosimple/unidecode v1.0.1 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect diff --git a/go.sum b/go.sum index f0129208fba..2151dcceef6 100644 --- a/go.sum +++ b/go.sum @@ -1213,8 +1213,11 @@ github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8 github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= -github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-github/v45 v45.2.0 h1:5oRLszbrkvxDDqBCNj2hjDZMKmvexaZ1xw/FCD+K3FI= +github.com/google/go-github/v45 v45.2.0/go.mod h1:FObaZJEDSTa/WGCzZ2Z3eoCDXWJKMenWWTrd8jrta28= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/go-replayers/grpcreplay v1.1.0/go.mod h1:qzAvJ8/wi57zq7gWqaE6AwLM6miiXUQwP1S+I9icmhk= github.com/google/go-replayers/httpreplay v1.1.1/go.mod h1:gN9GeLIs7l6NUoVaSSnv2RiqK1NiwAmD0MrKeC9IIks= github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= diff --git a/pkg/api/api.go b/pkg/api/api.go index 06c55d8a7bc..cdc42bf0a19 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -256,17 +256,7 @@ func (hs *HTTPServer) registerRoutes() { }) if hs.Features.IsEnabled(featuremgmt.FlagStorage) { - apiRoute.Group("/storage", func(storageRoute routing.RouteRegister) { - storageRoute.Get("/list/", routing.Wrap(hs.StorageService.List)) - storageRoute.Get("/list/*", routing.Wrap(hs.StorageService.List)) - storageRoute.Get("/read/*", routing.Wrap(hs.StorageService.Read)) - - // Write paths - storageRoute.Post("/delete/*", reqGrafanaAdmin, routing.Wrap(hs.StorageService.Delete)) - storageRoute.Post("/upload", reqGrafanaAdmin, routing.Wrap(hs.StorageService.Upload)) - storageRoute.Post("/createFolder", reqGrafanaAdmin, routing.Wrap(hs.StorageService.CreateFolder)) - storageRoute.Post("/deleteFolder", reqGrafanaAdmin, routing.Wrap(hs.StorageService.DeleteFolder)) - }) + apiRoute.Group("/storage", hs.StorageService.RegisterHTTPRoutes) } // current org diff --git a/pkg/api/http_server.go b/pkg/api/http_server.go index 26450733d22..1fef87eec81 100644 --- a/pkg/api/http_server.go +++ b/pkg/api/http_server.go @@ -128,7 +128,7 @@ type HTTPServer struct { LivePushGateway *pushhttp.Gateway ThumbService thumbs.Service ExportService export.ExportService - StorageService store.HTTPStorageService + StorageService store.StorageService ContextHandler *contexthandler.ContextHandler SQLStore sqlstore.Store AlertEngine *alerting.AlertEngine @@ -202,7 +202,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi pluginsUpdateChecker *updatechecker.PluginsService, searchUsersService searchusers.Service, dataSourcesService datasources.DataSourceService, secretsService secrets.Service, queryDataService *query.Service, ldapGroups ldap.Groups, teamGuardian teamguardian.TeamGuardian, serviceaccountsService serviceaccounts.Service, - authInfoService login.AuthInfoService, storageService store.HTTPStorageService, + authInfoService login.AuthInfoService, storageService store.StorageService, notificationService *notifications.NotificationService, dashboardService dashboards.DashboardService, dashboardProvisioningService dashboards.DashboardProvisioningService, folderService dashboards.FolderService, datasourcePermissionsService permissions.DatasourcePermissionsService, alertNotificationService *alerting.AlertNotificationService, diff --git a/pkg/server/wire.go b/pkg/server/wire.go index 11eb0b5d7a1..468d9cc866c 100644 --- a/pkg/server/wire.go +++ b/pkg/server/wire.go @@ -200,7 +200,6 @@ var wireBasicSet = wire.NewSet( search.ProvideService, searchV2.ProvideService, store.ProvideService, - store.ProvideHTTPService, export.ProvideService, live.ProvideService, pushhttp.ProvideService, diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index d9a5c242049..9f212954fe8 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -136,9 +136,10 @@ var ( State: FeatureStateAlpha, }, { - Name: "dashboardsFromStorage", - Description: "Load dashboards from the generic storage interface", - State: FeatureStateAlpha, + Name: "dashboardsFromStorage", + Description: "Load dashboards from the generic storage interface", + State: FeatureStateAlpha, + RequiresDevMode: true, // Also a gate on automatic git storage (for now) }, { Name: "export", diff --git a/pkg/services/store/config.go b/pkg/services/store/config.go index 206cfb7112a..acbd2a492ce 100644 --- a/pkg/services/store/config.go +++ b/pkg/services/store/config.go @@ -1,10 +1,110 @@ package store +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + + "github.com/grafana/grafana/pkg/services/featuremgmt" + "github.com/grafana/grafana/pkg/setting" +) + +// For now this file is stored in $GRAFANA_HOME/conf/storage.json and updated from the UI +type GlobalStorageConfig struct { + filepath string // Local file path + + // Defined in grafana.ini + AllowUnsanitizedSvgUpload bool `json:"allowUnsanitizedSvgUpload"` + + // Add dev environment + AddDevEnv bool `json:"addDevEnv"` + + // Paths under 'root' (NOTE: this is applied to all orgs) + Roots []RootStorageConfig `json:"roots"` +} + +func LoadStorageConfig(cfg *setting.Cfg, features featuremgmt.FeatureToggles) (*GlobalStorageConfig, error) { + changed := false + fpath := filepath.Join(cfg.DataPath, "storage", "storage.json") + g := &GlobalStorageConfig{} + if _, err := os.Stat(fpath); err == nil { + // nolint:gosec + // We can ignore the gosec G304 warning since the path is hardcoded above + body, err := ioutil.ReadFile(fpath) + if err != nil { + return g, err + } + err = json.Unmarshal(body, g) + if err != nil { + return g, err + } + } else { + g.AddDevEnv = true + changed = true + } + + if g.Roots == nil && features.IsEnabled(featuremgmt.FlagDashboardsFromStorage) { + g.Roots = append(g.Roots, RootStorageConfig{ + Type: "git", + Prefix: "it-A", + Name: "Repository that requires pull requests", + Git: &StorageGitConfig{ + Remote: "https://github.com/grafana/hackathon-2022-03-git-dash-A", + Branch: "main", + Root: "dashboards", // the dashboard files + RequirePullRequest: true, + AccessToken: "$GRAFANA_STORAGE_GITHUB_ACCESS_TOKEN", + }, + }) + g.Roots = append(g.Roots, RootStorageConfig{ + Type: "git", + Prefix: "it-B", + Name: "Another repo (can push to main)", + Git: &StorageGitConfig{ + Remote: "https://github.com/grafana/hackathon-2022-03-git-dash-B", + Branch: "main", + Root: "dashboards", // the dashboard files + RequirePullRequest: false, + AccessToken: "$GRAFANA_STORAGE_GITHUB_ACCESS_TOKEN", + }, + }) + changed = true + } + + g.filepath = fpath + + // Also configured from ini files + if cfg.Storage.AllowUnsanitizedSvgUpload { + g.AllowUnsanitizedSvgUpload = true + } + + // Save a template version in config + if changed && setting.Env != setting.Prod { + return g, g.save() + } + return g, nil +} + +func (c *GlobalStorageConfig) save() error { + out, err := json.MarshalIndent(c, "", " ") + if err != nil { + return err + } + err = os.MkdirAll(filepath.Dir(c.filepath), 0700) + if err != nil { + return err + } + return ioutil.WriteFile(c.filepath, out, 0600) +} + type RootStorageConfig struct { Type string `json:"type"` Prefix string `json:"prefix"` Name string `json:"name"` Description string `json:"description"` + Disabled bool `json:"disabled,omitempty"` // Depending on type, these will be configured Disk *StorageLocalDiskConfig `json:"disk,omitempty"` @@ -26,7 +126,8 @@ type StorageGitConfig struct { // Pull interval? // Requires pull request? - RequirePullRequest bool `json:"requirePullRequest"` + RequirePullRequest bool `json:"requirePullRequest"` + PullInterval string `json:"pullInterval"` // SECURE JSON :grimicing: AccessToken string `json:"accessToken,omitempty"` // Simplest auth method for github @@ -52,3 +153,14 @@ type StorageGCSConfig struct { CredentialsFile string `json:"credentialsFile"` } + +func newStorage(cfg RootStorageConfig, localWorkCache string) (storageRuntime, error) { + switch cfg.Type { + case rootStorageTypeDisk: + return newDiskStorage(RootStorageMeta{}, cfg), nil + case rootStorageTypeGit: + return newGitStorage(RootStorageMeta{}, cfg, localWorkCache), nil + } + + return nil, fmt.Errorf("unsupported store: " + cfg.Type) +} diff --git a/pkg/services/store/github_helper.go b/pkg/services/store/github_helper.go new file mode 100644 index 00000000000..12b8e0ba9cb --- /dev/null +++ b/pkg/services/store/github_helper.go @@ -0,0 +1,182 @@ +package store + +import ( + "context" + "fmt" + "net/url" + "strings" + "time" + + "github.com/google/go-github/v45/github" + "golang.org/x/oauth2" +) + +type githubHelper struct { + repoOwner string + repoName string + client *github.Client +} + +func newGithubHelper(ctx context.Context, uri string, token string) (*githubHelper, error) { + v, err := url.Parse(uri) + if err != nil { + return nil, err + } + + path := strings.TrimPrefix(v.Path, "/") + path = strings.TrimSuffix(path, ".git") + idx := strings.Index(path, "/") + if idx < 1 { + return nil, fmt.Errorf("invalid url") + } + + if token == "" { + return nil, fmt.Errorf("unauthorized: No token present") + } + ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}) + tc := oauth2.NewClient(ctx, ts) + return &githubHelper{ + client: github.NewClient(tc), + repoOwner: path[:idx], + repoName: path[idx+1:], + }, nil +} + +func (g *githubHelper) getRef(ctx context.Context, branch string) (*github.Reference, *github.Response, error) { + return g.client.Git.GetRef(ctx, g.repoOwner, g.repoName, "refs/heads/"+branch) +} + +func (g *githubHelper) createRef(ctx context.Context, base string, branch string) (ref *github.Reference, rsp *github.Response, err error) { + var baseRef *github.Reference + if baseRef, rsp, err = g.client.Git.GetRef(ctx, g.repoOwner, g.repoName, "refs/heads/"+base); err != nil { + return nil, rsp, err + } + newRef := &github.Reference{ + Ref: github.String("refs/heads/" + branch), + Object: &github.GitObject{SHA: baseRef.Object.SHA}, + } + return g.client.Git.CreateRef(ctx, g.repoOwner, g.repoName, newRef) +} + +func (g *githubHelper) getRepo(ctx context.Context) (*github.Repository, *github.Response, error) { + return g.client.Repositories.Get(ctx, g.repoOwner, g.repoName) +} + +// pushCommit creates the commit in the given reference using the given tree. +func (g *githubHelper) pushCommit(ctx context.Context, ref *github.Reference, cmd *WriteValueRequest) (err error) { + // Create a tree with what to commit. + entries := []*github.TreeEntry{ + { + Path: github.String(cmd.Path), + Type: github.String("blob"), + Content: github.String(string(cmd.Body)), + Mode: github.String("100644"), + }, + } + + tree, _, err := g.client.Git.CreateTree(ctx, g.repoOwner, g.repoName, *ref.Object.SHA, entries) + if err != nil { + return err + } + + // Get the parent commit to attach the commit to. + parent, _, err := g.client.Repositories.GetCommit(ctx, g.repoOwner, g.repoName, *ref.Object.SHA, nil) + if err != nil { + return err + } + + // This is not always populated, but is needed. + parent.Commit.SHA = parent.SHA + + user := cmd.User + name := firstRealString(user.Name, user.Login, user.Email, "?") + email := firstRealString(user.Email, user.Login, user.Name, "?") + + // Create the commit using the tree. + date := time.Now() + author := &github.CommitAuthor{ + Date: &date, + Name: &name, + Email: &email, + } + commit := &github.Commit{Author: author, Message: &cmd.Message, Tree: tree, Parents: []*github.Commit{parent.Commit}} + newCommit, _, err := g.client.Git.CreateCommit(ctx, g.repoOwner, g.repoName, commit) + if err != nil { + return err + } + + // Attach the commit to the main branch. + ref.Object.SHA = newCommit.SHA + _, _, err = g.client.Git.UpdateRef(ctx, g.repoOwner, g.repoName, ref, false) + return err +} + +type makePRCommand struct { + title string + body string + headBranch string + baseBranch string +} + +func (g *githubHelper) createPR(ctx context.Context, cmd makePRCommand) (*github.PullRequest, *github.Response, error) { + newPR := &github.NewPullRequest{ + Title: &cmd.title, + Head: &cmd.headBranch, + Base: &cmd.baseBranch, + Body: &cmd.body, + MaintainerCanModify: github.Bool(true), + } + + return g.client.PullRequests.Create(ctx, g.repoOwner, g.repoName, newPR) +} + +// func (g *githubHelper) getPR(config *Config, prSubject string) (*github.PullRequest, error) { + +// opts := github.PullRequestListOptions{} + +// prs, _, err := githubClient.PullRequests.List(ctx, config.RepoOwner, config.RepoName, &opts) +// if err != nil { +// return nil, err +// } +// for _, pr := range prs { +// log.Printf("PR: %s %s", *pr.Title, prSubject) +// if *pr.Title == prSubject { +// return pr, nil +// } +// } +// return nil, nil +// } + +// func (g *githubHelper) pushPR(config *Config, prSubject, prBranch, prFilename, prContent, commitMessage string) error { +// pr, err := getPR(config, prSubject) +// if err != nil { +// return err +// } +// if pr != nil { +// log.Println("Extending Existing PR", *pr.Title) +// ref, err := getRef(config, pr.GetHead().GetRef()) +// if err != nil { +// return err +// } +// err = pushCommit(config, ref, prFilename, prContent, commitMessage) +// if err != nil { +// return err +// } + +// } else { +// log.Println("Creating PR") +// ref, err := createRef(config, prBranch) +// if err != nil { +// return err +// } +// err = pushCommit(config, ref, prFilename, prContent, commitMessage) +// if err != nil { +// return err +// } +// pr, err = createPR(config, prSubject, prBranch) +// if err != nil { +// return err +// } +// } +// return nil +// } diff --git a/pkg/services/store/http.go b/pkg/services/store/http.go index f8e27fd112a..bac0d4d5062 100644 --- a/pkg/services/store/http.go +++ b/pkg/services/store/http.go @@ -10,34 +10,14 @@ import ( "strings" "github.com/grafana/grafana/pkg/api/response" + "github.com/grafana/grafana/pkg/api/routing" + "github.com/grafana/grafana/pkg/middleware" "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/services/quota" + ac "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/util" "github.com/grafana/grafana/pkg/web" ) -// HTTPStorageService passes raw HTTP requests to a well typed storage service -type HTTPStorageService interface { - List(c *models.ReqContext) response.Response - Read(c *models.ReqContext) response.Response - Delete(c *models.ReqContext) response.Response - DeleteFolder(c *models.ReqContext) response.Response - CreateFolder(c *models.ReqContext) response.Response - Upload(c *models.ReqContext) response.Response -} - -type httpStorage struct { - store StorageService - quotaService quota.Service -} - -func ProvideHTTPService(store StorageService, quotaService quota.Service) HTTPStorageService { - return &httpStorage{ - store: store, - quotaService: quotaService, - } -} - func UploadErrorToStatusCode(err error) int { switch { case errors.Is(err, ErrStorageNotFound): @@ -60,7 +40,37 @@ func UploadErrorToStatusCode(err error) int { } } -func (s *httpStorage) Upload(c *models.ReqContext) response.Response { +func (s *standardStorageService) RegisterHTTPRoutes(storageRoute routing.RouteRegister) { + storageRoute.Get("/list/", routing.Wrap(s.list)) + storageRoute.Get("/list/*", routing.Wrap(s.list)) + storageRoute.Get("/read/*", routing.Wrap(s.read)) + storageRoute.Get("/options/*", routing.Wrap(s.getOptions)) + + // Write paths + reqGrafanaAdmin := middleware.ReqGrafanaAdmin + storageRoute.Post("/write/*", reqGrafanaAdmin, routing.Wrap(s.doWrite)) + storageRoute.Post("/delete/*", reqGrafanaAdmin, routing.Wrap(s.doDelete)) + storageRoute.Post("/upload", reqGrafanaAdmin, routing.Wrap(s.doUpload)) + storageRoute.Post("/createFolder", reqGrafanaAdmin, routing.Wrap(s.doCreateFolder)) + storageRoute.Post("/deleteFolder", reqGrafanaAdmin, routing.Wrap(s.doDeleteFolder)) + storageRoute.Get("/config", reqGrafanaAdmin, routing.Wrap(s.getConfig)) +} + +func (s *standardStorageService) doWrite(c *models.ReqContext) response.Response { + scope, path := getPathAndScope(c) + cmd := &WriteValueRequest{} + if err := web.Bind(c.Req, cmd); err != nil { + return response.Error(http.StatusBadRequest, "bad request data", err) + } + cmd.Path = scope + "/" + path + rsp, err := s.write(c.Req.Context(), c.SignedInUser, cmd) + if err != nil { + return response.Error(http.StatusBadRequest, "save error", err) + } + return response.JSON(200, rsp) +} + +func (s *standardStorageService) doUpload(c *models.ReqContext) response.Response { // assumes we are only uploading to the SQL database - TODO: refactor once we introduce object stores quotaReached, err := s.quotaService.CheckQuotaReached(c.Req.Context(), "file", nil) if err != nil { @@ -127,7 +137,7 @@ func (s *httpStorage) Upload(c *models.ReqContext) response.Response { entityType = EntityTypeImage } - err = s.store.Upload(c.Req.Context(), c.SignedInUser, &UploadRequest{ + err = s.Upload(c.Req.Context(), c.SignedInUser, &UploadRequest{ Contents: data, EntityType: entityType, Path: path, @@ -157,10 +167,10 @@ func getMultipartFormValue(req *http.Request, key string) string { return v[0] } -func (s *httpStorage) Read(c *models.ReqContext) response.Response { +func (s *standardStorageService) read(c *models.ReqContext) response.Response { // full path is api/storage/read/upload/example.jpg, but we only want the part after read scope, path := getPathAndScope(c) - file, err := s.store.Read(c.Req.Context(), c.SignedInUser, scope+"/"+path) + file, err := s.Read(c.Req.Context(), c.SignedInUser, scope+"/"+path) if err != nil { return response.Error(400, "cannot call read", err) } @@ -176,11 +186,20 @@ func (s *httpStorage) Read(c *models.ReqContext) response.Response { return response.Respond(200, file.Contents) } -func (s *httpStorage) Delete(c *models.ReqContext) response.Response { +func (s *standardStorageService) getOptions(c *models.ReqContext) response.Response { + scope, path := getPathAndScope(c) + opts, err := s.getWorkflowOptions(c.Req.Context(), c.SignedInUser, scope+"/"+path) + if err != nil { + return response.Error(400, err.Error(), err) + } + return response.JSON(200, opts) +} + +func (s *standardStorageService) doDelete(c *models.ReqContext) response.Response { // full path is api/storage/delete/upload/example.jpg, but we only want the part after upload scope, path := getPathAndScope(c) - err := s.store.Delete(c.Req.Context(), c.SignedInUser, scope+"/"+path) + err := s.Delete(c.Req.Context(), c.SignedInUser, scope+"/"+path) if err != nil { return response.Error(400, "failed to delete the file: "+err.Error(), err) } @@ -191,7 +210,7 @@ func (s *httpStorage) Delete(c *models.ReqContext) response.Response { }) } -func (s *httpStorage) DeleteFolder(c *models.ReqContext) response.Response { +func (s *standardStorageService) doDeleteFolder(c *models.ReqContext) response.Response { body, err := io.ReadAll(c.Req.Body) if err != nil { return response.Error(500, "error reading bytes", err) @@ -209,7 +228,7 @@ func (s *httpStorage) DeleteFolder(c *models.ReqContext) response.Response { // full path is api/storage/delete/upload/example.jpg, but we only want the part after upload _, path := getPathAndScope(c) - if err := s.store.DeleteFolder(c.Req.Context(), c.SignedInUser, cmd); err != nil { + if err := s.DeleteFolder(c.Req.Context(), c.SignedInUser, cmd); err != nil { return response.Error(400, "failed to delete the folder: "+err.Error(), err) } @@ -220,7 +239,7 @@ func (s *httpStorage) DeleteFolder(c *models.ReqContext) response.Response { }) } -func (s *httpStorage) CreateFolder(c *models.ReqContext) response.Response { +func (s *standardStorageService) doCreateFolder(c *models.ReqContext) response.Response { body, err := io.ReadAll(c.Req.Body) if err != nil { return response.Error(500, "error reading bytes", err) @@ -236,7 +255,7 @@ func (s *httpStorage) CreateFolder(c *models.ReqContext) response.Response { return response.Error(400, "empty path", err) } - if err := s.store.CreateFolder(c.Req.Context(), c.SignedInUser, cmd); err != nil { + if err := s.CreateFolder(c.Req.Context(), c.SignedInUser, cmd); err != nil { return response.Error(400, "failed to create the folder: "+err.Error(), err) } @@ -247,10 +266,10 @@ func (s *httpStorage) CreateFolder(c *models.ReqContext) response.Response { }) } -func (s *httpStorage) List(c *models.ReqContext) response.Response { +func (s *standardStorageService) list(c *models.ReqContext) response.Response { params := web.Params(c.Req) path := params["*"] - frame, err := s.store.List(c.Req.Context(), c.SignedInUser, path) + frame, err := s.List(c.Req.Context(), c.SignedInUser, path) if err != nil { return response.Error(400, "error reading path", err) } @@ -259,3 +278,19 @@ func (s *httpStorage) List(c *models.ReqContext) response.Response { } return response.JSONStreaming(http.StatusOK, frame) } + +func (s *standardStorageService) getConfig(c *models.ReqContext) response.Response { + roots := make([]RootStorageMeta, 0) + orgId := c.OrgId + t := s.tree + t.assureOrgIsInitialized(orgId) + for _, f := range t.rootsByOrgId[ac.GlobalOrgID] { + roots = append(roots, f.Meta()) + } + if orgId != ac.GlobalOrgID { + for _, f := range t.rootsByOrgId[orgId] { + roots = append(roots, f.Meta()) + } + } + return response.JSON(200, roots) +} diff --git a/pkg/services/store/sanitize.go b/pkg/services/store/sanitize.go index 8efaaf8660e..98435e7d2ad 100644 --- a/pkg/services/store/sanitize.go +++ b/pkg/services/store/sanitize.go @@ -20,7 +20,7 @@ func (s *standardStorageService) sanitizeContents(ctx context.Context, user *mod Content: req.Contents, }) if err != nil { - if s.cfg.allowUnsanitizedSvgUpload { + if s.cfg != nil && s.cfg.AllowUnsanitizedSvgUpload { grafanaStorageLogger.Debug("allowing unsanitized svg upload", "filename", req.Path, "sanitizationError", err) return req.Contents, nil } else { diff --git a/pkg/services/store/service.go b/pkg/services/store/service.go index fbf0244ce65..1e2661a8ca2 100644 --- a/pkg/services/store/service.go +++ b/pkg/services/store/service.go @@ -1,17 +1,22 @@ package store import ( + "bytes" "context" + "encoding/json" "errors" + "fmt" "os" "path/filepath" + "github.com/grafana/grafana/pkg/api/routing" "github.com/grafana/grafana/pkg/infra/filestorage" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/registry" ac "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/featuremgmt" + "github.com/grafana/grafana/pkg/services/quota" "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/setting" ) @@ -24,6 +29,7 @@ var ErrValidationFailed = errors.New("request validation failed") var ErrFileAlreadyExists = errors.New("file exists") var ErrStorageNotFound = errors.New("storage not found") var ErrAccessDenied = errors.New("access denied") +var ErrOnlyDashboardSaveSupported = errors.New("only dashboard save is currently supported") const RootPublicStatic = "public-static" const RootResources = "resources" @@ -52,6 +58,9 @@ type CreateFolderCmd struct { type StorageService interface { registry.BackgroundService + // Register the HTTP + RegisterHTTPRoutes(routing.RouteRegister) + // List folder contents List(ctx context.Context, user *models.SignedInUser, path string) (*StorageListFrame, error) @@ -72,20 +81,31 @@ type StorageService interface { sanitizeUploadRequest(ctx context.Context, user *models.SignedInUser, req *UploadRequest, storagePath string) (*filestorage.UpsertFileCommand, error) } -type storageServiceConfig struct { - allowUnsanitizedSvgUpload bool -} - type standardStorageService struct { - sql *sqlstore.SQLStore - tree *nestedTree - cfg storageServiceConfig - authService storageAuthService + sql *sqlstore.SQLStore + tree *nestedTree + cfg *GlobalStorageConfig + authService storageAuthService + quotaService quota.Service } -func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles, cfg *setting.Cfg) StorageService { +func ProvideService( + sql *sqlstore.SQLStore, + features featuremgmt.FeatureToggles, + cfg *setting.Cfg, + quotaService quota.Service, +) StorageService { + settings, err := LoadStorageConfig(cfg, features) + if err != nil { + grafanaStorageLogger.Warn("error loading storage config", "error", err) + } + + // always exists globalRoots := []storageRuntime{ - newDiskStorage(RootStorageConfig{ + newDiskStorage(RootStorageMeta{ + ReadOnly: true, + Builtin: true, + }, RootStorageConfig{ Prefix: RootPublicStatic, Name: "Public static files", Description: "Access files from the static public files", @@ -98,14 +118,16 @@ func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles, "/maps/", }, }, - }).setReadOnly(true).setBuiltin(true), + }), } // Development dashboards - if setting.Env != setting.Prod { + if settings.AddDevEnv && setting.Env != setting.Prod { devenv := filepath.Join(cfg.StaticRootPath, "..", "devenv") if _, err := os.Stat(devenv); !os.IsNotExist(err) { - s := newDiskStorage(RootStorageConfig{ + s := newDiskStorage(RootStorageMeta{ + ReadOnly: false, + }, RootStorageConfig{ Prefix: RootDevenv, Name: "Development Environment", Description: "Explore files within the developer environment directly", @@ -114,8 +136,21 @@ func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles, Roots: []string{ "/dev-dashboards/", }, - }}).setReadOnly(false) + }}) + globalRoots = append(globalRoots, s) + } + } + for _, root := range settings.Roots { + if root.Prefix == "" { + grafanaStorageLogger.Warn("Invalid root configuration", "cfg", root) + continue + } + s, err := newStorage(root, filepath.Join(cfg.DataPath, "storage", "cache", root.Prefix)) + if err != nil { + grafanaStorageLogger.Warn("error loading storage config", "error", err) + } + if s != nil { globalRoots = append(globalRoots, s) } } @@ -125,19 +160,21 @@ func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles, // Custom upload files storages = append(storages, - newSQLStorage(RootResources, + newSQLStorage(RootStorageMeta{ + Builtin: true, + }, RootResources, "Resources", "Upload custom resource files", - &StorageSQLConfig{}, sql, orgId). - setBuiltin(true)) + &StorageSQLConfig{}, sql, orgId)) // System settings storages = append(storages, - newSQLStorage(RootSystem, + newSQLStorage(RootStorageMeta{ + Builtin: true, + }, RootResources, "System", "Grafana system storage", - &StorageSQLConfig{}, sql, orgId). - setBuiltin(true)) + &StorageSQLConfig{}, sql, orgId)) return storages } @@ -179,25 +216,18 @@ func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles, return nil } - switch storageName { - case RootDevenv: - return map[string]filestorage.PathFilter{ - ActionFilesRead: allowAllPathFilter, - ActionFilesWrite: denyAllPathFilter, - ActionFilesDelete: denyAllPathFilter, - } - case RootResources: - return map[string]filestorage.PathFilter{ - ActionFilesRead: allowAllPathFilter, - ActionFilesWrite: allowAllPathFilter, - ActionFilesDelete: allowAllPathFilter, - } - default: - return nil + // Admin can do anything + return map[string]filestorage.PathFilter{ + ActionFilesRead: allowAllPathFilter, + ActionFilesWrite: allowAllPathFilter, + ActionFilesDelete: allowAllPathFilter, } }) - return newStandardStorageService(sql, globalRoots, initializeOrgStorages, authService, cfg) + s := newStandardStorageService(sql, globalRoots, initializeOrgStorages, authService, cfg) + s.quotaService = quotaService + s.cfg = settings + return s } func createSystemBrandingPathFilter() filestorage.PathFilter { @@ -208,7 +238,13 @@ func createSystemBrandingPathFilter() filestorage.PathFilter { nil) } -func newStandardStorageService(sql *sqlstore.SQLStore, globalRoots []storageRuntime, initializeOrgStorages func(orgId int64) []storageRuntime, authService storageAuthService, cfg *setting.Cfg) *standardStorageService { +func newStandardStorageService( + sql *sqlstore.SQLStore, + globalRoots []storageRuntime, + initializeOrgStorages func(orgId int64) []storageRuntime, + authService storageAuthService, + cfg *setting.Cfg, +) *standardStorageService { rootsByOrgId := make(map[int64][]storageRuntime) rootsByOrgId[ac.GlobalOrgID] = globalRoots @@ -221,9 +257,6 @@ func newStandardStorageService(sql *sqlstore.SQLStore, globalRoots []storageRunt sql: sql, tree: res, authService: authService, - cfg: storageServiceConfig{ - allowUnsanitizedSvgUpload: cfg.Storage.AllowUnsanitizedSvgUpload, - }, } } @@ -377,3 +410,86 @@ func (s *standardStorageService) Delete(ctx context.Context, user *models.Signed } return nil } + +func (s *standardStorageService) write(ctx context.Context, user *models.SignedInUser, req *WriteValueRequest) (*WriteValueResponse, error) { + guardian := s.authService.newGuardian(ctx, user, getFirstSegment(req.Path)) + if !guardian.canWrite(req.Path) { + return nil, ErrAccessDenied + } + + root, storagePath := s.tree.getRoot(getOrgId(user), req.Path) + if root == nil { + return nil, ErrStorageNotFound + } + + if root.Meta().ReadOnly { + return nil, ErrUnsupportedStorage + } + + // not svg! + if req.EntityType != EntityTypeDashboard { + return nil, ErrOnlyDashboardSaveSupported + } + + // Save pretty JSON + var prettyJSON bytes.Buffer + if err := json.Indent(&prettyJSON, req.Body, "", " "); err != nil { + return nil, err + } + req.Body = prettyJSON.Bytes() + + // Modify the save request + req.Path = storagePath + req.User = user + return root.Write(ctx, req) +} + +type workflowInfo struct { + Type WriteValueWorkflow `json:"value"` // value matches selectable value + Label string `json:"label"` + Description string `json:"description,omitempty"` +} +type optionInfo struct { + Path string `json:"path,omitempty"` + Workflows []workflowInfo `json:"workflows"` +} + +func (s *standardStorageService) getWorkflowOptions(ctx context.Context, user *models.SignedInUser, path string) (optionInfo, error) { + options := optionInfo{ + Path: path, + Workflows: make([]workflowInfo, 0), + } + + scope, _ := splitFirstSegment(path) + root, _ := s.tree.getRoot(user.OrgId, scope) + if root == nil { + return options, fmt.Errorf("can not read") + } + + meta := root.Meta() + if meta.Config.Type == rootStorageTypeGit && meta.Config.Git != nil { + cfg := meta.Config.Git + options.Workflows = append(options.Workflows, workflowInfo{ + Type: WriteValueWorkflow_PR, + Label: "Create pull request", + Description: "Create a new upstream pull request", + }) + if !cfg.RequirePullRequest { + options.Workflows = append(options.Workflows, workflowInfo{ + Type: WriteValueWorkflow_Push, + Label: "Push to " + cfg.Branch, + Description: "Push commit to upstrem repository", + }) + } + } else if meta.ReadOnly { + // nothing? + } else { + options.Workflows = append(options.Workflows, workflowInfo{ + Type: WriteValueWorkflow_Save, + Label: "Save", + Description: "Save directly", + }) + } + + return options, nil +} diff --git a/pkg/services/store/service_test.go b/pkg/services/store/service_test.go index 701f01b52bd..871c81c3f86 100644 --- a/pkg/services/store/service_test.go +++ b/pkg/services/store/service_test.go @@ -42,20 +42,24 @@ var ( } }) publicRoot, _ = filepath.Abs("../../../public") - publicStaticFilesStorage = newDiskStorage(RootStorageConfig{ - Prefix: "public", - Name: "Public static files", - Disk: &StorageLocalDiskConfig{ - Path: publicRoot, - Roots: []string{ - "/testdata/", - "/img/icons/", - "/img/bg/", - "/gazetteer/", - "/maps/", - "/upload/", - }, - }}).setReadOnly(true).setBuiltin(true) + publicStaticFilesStorage = newDiskStorage( + RootStorageMeta{ + Builtin: true, + ReadOnly: true, + }, RootStorageConfig{ + Prefix: "public", + Name: "Public static files", + Disk: &StorageLocalDiskConfig{ + Path: publicRoot, + Roots: []string{ + "/testdata/", + "/img/icons/", + "/img/bg/", + "/gazetteer/", + "/maps/", + "/upload/", + }, + }}) ) func TestListFiles(t *testing.T) { @@ -96,6 +100,7 @@ func setupUploadStore(t *testing.T, authService storageAuthService) (StorageServ storageName := "resources" mockStorage := &filestorage.MockFileStorage{} sqlStorage := newSQLStorage( + RootStorageMeta{}, storageName, "Testing upload", "dummy descr", &StorageSQLConfig{}, sqlstore.InitTestDB(t), @@ -109,6 +114,9 @@ func setupUploadStore(t *testing.T, authService storageAuthService) (StorageServ store := newStandardStorageService(sqlstore.InitTestDB(t), []storageRuntime{sqlStorage}, func(orgId int64) []storageRuntime { return make([]storageRuntime, 0) }, authService, cfg) + store.cfg = &GlobalStorageConfig{ + AllowUnsanitizedSvgUpload: true, + } return store, mockStorage, storageName } diff --git a/pkg/services/store/storage_disk.go b/pkg/services/store/storage_disk.go index c7a523161ad..7ac33428a1d 100644 --- a/pkg/services/store/storage_disk.go +++ b/pkg/services/store/storage_disk.go @@ -12,23 +12,22 @@ import ( const rootStorageTypeDisk = "disk" -type rootStorageDisk struct { - baseStorageRuntime +var _ storageRuntime = &rootStorageDisk{} +type rootStorageDisk struct { settings *StorageLocalDiskConfig + meta RootStorageMeta + store filestorage.FileStorage } -func newDiskStorage(scfg RootStorageConfig) *rootStorageDisk { +func newDiskStorage(meta RootStorageMeta, scfg RootStorageConfig) *rootStorageDisk { cfg := scfg.Disk if cfg == nil { cfg = &StorageLocalDiskConfig{} scfg.Disk = cfg } scfg.Type = rootStorageTypeDisk - - meta := RootStorageMeta{ - Config: scfg, - } + meta.Config = scfg if scfg.Prefix == "" { meta.Notice = append(meta.Notice, data.Notice{ Severity: data.NoticeSeverityError, @@ -42,7 +41,9 @@ func newDiskStorage(scfg RootStorageConfig) *rootStorageDisk { }) } - s := &rootStorageDisk{} + s := &rootStorageDisk{ + settings: cfg, + } if meta.Notice == nil { path := fmt.Sprintf("file://%s", cfg.Path) @@ -63,10 +64,17 @@ func newDiskStorage(scfg RootStorageConfig) *rootStorageDisk { } s.meta = meta - s.settings = cfg return s } +func (s *rootStorageDisk) Meta() RootStorageMeta { + return s.meta +} + +func (s *rootStorageDisk) Store() filestorage.FileStorage { + return s.store +} + func (s *rootStorageDisk) Sync() error { return nil // already in sync } diff --git a/pkg/services/store/storage_git.go b/pkg/services/store/storage_git.go new file mode 100644 index 00000000000..3f62e398308 --- /dev/null +++ b/pkg/services/store/storage_git.go @@ -0,0 +1,387 @@ +package store + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/object" + + "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/grafana/grafana/pkg/infra/filestorage" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/setting" + "gocloud.dev/blob" +) + +const rootStorageTypeGit = "git" + +var _ storageRuntime = &rootStorageGit{} + +type rootStorageGit struct { + settings *StorageGitConfig + repo *git.Repository + root string // repostitory root + + github *githubHelper + meta RootStorageMeta + store filestorage.FileStorage +} + +func newGitStorage(meta RootStorageMeta, scfg RootStorageConfig, localWorkCache string) *rootStorageGit { + cfg := scfg.Git + if cfg == nil { + cfg = &StorageGitConfig{} + } + scfg.Type = rootStorageTypeGit + scfg.GCS = nil + scfg.SQL = nil + scfg.S3 = nil + scfg.Git = cfg + + meta.Config = scfg + if scfg.Prefix == "" { + meta.Notice = append(meta.Notice, data.Notice{ + Severity: data.NoticeSeverityError, + Text: "Missing prefix", + }) + } + if cfg.Remote == "" { + meta.Notice = append(meta.Notice, data.Notice{ + Severity: data.NoticeSeverityError, + Text: "Missing remote path configuration", + }) + } + + if len(localWorkCache) < 2 { + meta.Notice = append(meta.Notice, data.Notice{ + Severity: data.NoticeSeverityError, + Text: "Invalid local root folder", + }) + } + + s := &rootStorageGit{ + settings: cfg, + } + if meta.Notice == nil { + err := os.MkdirAll(localWorkCache, 0750) + if err != nil { + meta.Notice = append(meta.Notice, data.Notice{ + Severity: data.NoticeSeverityError, + Text: err.Error(), + }) + } + } + + if scfg.Disabled { + meta.Notice = append(meta.Notice, data.Notice{ + Severity: data.NoticeSeverityWarning, + Text: "folder is disabled (in configuration)", + }) + } else if setting.Env == setting.Prod { + meta.Notice = append(meta.Notice, data.Notice{ + Severity: data.NoticeSeverityError, + Text: "git is only supported in dev mode (for now)", + }) + } + + if meta.Notice == nil { + repo, err := git.PlainOpen(localWorkCache) + if errors.Is(err, git.ErrRepositoryNotExists) { + repo, err = git.PlainClone(localWorkCache, false, &git.CloneOptions{ + URL: cfg.Remote, + Progress: os.Stdout, + //Depth: 1, + //SingleBranch: true, + }) + } + + if err != nil { + meta.Notice = append(meta.Notice, data.Notice{ + Severity: data.NoticeSeverityError, + Text: err.Error(), + }) + } + + if err == nil { + p := localWorkCache + if cfg.Root != "" { + p = filepath.Join(p, cfg.Root) + } + + path := fmt.Sprintf("file://%s", p) + bucket, err := blob.OpenBucket(context.Background(), path) + if err != nil { + grafanaStorageLogger.Warn("error loading storage", "prefix", scfg.Prefix, "err", err) + meta.Notice = append(meta.Notice, data.Notice{ + Severity: data.NoticeSeverityError, + Text: "Failed to initialize storage", + }) + } else { + s.store = filestorage.NewCdkBlobStorage( + grafanaStorageLogger, + bucket, "", nil) + + meta.Ready = true // exists! + s.root = p + + token := cfg.AccessToken + if strings.HasPrefix(token, "$") { + token = os.Getenv(token[1:]) + if token == "" { + meta.Notice = append(meta.Notice, data.Notice{ + Severity: data.NoticeSeverityError, + Text: "Unable to find token environment variable: " + cfg.AccessToken, + }) + } + } + + if token != "" { + s.github, err = newGithubHelper(context.Background(), cfg.Remote, token) + if err != nil { + meta.Notice = append(meta.Notice, data.Notice{ + Severity: data.NoticeSeverityError, + Text: "error creating github client: " + err.Error(), + }) + s.github = nil + } else { + ghrepo, _, err := s.github.getRepo(context.Background()) + if err != nil { + meta.Notice = append(meta.Notice, data.Notice{ + Severity: data.NoticeSeverityError, + Text: err.Error(), + }) + s.github = nil + } else { + grafanaStorageLogger.Info("default branch", "branch", *ghrepo.DefaultBranch) + } + } + } + } + } + s.repo = repo + + // Try pulling after init + if s.repo != nil && !scfg.Disabled { + err = s.Sync() + if err != nil { + meta.Notice = append(meta.Notice, data.Notice{ + Severity: data.NoticeSeverityError, + Text: "unable to pull: " + err.Error(), + }) + } else if cfg.PullInterval != "" { + t, err := time.ParseDuration(cfg.PullInterval) + if err != nil { + meta.Notice = append(meta.Notice, data.Notice{ + Severity: data.NoticeSeverityError, + Text: "Invalid pull interval " + cfg.PullInterval, + }) + } else { + ticker := time.NewTicker(t) + go func() { + for range ticker.C { + grafanaStorageLogger.Info("try git pull", "branch", s.settings.Remote) + err = s.Sync() + if err != nil { + grafanaStorageLogger.Info("error pulling", "error", err) + } + } + }() + } + } + } + } + + s.meta = meta + return s +} + +func (s *rootStorageGit) Meta() RootStorageMeta { + return s.meta +} + +func (s *rootStorageGit) Store() filestorage.FileStorage { + return s.store +} + +func (s *rootStorageGit) Pull() error { + w, err := s.repo.Worktree() + if err != nil { + return err + } + + err = w.Pull(&git.PullOptions{ + // Depth: 1, + //SingleBranch: true, + }) + if err != nil { + return err + } + return nil +} + +func (s *rootStorageGit) Write(ctx context.Context, cmd *WriteValueRequest) (*WriteValueResponse, error) { + if s.github == nil { + return nil, fmt.Errorf("github client not initialized") + } + // Write to the correct subfolder + if s.settings.Root != "" { + cmd.Path = s.settings.Root + cmd.Path + } + + if cmd.Workflow == WriteValueWorkflow_PR { + prcmd := makePRCommand{ + baseBranch: s.settings.Branch, + headBranch: fmt.Sprintf("grafana_ui_%d", time.Now().UnixMilli()), + title: cmd.Title, + body: cmd.Message, + } + res := &WriteValueResponse{ + Branch: prcmd.headBranch, + } + + ref, _, err := s.github.createRef(ctx, prcmd.baseBranch, prcmd.headBranch) + if err != nil { + res.Code = 500 + res.Message = "unable to create branch" + return res, nil + } + + err = s.github.pushCommit(ctx, ref, cmd) + if err != nil { + res.Code = 500 + res.Message = fmt.Sprintf("error creating commit: %s", err.Error()) + return res, nil + } + + if prcmd.title == "" { + prcmd.title = "Dashboard save: " + time.Now().String() + } + if prcmd.body == "" { + prcmd.body = "Dashboard save: " + time.Now().String() + } + + pr, _, err := s.github.createPR(ctx, prcmd) + if err != nil { + res.Code = 500 + res.Message = "error creating PR: " + err.Error() + return res, nil + } + + res.Code = 200 + res.URL = pr.GetHTMLURL() + res.Pending = true + res.Hash = *ref.Object.SHA + res.Branch = prcmd.headBranch + return res, nil + } + + // Push to remote branch (save) + if cmd.Workflow == WriteValueWorkflow_Push || true { + res := &WriteValueResponse{ + Branch: s.settings.Branch, + } + ref, _, err := s.github.getRef(ctx, s.settings.Branch) + if err != nil { + res.Code = 500 + res.Message = "unable to create branch" + return res, nil + } + err = s.github.pushCommit(ctx, ref, cmd) + if err != nil { + res.Code = 500 + res.Message = "error creating commit" + return res, nil + } + ref, _, _ = s.github.getRef(ctx, s.settings.Branch) + if ref != nil { + res.Hash = *ref.Object.SHA + res.URL = ref.GetURL() + } + + err = s.Pull() + if err != nil { + res.Message = "error pulling: " + err.Error() + } + + res.Code = 200 + return res, nil + } + + rel := cmd.Path + if s.meta.Config.Git.Root != "" { + rel = filepath.Join(s.meta.Config.Git.Root, cmd.Path) + } + + fpath := filepath.Join(s.root, rel) + err := os.WriteFile(fpath, cmd.Body, 0644) + if err != nil { + return nil, err + } + + w, err := s.repo.Worktree() + if err != nil { + return nil, err + } + + // The file we just wrote + _, err = w.Add(rel) + if err != nil { + return nil, err + } + + msg := cmd.Message + if msg == "" { + msg = "changes from grafana ui" + } + user := cmd.User + if user == nil { + user = &models.SignedInUser{} + } + + hash, err := w.Commit(msg, &git.CommitOptions{ + Author: &object.Signature{ + Name: firstRealString(user.Name, user.Login, user.Email, "?"), + Email: firstRealString(user.Email, user.Login, user.Name, "?"), + When: time.Now(), + }, + }) + if err != nil { + return nil, err + } + + grafanaStorageLogger.Info("made commit", "hash", hash) + // err = s.repo.Push(&git.PushOptions{ + // InsecureSkipTLS: true, + // }) + + return &WriteValueResponse{ + Hash: hash.String(), + Message: "made commit", + }, nil +} + +func (s *rootStorageGit) Sync() error { + grafanaStorageLogger.Info("GIT PULL", "remote", s.settings.Remote) + err := s.Pull() + if err != nil { + if err.Error() == "already up-to-date" { + return nil + } + } + return err +} + +func firstRealString(vals ...string) string { + for _, v := range vals { + if v != "" { + return v + } + } + return "?" +} diff --git a/pkg/services/store/storage_sql.go b/pkg/services/store/storage_sql.go index ecfb3652192..8b3da04a3a2 100644 --- a/pkg/services/store/storage_sql.go +++ b/pkg/services/store/storage_sql.go @@ -13,10 +13,12 @@ import ( const rootStorageTypeSQL = "sql" -type rootStorageSQL struct { - baseStorageRuntime +var _ storageRuntime = &rootStorageSQL{} +type rootStorageSQL struct { settings *StorageSQLConfig + meta RootStorageMeta + store filestorage.FileStorage } // getDbRootFolder creates a DB path prefix for a given storage name and orgId. @@ -28,19 +30,17 @@ func getDbStoragePathPrefix(orgId int64, storageName string) string { return filestorage.Join(fmt.Sprintf("%d", orgId), storageName+filestorage.Delimiter) } -func newSQLStorage(prefix string, name string, descr string, cfg *StorageSQLConfig, sql *sqlstore.SQLStore, orgId int64) *rootStorageSQL { +func newSQLStorage(meta RootStorageMeta, prefix string, name string, descr string, cfg *StorageSQLConfig, sql *sqlstore.SQLStore, orgId int64) *rootStorageSQL { if cfg == nil { cfg = &StorageSQLConfig{} } - meta := RootStorageMeta{ - Config: RootStorageConfig{ - Type: rootStorageTypeSQL, - Prefix: prefix, - Name: name, - Description: descr, - SQL: cfg, - }, + meta.Config = RootStorageConfig{ + Type: rootStorageTypeSQL, + Prefix: prefix, + Name: name, + Description: descr, + SQL: cfg, } if prefix == "" { @@ -78,6 +78,14 @@ func (s *rootStorageSQL) Write(ctx context.Context, cmd *WriteValueRequest) (*Wr return &WriteValueResponse{Code: 200}, nil } +func (s *rootStorageSQL) Meta() RootStorageMeta { + return s.meta +} + +func (s *rootStorageSQL) Store() filestorage.FileStorage { + return s.store +} + func (s *rootStorageSQL) Sync() error { return nil // already in sync } diff --git a/pkg/services/store/tree.go b/pkg/services/store/tree.go index a28a7689410..263a4ffe929 100644 --- a/pkg/services/store/tree.go +++ b/pkg/services/store/tree.go @@ -2,6 +2,7 @@ package store import ( "context" + "fmt" "sync" "github.com/grafana/grafana-plugin-sdk-go/data" @@ -77,12 +78,15 @@ func (t *nestedTree) GetFile(ctx context.Context, orgId int64, path string) (*fi if path == "" { return nil, nil // not found } - root, path := t.getRoot(orgId, path) if root == nil { return nil, nil // not found (or not ready) } - return root.Store().Get(ctx, path) + store := root.Store() + if store == nil { + return nil, fmt.Errorf("store not ready") + } + return store.Get(ctx, path) } func (t *nestedTree) ListFolder(ctx context.Context, orgId int64, path string, accessFilter filestorage.PathFilter) (*StorageListFrame, error) { @@ -98,26 +102,17 @@ func (t *nestedTree) ListFolder(ctx context.Context, orgId int64, path string, a names := data.NewFieldFromFieldType(data.FieldTypeString, count) title := data.NewFieldFromFieldType(data.FieldTypeString, count) descr := data.NewFieldFromFieldType(data.FieldTypeString, count) - types := data.NewFieldFromFieldType(data.FieldTypeString, count) - readOnly := data.NewFieldFromFieldType(data.FieldTypeBool, count) - builtIn := data.NewFieldFromFieldType(data.FieldTypeBool, count) mtype := data.NewFieldFromFieldType(data.FieldTypeString, count) title.Name = titleListFrameField names.Name = nameListFrameField descr.Name = descriptionListFrameField mtype.Name = mediaTypeListFrameField - types.Name = storageTypeListFrameField - readOnly.Name = readOnlyListFrameField - builtIn.Name = builtInListFrameField for _, f := range t.rootsByOrgId[ac.GlobalOrgID] { meta := f.Meta() names.Set(idx, meta.Config.Prefix) title.Set(idx, meta.Config.Name) descr.Set(idx, meta.Config.Description) mtype.Set(idx, "directory") - types.Set(idx, meta.Config.Type) - readOnly.Set(idx, meta.ReadOnly) - builtIn.Set(idx, meta.Builtin) idx++ } if orgId != ac.GlobalOrgID { @@ -127,14 +122,11 @@ func (t *nestedTree) ListFolder(ctx context.Context, orgId int64, path string, a title.Set(idx, meta.Config.Name) descr.Set(idx, meta.Config.Description) mtype.Set(idx, "directory") - types.Set(idx, meta.Config.Type) - readOnly.Set(idx, meta.ReadOnly) - builtIn.Set(idx, meta.Builtin) idx++ } } - frame := data.NewFrame("", names, title, descr, mtype, types, readOnly, builtIn) + frame := data.NewFrame("", names, title, descr, mtype) frame.SetMeta(&data.FrameMeta{ Type: data.FrameTypeDirectoryListing, }) @@ -146,7 +138,12 @@ func (t *nestedTree) ListFolder(ctx context.Context, orgId int64, path string, a return nil, nil // not found (or not ready) } - listResponse, err := root.Store().List(ctx, path, nil, &filestorage.ListOptions{ + store := root.Store() + if store == nil { + return nil, fmt.Errorf("store not ready") + } + + listResponse, err := store.List(ctx, path, nil, &filestorage.ListOptions{ Recursive: false, WithFolders: true, WithFiles: true, diff --git a/pkg/services/store/types.go b/pkg/services/store/types.go index 20c345fd1f3..973d8982db7 100644 --- a/pkg/services/store/types.go +++ b/pkg/services/store/types.go @@ -9,13 +9,22 @@ import ( "github.com/grafana/grafana/pkg/models" ) +type WriteValueWorkflow = string + +var ( + WriteValueWorkflow_Save WriteValueWorkflow = "save" // or empty + WriteValueWorkflow_PR WriteValueWorkflow = "pr" + WriteValueWorkflow_Push WriteValueWorkflow = "push" +) + type WriteValueRequest struct { - Path string - User *models.SignedInUser - Body json.RawMessage `json:"body,omitempty"` - Message string `json:"message,omitempty"` - Title string `json:"title,omitempty"` // For PRs - Action string `json:"action,omitempty"` // pr | save + User *models.SignedInUser + Path string // added from URL + EntityType EntityType `json:"kind,omitempty"` // for now only dashboard + Body json.RawMessage `json:"body,omitempty"` + Message string `json:"message,omitempty"` + Title string `json:"title,omitempty"` // For PRs + Workflow WriteValueWorkflow `json:"workflow,omitempty"` // save | pr | push } type WriteValueResponse struct { @@ -48,40 +57,6 @@ type storageRuntime interface { Write(ctx context.Context, cmd *WriteValueRequest) (*WriteValueResponse, error) } -type baseStorageRuntime struct { - meta RootStorageMeta - store filestorage.FileStorage -} - -func (t *baseStorageRuntime) Meta() RootStorageMeta { - return t.meta -} - -func (t *baseStorageRuntime) Store() filestorage.FileStorage { - return t.store -} - -func (t *baseStorageRuntime) Sync() error { - return nil -} - -func (t *baseStorageRuntime) Write(ctx context.Context, cmd *WriteValueRequest) (*WriteValueResponse, error) { - return &WriteValueResponse{ - Code: 500, - Message: "unsupportted operation (base)", - }, nil -} - -func (t *baseStorageRuntime) setReadOnly(val bool) *baseStorageRuntime { - t.meta.ReadOnly = val - return t -} - -func (t *baseStorageRuntime) setBuiltin(val bool) *baseStorageRuntime { - t.meta.Builtin = val - return t -} - type RootStorageMeta struct { ReadOnly bool `json:"editable,omitempty"` Builtin bool `json:"builtin,omitempty"` @@ -100,9 +75,6 @@ const ( nameListFrameField = "name" descriptionListFrameField = "description" mediaTypeListFrameField = "mediaType" - storageTypeListFrameField = "storageType" - readOnlyListFrameField = "readOnly" - builtInListFrameField = "builtIn" sizeListFrameField = "size" ) diff --git a/public/app/features/dashboard/components/SaveDashboard/SaveDashboardDrawer.tsx b/public/app/features/dashboard/components/SaveDashboard/SaveDashboardDrawer.tsx index 05f0c2abad0..01fd4680c12 100644 --- a/public/app/features/dashboard/components/SaveDashboard/SaveDashboardDrawer.tsx +++ b/public/app/features/dashboard/components/SaveDashboard/SaveDashboardDrawer.tsx @@ -1,6 +1,7 @@ import React, { useMemo, useState } from 'react'; import { useAsync } from 'react-use'; +import { config } from '@grafana/runtime'; import { Drawer, Spinner, Tab, TabsBar } from '@grafana/ui'; import { backendSrv } from 'app/core/services/backend_srv'; @@ -11,14 +12,16 @@ import { SaveDashboardErrorProxy } from './SaveDashboardErrorProxy'; import { SaveDashboardAsForm } from './forms/SaveDashboardAsForm'; import { SaveDashboardForm } from './forms/SaveDashboardForm'; import { SaveProvisionedDashboardForm } from './forms/SaveProvisionedDashboardForm'; +import { SaveToStorageForm } from './forms/SaveToStorageForm'; import { SaveDashboardData, SaveDashboardModalProps, SaveDashboardOptions } from './types'; import { useDashboardSave } from './useDashboardSave'; export const SaveDashboardDrawer = ({ dashboard, onDismiss, onSaveSuccess, isCopy }: SaveDashboardModalProps) => { const [options, setOptions] = useState({}); - const isProvisioned = dashboard.meta.provisioned; - const isNew = dashboard.version === 0; + const isFromStorage = config.featureToggles.dashboardsFromStorage && dashboard.uid.indexOf('/') > 0; + const isProvisioned = dashboard.meta.provisioned && !isFromStorage; + const isNew = dashboard.version === 0 && !isFromStorage; const previous = useAsync(async () => { if (isNew) { @@ -78,6 +81,22 @@ export const SaveDashboardDrawer = ({ dashboard, onDismiss, onSaveSuccess, isCop ); } + if (isFromStorage) { + return ( + + ); + } + if (isNew || isCopy) { return ( void; @@ -28,7 +28,7 @@ export const SaveDashboardForm = ({ onCancel, onSuccess, onOptionsChange, -}: Props) => { +}: SaveProps) => { const hasTimeChanged = useMemo(() => dashboard.hasTimeChanged(), [dashboard]); const hasVariableChanged = useMemo(() => dashboard.hasVariableValuesChanged(), [dashboard]); diff --git a/public/app/features/dashboard/components/SaveDashboard/forms/SaveToStorageForm.tsx b/public/app/features/dashboard/components/SaveDashboard/forms/SaveToStorageForm.tsx new file mode 100644 index 00000000000..de34b050d69 --- /dev/null +++ b/public/app/features/dashboard/components/SaveDashboard/forms/SaveToStorageForm.tsx @@ -0,0 +1,222 @@ +import React, { useMemo, useState } from 'react'; +import { useAsync } from 'react-use'; + +import { selectors } from '@grafana/e2e-selectors'; +import { locationService } from '@grafana/runtime'; +import { + Button, + Checkbox, + Field, + Form, + HorizontalGroup, + Input, + RadioButtonGroup, + Spinner, + Stack, + TextArea, +} from '@grafana/ui'; +import { getGrafanaStorage } from 'app/features/storage/storage'; +import { ItemOptions, WorkflowID, WriteValueResponse } from 'app/features/storage/types'; + +import { SaveProps } from './SaveDashboardForm'; + +interface FormDTO { + title?: string; + message: string; +} + +interface Props extends SaveProps { + isNew?: boolean; + isCopy?: boolean; +} + +export function SaveToStorageForm(props: Props) { + const { dashboard, saveModel, onSubmit, onCancel, onSuccess, onOptionsChange, isNew, isCopy } = props; + const hasTimeChanged = useMemo(() => dashboard.hasTimeChanged(), [dashboard]); + const hasVariableChanged = useMemo(() => dashboard.hasVariableValuesChanged(), [dashboard]); + const [saving, setSaving] = useState(false); + const [response, setResponse] = useState(); + const [path, setPath] = useState(dashboard.uid); + const [workflow, setWorkflow] = useState(WorkflowID.Save); + const saveText = useMemo(() => { + switch (workflow) { + case WorkflowID.PR: + return 'Create PR'; + case WorkflowID.Push: + return 'Push'; + } + console.log('???', workflow); + return 'Save'; + }, [workflow]); + + const item = useAsync(async () => { + const opts = await getGrafanaStorage().getOptions(dashboard.uid); + setWorkflow(opts.workflows[0]?.value ?? WorkflowID.Save); + return opts; + }, [dashboard.uid]); + + if (item.error) { + return
Error loading workflows
; + } + + if (item.loading || !item.value) { + return ; + } + + if (response) { + return ( +
+ {response.url && ( +
+

View pull request

+ {response.url} +
+ )} + + 
{JSON.stringify(response)}
+ + + + +
+ ); + } + + let options = props.options; + const workflows = item.value?.workflows ?? []; + const canSave = saveModel.hasChanges || isNew || isCopy; + + return ( +
{ + if (!onSubmit) { + return; + } + setSaving(true); + + let uid = saveModel.clone.uid; + if (isNew || isCopy) { + uid = path; + if (!uid.endsWith('-dash.json')) { + uid += '-dash.json'; + } + } + const rsp = await getGrafanaStorage().write(uid, { + body: saveModel.clone, + kind: 'dashboard', + title: data.title, + message: data.message, + workflow: workflow, + }); + + console.log('GOT', rsp); + if (rsp.code === 200) { + if (options.saveVariables) { + dashboard.resetOriginalVariables(); + } + if (options.saveTimerange) { + dashboard.resetOriginalTime(); + } + + if (!rsp.pending) { + // should close + onSuccess(); + + // Need to update the URL + if (isNew || isCopy) { + locationService.push(`/g/${uid}`); + } + } + } else { + setSaving(false); + } + setResponse(rsp); + }} + > + {({ register, errors }) => ( + + + {hasTimeChanged && ( + + onOptionsChange({ + ...options, + saveTimerange: !options.saveTimerange, + }) + } + label="Save current time range as dashboard default" + aria-label={selectors.pages.SaveDashboardModal.saveTimerange} + /> + )} + {hasVariableChanged && ( + + onOptionsChange({ + ...options, + saveVariables: !options.saveVariables, + }) + } + label="Save current variable values as dashboard default" + aria-label={selectors.pages.SaveDashboardModal.saveVariables} + /> + )} + + + {(isNew || isCopy) && ( + + setPath(v.currentTarget.value)} + /> + + )} + + {!isJustSave(item.value) && ( + + + + )} + + {workflow === WorkflowID.PR && ( + + + + )} + + +