diff --git a/pkg/services/export/commit_helper.go b/pkg/services/export/commit_helper.go index 7cdd4ef992c..221912c7738 100644 --- a/pkg/services/export/commit_helper.go +++ b/pkg/services/export/commit_helper.go @@ -14,6 +14,8 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/grafana/grafana/pkg/infra/db" + "github.com/grafana/grafana/pkg/services/store" + "github.com/grafana/grafana/pkg/services/user" ) type commitHelper struct { @@ -44,13 +46,17 @@ type commitOptions struct { comment string } -func (ch *commitHelper) initOrg(sql db.DB, orgID int64) error { +func (ch *commitHelper) initOrg(ctx context.Context, sql db.DB, orgID int64) error { return sql.WithDbSession(ch.ctx, func(sess *db.Session) error { + userprefix := "user" + if isPostgreSQL(sql) { + userprefix = `"user"` // postgres has special needs + } sess.Table("user"). - Join("inner", "org_user", "user.id = org_user.user_id"). - Cols("user.*", "org_user.role"). + Join("inner", "org_user", userprefix+`.id = org_user.user_id`). + Cols(userprefix+`.*`, "org_user.role"). Where("org_user.org_id = ?", orgID). - Asc("user.id") + Asc(userprefix + `.id`) rows := make([]*userInfo, 0) err := sess.Find(&rows) @@ -64,6 +70,14 @@ func (ch *commitHelper) initOrg(sql db.DB, orgID int64) error { } ch.users = lookup ch.orgID = orgID + + // Set an admin user with the + rowUser := &user.SignedInUser{ + Login: "", + OrgID: orgID, // gets filled in from each row + UserID: 0, + } + ch.ctx = store.ContextWithUser(context.Background(), rowUser) return err }) } @@ -140,19 +154,28 @@ func (ch *commitHelper) add(opts commitOptions) error { } type userInfo struct { - ID int64 `json:"-" xorm:"id"` + ID int64 `json:"-" db:"id"` Login string `json:"login"` Email string `json:"email"` Name string `json:"name"` Password string `json:"password"` Salt string `json:"salt"` + Company string `json:"company,omitempty"` + Rands string `json:"-"` Role string `json:"org_role"` // org role Theme string `json:"-"` // managed in preferences Created time.Time `json:"-"` // managed in git or external source Updated time.Time `json:"-"` // managed in git or external source - IsDisabled bool `json:"disabled" xorm:"is_disabled"` - IsServiceAccount bool `json:"serviceAccount" xorm:"is_service_account"` - LastSeenAt time.Time `json:"-" xorm:"last_seen_at"` + IsDisabled bool `json:"disabled" db:"is_disabled"` + IsServiceAccount bool `json:"serviceAccount" db:"is_service_account"` + LastSeenAt time.Time `json:"-" db:"last_seen_at"` + + // Added to make sqlx happy + Version int `json:"-"` + HelpFlags1 int `json:"-" db:"help_flags1"` + OrgID int64 `json:"-" db:"org_id"` + EmailVerified bool `json:"-" db:"email_verified"` + IsAdmin bool `json:"-" db:"is_admin"` } func (u *userInfo) getAuthor() object.Signature { diff --git a/pkg/services/export/export_auth.go b/pkg/services/export/export_auth.go index 18b324efe0b..d3315a6599c 100644 --- a/pkg/services/export/export_auth.go +++ b/pkg/services/export/export_auth.go @@ -3,7 +3,6 @@ package export import ( "path" "strconv" - "strings" "github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana-plugin-sdk-go/data/sqlutil" @@ -12,6 +11,8 @@ import ( ) func dumpAuthTables(helper *commitHelper, job *gitExportJob) error { + isMySQL := isMySQLEngine(job.sql) + return job.sql.WithDbSession(helper.ctx, func(sess *db.Session) error { commit := commitOptions{ comment: "auth tables dump", @@ -27,11 +28,11 @@ func dumpAuthTables(helper *commitHelper, job *gitExportJob) error { dump := []statsTables{ { table: "user", - sql: ` - SELECT user.*, org_user.role - FROM user - JOIN org_user ON user.id = org_user.user_id - WHERE org_user.org_id =` + strconv.FormatInt(helper.orgID, 10), + sql: removeQuotesFromQuery(` + SELECT "user".*, org_user.role + FROM "user" + JOIN org_user ON "user".id = org_user.user_id + WHERE org_user.org_id =`+strconv.FormatInt(helper.orgID, 10), isMySQL), converters: []sqlutil.Converter{{Dynamic: true}}, drop: []string{ "id", "version", @@ -74,7 +75,6 @@ func dumpAuthTables(helper *commitHelper, job *gitExportJob) error { WHERE org_user.org_id =` + strconv.FormatInt(helper.orgID, 10), }, {table: "team"}, - {table: "team_group"}, {table: "team_role"}, {table: "team_member"}, {table: "temp_user"}, @@ -99,7 +99,7 @@ func dumpAuthTables(helper *commitHelper, job *gitExportJob) error { rows, err := sess.DB().QueryContext(helper.ctx, auth.sql) if err != nil { - if strings.HasPrefix(err.Error(), "no such table") { + if isTableNotExistsError(err) { continue } return err diff --git a/pkg/services/export/export_dash.go b/pkg/services/export/export_dash.go index 59dec7a0c80..574c83d046f 100644 --- a/pkg/services/export/export_dash.go +++ b/pkg/services/export/export_dash.go @@ -12,6 +12,7 @@ import ( "github.com/grafana/grafana/pkg/infra/db" "github.com/grafana/grafana/pkg/infra/filestorage" + "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/store/kind/dashboard" ) @@ -30,7 +31,7 @@ func exportDashboards(helper *commitHelper, job *gitExportJob) error { return err } - rootDir := path.Join(helper.orgDir, "root") + rootDir := path.Join(helper.orgDir, models.ObjectStoreScopeDrive) folderStructure := commitOptions{ when: time.Now(), comment: "Exported folder structure", @@ -95,7 +96,7 @@ func exportDashboards(helper *commitHelper, job *gitExportJob) error { if row.IsFolder { continue } - fname := row.Slug + "-dash.json" + fname := row.Slug + "-dashboard.json" fpath, ok := folders[row.FolderID] if ok { fpath = path.Join(fpath, fname) diff --git a/pkg/services/export/export_dash_thumbs.go b/pkg/services/export/export_dash_thumbs.go index 22e0fdbaf2a..fa90dd8bc07 100644 --- a/pkg/services/export/export_dash_thumbs.go +++ b/pkg/services/export/export_dash_thumbs.go @@ -46,7 +46,7 @@ func exportDashboardThumbnails(helper *commitHelper, job *gitExportJob) error { err := sess.Find(&rows) if err != nil { - if strings.HasPrefix(err.Error(), "no such table") { + if isTableNotExistsError(err) { return nil } return err diff --git a/pkg/services/export/export_live.go b/pkg/services/export/export_live.go index 4564203475f..c8607353ab0 100644 --- a/pkg/services/export/export_live.go +++ b/pkg/services/export/export_live.go @@ -3,7 +3,6 @@ package export import ( "fmt" "path" - "strings" "time" "github.com/grafana/grafana/pkg/infra/db" @@ -26,7 +25,7 @@ func exportLive(helper *commitHelper, job *gitExportJob) error { err := sess.Find(&rows) if err != nil { - if strings.HasPrefix(err.Error(), "no such table") { + if isTableNotExistsError(err) { return nil } return err diff --git a/pkg/services/export/export_plugins.go b/pkg/services/export/export_plugins.go index 95398de0be2..37dde5ec378 100644 --- a/pkg/services/export/export_plugins.go +++ b/pkg/services/export/export_plugins.go @@ -4,7 +4,6 @@ import ( "encoding/json" "fmt" "path" - "strings" "time" "github.com/grafana/grafana/pkg/infra/db" @@ -29,7 +28,7 @@ func exportPlugins(helper *commitHelper, job *gitExportJob) error { err := sess.Find(&rows) if err != nil { - if strings.HasPrefix(err.Error(), "no such table") { + if isTableNotExistsError(err) { return nil } return err diff --git a/pkg/services/export/export_sys_playlists.go b/pkg/services/export/export_sys_playlists.go index 97578dadc83..5faf70414aa 100644 --- a/pkg/services/export/export_sys_playlists.go +++ b/pkg/services/export/export_sys_playlists.go @@ -5,6 +5,7 @@ import ( "path/filepath" "time" + "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/playlist" ) @@ -37,8 +38,12 @@ func exportSystemPlaylists(helper *commitHelper, job *gitExportJob) error { } gitcmd.body = append(gitcmd.body, commitBody{ - fpath: filepath.Join(helper.orgDir, "system", "playlists", fmt.Sprintf("%s-playlist.json", playlist.Uid)), - body: prettyJSON(playlist), + fpath: filepath.Join( + helper.orgDir, + models.ObjectStoreScopeEntity, + models.StandardKindPlaylist, + fmt.Sprintf("%s.json", playlist.Uid)), + body: prettyJSON(playlist), }) } diff --git a/pkg/services/export/export_usage.go b/pkg/services/export/export_usage.go index 2e7dc2c9da2..85c16e98552 100644 --- a/pkg/services/export/export_usage.go +++ b/pkg/services/export/export_usage.go @@ -3,7 +3,6 @@ package export import ( "path" "strconv" - "strings" "github.com/grafana/grafana-plugin-sdk-go/data/sqlutil" @@ -64,7 +63,7 @@ func exportUsage(helper *commitHelper, job *gitExportJob) error { for _, usage := range dump { rows, err := sess.DB().QueryContext(helper.ctx, usage.sql) if err != nil { - if strings.HasPrefix(err.Error(), "no such table") { + if isTableNotExistsError(err) { continue } return err diff --git a/pkg/services/export/git_export_job.go b/pkg/services/export/git_export_job.go index e4ba735ff21..ab8822ae91c 100644 --- a/pkg/services/export/git_export_job.go +++ b/pkg/services/export/git_export_job.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "path" + "runtime/debug" "sync" "time" @@ -37,7 +38,7 @@ type gitExportJob struct { helper *commitHelper } -func startGitExportJob(cfg ExportConfig, sql db.DB, +func startGitExportJob(ctx context.Context, cfg ExportConfig, sql db.DB, dashboardsnapshotsService dashboardsnapshots.Service, rootDir string, orgID int64, broadcaster statusBroadcaster, playlistService playlist.Service, orgService org.Service, datasourceService datasources.DataSourceService) (Job, error) { @@ -60,7 +61,7 @@ func startGitExportJob(cfg ExportConfig, sql db.DB, } broadcaster(job.status) - go job.start() + go job.start(ctx) return job, nil } @@ -83,7 +84,7 @@ func (e *gitExportJob) requestStop() { } // Utility function to export dashboards -func (e *gitExportJob) start() { +func (e *gitExportJob) start(ctx context.Context) { defer func() { e.logger.Info("Finished git export job") e.statusMu.Lock() @@ -91,6 +92,7 @@ func (e *gitExportJob) start() { s := e.status if err := recover(); err != nil { e.logger.Error("export panic", "error", err) + e.logger.Error("trace", "error", string(debug.Stack())) s.Status = fmt.Sprintf("ERROR: %v", err) } // Make sure it finishes OK @@ -106,7 +108,7 @@ func (e *gitExportJob) start() { e.broadcaster(s) }() - err := e.doExportWithHistory() + err := e.doExportWithHistory(ctx) if err != nil { e.logger.Error("ERROR", "e", err) e.status.Status = "ERROR" @@ -115,7 +117,7 @@ func (e *gitExportJob) start() { } } -func (e *gitExportJob) doExportWithHistory() error { +func (e *gitExportJob) doExportWithHistory(ctx context.Context) error { r, err := git.PlainInit(e.rootDir, false) if err != nil { return err @@ -134,7 +136,7 @@ func (e *gitExportJob) doExportWithHistory() error { e.helper = &commitHelper{ repo: r, work: w, - ctx: context.Background(), + ctx: ctx, workDir: e.rootDir, orgDir: e.rootDir, broadcast: func(p string) { @@ -157,7 +159,7 @@ func (e *gitExportJob) doExportWithHistory() error { e.helper.orgDir = path.Join(e.rootDir, fmt.Sprintf("org_%d", org.ID)) e.status.Count["orgs"] += 1 } - err = e.helper.initOrg(e.sql, org.ID) + err = e.helper.initOrg(ctx, e.sql, org.ID) if err != nil { return err } diff --git a/pkg/services/export/object_store.go b/pkg/services/export/object_store.go index 990c4aca0c3..1b902e64589 100644 --- a/pkg/services/export/object_store.go +++ b/pkg/services/export/object_store.go @@ -28,7 +28,7 @@ type objectStoreJob struct { cfg ExportConfig broadcaster statusBroadcaster stopRequested bool - user *user.SignedInUser + ctx context.Context sess *session.SessionDB playlistService playlist.Service @@ -36,7 +36,7 @@ type objectStoreJob struct { dashboardsnapshots dashboardsnapshots.Service } -func startObjectStoreJob(user *user.SignedInUser, +func startObjectStoreJob(ctx context.Context, cfg ExportConfig, broadcaster statusBroadcaster, db db.DB, @@ -47,7 +47,7 @@ func startObjectStoreJob(user *user.SignedInUser, job := &objectStoreJob{ logger: log.New("export_to_object_store_job"), cfg: cfg, - user: user, + ctx: ctx, broadcaster: broadcaster, status: ExportStatus{ Running: true, @@ -63,7 +63,7 @@ func startObjectStoreJob(user *user.SignedInUser, } broadcaster(job.status) - go job.start() + go job.start(ctx) return job, nil } @@ -71,7 +71,7 @@ func (e *objectStoreJob) requestStop() { e.stopRequested = true } -func (e *objectStoreJob) start() { +func (e *objectStoreJob) start(ctx context.Context) { defer func() { e.logger.Info("Finished dummy export job") @@ -97,11 +97,11 @@ func (e *objectStoreJob) start() { e.logger.Info("Starting dummy export job") // Select all dashboards rowUser := &user.SignedInUser{ - Login: "?", + Login: "", OrgID: 0, // gets filled in from each row UserID: 0, } - ctx := store.ContextWithUser(context.Background(), rowUser) + ctx = store.ContextWithUser(ctx, rowUser) what := models.StandardKindDashboard e.status.Count[what] = 0 @@ -190,10 +190,12 @@ func (e *objectStoreJob) start() { orgIDs := []int64{1} what = "snapshot" for _, orgId := range orgIDs { + rowUser.OrgID = orgId + rowUser.UserID = 1 cmd := &dashboardsnapshots.GetDashboardSnapshotsQuery{ OrgId: orgId, Limit: 500000, - SignedInUser: e.user, + SignedInUser: rowUser, } err := e.dashboardsnapshots.SearchDashboardSnapshots(ctx, cmd) diff --git a/pkg/services/export/service.go b/pkg/services/export/service.go index fe85f03df0c..9867dfc279a 100644 --- a/pkg/services/export/service.go +++ b/pkg/services/export/service.go @@ -1,6 +1,7 @@ package export import ( + "context" "encoding/json" "fmt" "net/http" @@ -224,7 +225,7 @@ func (ex *StandardExport) HandleRequestExport(c *models.ReqContext) response.Res return response.Error(http.StatusLocked, "export already running", nil) } - user := store.UserFromContext(c.Req.Context()) + ctx := store.ContextWithUser(context.Background(), c.SignedInUser) var job Job broadcast := func(s ExportStatus) { ex.broadcastStatus(c.OrgID, s) @@ -233,13 +234,13 @@ func (ex *StandardExport) HandleRequestExport(c *models.ReqContext) response.Res case "dummy": job, err = startDummyExportJob(cfg, broadcast) case "objectStore": - job, err = startObjectStoreJob(user, cfg, broadcast, ex.db, ex.playlistService, ex.store, ex.dashboardsnapshotsService) + job, err = startObjectStoreJob(ctx, cfg, broadcast, ex.db, ex.playlistService, ex.store, ex.dashboardsnapshotsService) case "git": dir := filepath.Join(ex.dataDir, "export_git", fmt.Sprintf("git_%d", time.Now().Unix())) if err := os.MkdirAll(dir, os.ModePerm); err != nil { return response.Error(http.StatusBadRequest, "Error creating export folder", nil) } - job, err = startGitExportJob(cfg, ex.db, ex.dashboardsnapshotsService, dir, c.OrgID, broadcast, ex.playlistService, ex.orgService, ex.datasourceService) + job, err = startGitExportJob(ctx, cfg, ex.db, ex.dashboardsnapshotsService, dir, c.OrgID, broadcast, ex.playlistService, ex.orgService, ex.datasourceService) default: return response.Error(http.StatusBadRequest, "Unsupported job format", nil) } diff --git a/pkg/services/export/utils.go b/pkg/services/export/utils.go new file mode 100644 index 00000000000..567264c9c39 --- /dev/null +++ b/pkg/services/export/utils.go @@ -0,0 +1,29 @@ +package export + +import ( + "strings" + + "github.com/grafana/grafana/pkg/infra/db" +) + +func isTableNotExistsError(err error) bool { + txt := err.Error() + return strings.HasPrefix(txt, "no such table") || // SQLite + strings.HasSuffix(txt, " does not exist") || // PostgreSQL + strings.HasSuffix(txt, " doesn't exist") // MySQL +} + +func removeQuotesFromQuery(query string, remove bool) string { + if remove { + return strings.ReplaceAll(query, `"`, "") + } + return query +} + +func isMySQLEngine(sql db.DB) bool { + return sql.GetDBType() == "mysql" +} + +func isPostgreSQL(sql db.DB) bool { + return sql.GetDBType() == "postgres" +} diff --git a/pkg/services/store/object/sqlstash/sql_storage_server.go b/pkg/services/store/object/sqlstash/sql_storage_server.go index 4a10c75479e..04a78396ef7 100644 --- a/pkg/services/store/object/sqlstash/sql_storage_server.go +++ b/pkg/services/store/object/sqlstash/sql_storage_server.go @@ -552,6 +552,10 @@ func (s *sqlObjectServer) History(ctx context.Context, r *object.ObjectHistoryRe func (s *sqlObjectServer) Search(ctx context.Context, r *object.ObjectSearchRequest) (*object.ObjectSearchResponse, error) { user := store.UserFromContext(ctx) + if user == nil { + return nil, fmt.Errorf("missing user in context") + } + if r.NextPageToken != "" || len(r.Sort) > 0 || len(r.Labels) > 0 { return nil, fmt.Errorf("not yet supported") }