public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Export: save all dashboards to git (#48233)

Browse Source
This commit is contained in:
Ryan McKinley
2022-07-05 10:54:07 -07:00
committed by GitHub
parent 4a00c7ebde
commit 4fa606c600
17 changed files with 1365 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/export/commit_helper.go
+153
View File
@@ -0,0 +1,153 @@
package export
import (
"context"
"fmt"
"io/ioutil"
"os"
"path"
"strings"
"time"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/services/sqlstore"
jsoniter "github.com/json-iterator/go"
)
type commitHelper struct {
ctx context.Context
repo *git.Repository
work *git.Worktree
orgDir string // includes the orgID
workDir string // same as the worktree root
orgID int64
users map[int64]*userInfo
}
type commitBody struct {
fpath string // absolute
body []byte
frame *data.Frame
}
type commitOptions struct {
body []commitBody
when time.Time
userID int64
comment string
}
func (ch *commitHelper) initOrg(sql *sqlstore.SQLStore, orgID int64) error {
return sql.WithDbSession(ch.ctx, func(sess *sqlstore.DBSession) error {
sess.Table("user").
Join("inner", "org_user", "user.id = org_user.user_id").
Cols("user.*", "org_user.role").
Where("org_user.org_id = ?", orgID).
Asc("user.id")
rows := make([]*userInfo, 0)
err := sess.Find(&rows)
if err != nil {
return err
}
lookup := make(map[int64]*userInfo, len(rows))
for _, row := range rows {
lookup[row.ID] = row
}
ch.users = lookup
ch.orgID = orgID
return err
})
}
func (ch *commitHelper) add(opts commitOptions) error {
for _, b := range opts.body {
if !strings.HasPrefix(b.fpath, ch.orgDir) {
return fmt.Errorf("invalid path, must be within the root folder")
}
// make sure the parent exists
err := os.MkdirAll(path.Dir(b.fpath), 0750)
if err != nil {
return err
}
body := b.body
if b.frame != nil {
body, err = jsoniter.ConfigCompatibleWithStandardLibrary.MarshalIndent(b.frame, "", " ")
if err != nil {
return err
}
}
err = ioutil.WriteFile(b.fpath, body, 0644)
if err != nil {
return err
}
sub := b.fpath[len(ch.workDir)+1:]
_, err = ch.work.Add(sub)
if err != nil {
status, e2 := ch.work.Status()
if e2 != nil {
return fmt.Errorf("error adding: %s (invalud work status: %s)", sub, e2.Error())
}
fmt.Printf("STATUS: %+v\n", status)
return fmt.Errorf("unable to add file: %s (%d)", sub, len(b.body))
}
}
user, ok := ch.users[opts.userID]
if !ok {
user = &userInfo{
Name: "admin",
Email: "admin@unknown.org",
}
}
sig := user.getAuthor()
if opts.when.Unix() > 10 {
sig.When = opts.when
}
copts := &git.CommitOptions{
Author: &sig,
}
_, err := ch.work.Commit(opts.comment, copts)
return err
}
type userInfo struct {
ID int64 `json:"-" xorm:"id"`
Login string `json:"login"`
Email string `json:"email"`
Name string `json:"name"`
Password string `json:"password"`
Salt string `json:"salt"`
Role string `json:"org_role"` // org role
Theme string `json:"-"` // managed in preferences
Created time.Time `json:"-"` // managed in git or external source
Updated time.Time `json:"-"` // managed in git or external source
IsDisabled bool `json:"disabled" xorm:"is_disabled"`
IsServiceAccount bool `json:"serviceAccount" xorm:"is_service_account"`
LastSeenAt time.Time `json:"-" xorm:"last_seen_at"`
}
func (u *userInfo) getAuthor() object.Signature {
return object.Signature{
Name: firstRealStringX(u.Name, u.Login, u.Email, "?"),
Email: firstRealStringX(u.Email, u.Login, u.Name, "?"),
}
}
func firstRealStringX(vals ...string) string {
for _, v := range vals {
if v != "" {
return v
}
}
return "?"
}
pkg/services/export/dummy_job.go
-5
View File
@@ -1,7 +1,6 @@
package export
import (
"errors"
"fmt"
"math"
"math/rand"
@@ -23,10 +22,6 @@ type dummyExportJob struct {
}
func startDummyExportJob(cfg ExportConfig, broadcaster statusBroadcaster) (Job, error) {
if cfg.Format != "git" {
return nil, errors.New("only git format is supported")
}
job := &dummyExportJob{
logger: log.New("dummy_export_job"),
cfg: cfg,
pkg/services/export/export_anno.go
+129
View File
@@ -0,0 +1,129 @@
package export
import (
"encoding/json"
"fmt"
"path/filepath"
"time"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/services/sqlstore"
jsoniter "github.com/json-iterator/go"
)
func exportAnnotations(helper *commitHelper, job *gitExportJob) error {
return job.sql.WithDbSession(helper.ctx, func(sess *sqlstore.DBSession) error {
type annoResult struct {
ID int64 `xorm:"id"`
DashboardID int64 `xorm:"dashboard_id"`
PanelID int64 `xorm:"panel_id"`
UserID int64 `xorm:"user_id"`
Text string `xorm:"text"`
Epoch int64 `xorm:"epoch"`
EpochEnd int64 `xorm:"epoch_end"`
Created int64 `xorm:"created"` // not used
Tags string `xorm:"tags"` // JSON Array
}
type annoEvent struct {
PanelID int64 `json:"panel"`
Text string `json:"text"`
Epoch int64 `json:"epoch"` // dashboard/start+end is really the UID
EpochEnd int64 `json:"epoch_end,omitempty"`
Tags []string
}
rows := make([]*annoResult, 0)
sess.Table("annotation").
Where("org_id = ? AND alert_id = 0", helper.orgID).Asc("epoch")
err := sess.Find(&rows)
if err != nil {
return err
}
count := len(rows)
f_ID := data.NewFieldFromFieldType(data.FieldTypeInt64, count)
f_DashboardID := data.NewFieldFromFieldType(data.FieldTypeInt64, count)
f_PanelID := data.NewFieldFromFieldType(data.FieldTypeInt64, count)
f_Epoch := data.NewFieldFromFieldType(data.FieldTypeTime, count)
f_EpochEnd := data.NewFieldFromFieldType(data.FieldTypeNullableTime, count)
f_Text := data.NewFieldFromFieldType(data.FieldTypeString, count)
f_Tags := data.NewFieldFromFieldType(data.FieldTypeJSON, count)
f_ID.Name = "ID"
f_DashboardID.Name = "DashboardID"
f_PanelID.Name = "PanelID"
f_Epoch.Name = "Epoch"
f_EpochEnd.Name = "EpochEnd"
f_Text.Name = "Text"
f_Tags.Name = "Tags"
for id, row := range rows {
f_ID.Set(id, row.ID)
f_DashboardID.Set(id, row.DashboardID)
f_PanelID.Set(id, row.PanelID)
f_Epoch.Set(id, time.UnixMilli(row.Epoch))
if row.Epoch != row.EpochEnd {
f_EpochEnd.SetConcrete(id, time.UnixMilli(row.EpochEnd))
}
f_Text.Set(id, row.Text)
f_Tags.Set(id, json.RawMessage(row.Tags))
// Save a file for each
event := &annoEvent{
PanelID: row.PanelID,
Text: row.Text,
}
err = json.Unmarshal([]byte(row.Tags), &event.Tags)
if err != nil {
return err
}
fname := fmt.Sprintf("%d", row.Epoch)
if row.Epoch != row.EpochEnd {
fname += "-" + fmt.Sprintf("%d", row.EpochEnd)
}
err = helper.add(commitOptions{
body: []commitBody{
{
fpath: filepath.Join(helper.orgDir,
"annotations",
"dashboard",
fmt.Sprintf("id-%d", row.DashboardID),
fname+".json"),
body: prettyJSON(event),
},
},
when: time.UnixMilli(row.Epoch),
comment: fmt.Sprintf("Added annotation (%d)", row.ID),
userID: row.UserID,
})
if err != nil {
return err
}
}
frame := data.NewFrame("", f_ID, f_DashboardID, f_PanelID, f_Epoch, f_EpochEnd, f_Text, f_Tags)
js, err := jsoniter.ConfigCompatibleWithStandardLibrary.MarshalIndent(frame, "", " ")
if err != nil {
return err
}
err = helper.add(commitOptions{
body: []commitBody{
{
fpath: filepath.Join(helper.orgDir, "annotations", "annotations.json"),
body: js, // TODO, pretty?
},
},
when: time.Now(),
comment: "Exported annotations",
})
if err != nil {
return err
}
return err
})
}
pkg/services/export/export_auth.go
+73
View File
@@ -0,0 +1,73 @@
package export
import (
"fmt"
"path"
"github.com/grafana/grafana/pkg/services/sqlstore"
)
func dumpAuthTables(helper *commitHelper, job *gitExportJob) error {
return job.sql.WithDbSession(helper.ctx, func(sess *sqlstore.DBSession) error {
commit := commitOptions{
comment: "auth tables dump",
}
tables := []string{
"user", // joined with "org_user" to get the role
"user_role",
"builtin_role",
"api_key",
"team", "team_group", "team_role", "team_member",
"role",
"temp_user",
"user_auth_token", // no org_id... is it temporary?
"permission",
}
for _, table := range tables {
switch table {
case "permission":
sess.Table(table).
Join("left", "role", "permission.role_id = role.id").
Cols("permission.*").
Where("org_id = ?", helper.orgID).
Asc("permission.id")
case "user":
sess.Table(table).
Join("inner", "org_user", "user.id = org_user.user_id").
Cols("user.*", "org_user.role").
Where("org_user.org_id = ?", helper.orgID).
Asc("user.id")
case "user_auth_token":
sess.Table(table).
Join("inner", "org_user", "user_auth_token.id = org_user.user_id").
Cols("user_auth_token.*").
Where("org_user.org_id = ?", helper.orgID).
Asc("user_auth_token.id")
default:
sess.Table(table).Where("org_id = ?", helper.orgID).Asc("id")
}
raw, err := sess.QueryInterface()
if err != nil {
return fmt.Errorf("unable to read: %s // %s", table, err.Error())
}
if len(raw) < 1 {
continue // don't write empty files
}
frame, err := queryResultToDataFrame(raw, frameOpts{
skip: []string{"org_id", "version", "help_flags1", "theme"},
})
if err != nil {
return err
}
frame.Name = table
commit.body = append(commit.body, commitBody{
fpath: path.Join(helper.orgDir, "auth", "sql.dump", table+".json"),
frame: frame,
})
}
return helper.add(commit)
})
}
pkg/services/export/export_dashboards.go
+229
View File
@@ -0,0 +1,229 @@
package export
import (
"bytes"
"encoding/json"
"fmt"
"path"
"path/filepath"
"strings"
"time"
"github.com/google/uuid"
"github.com/grafana/grafana/pkg/infra/filestorage"
"github.com/grafana/grafana/pkg/services/searchV2/extract"
"github.com/grafana/grafana/pkg/services/sqlstore"
)
func exportDashboards(helper *commitHelper, job *gitExportJob, lookup dsLookup) error {
alias := make(map[string]string, 100)
ids := make(map[int64]string, 100)
folders := make(map[int64]string, 100)
// Should root files be at the root or in a subfolder called "general"?
if true {
folders[0] = "general"
}
rootDir := path.Join(helper.orgDir, "root")
folderStructure := commitOptions{
when: time.Now(),
comment: "Exported folder structure",
}
err := job.sql.WithDbSession(helper.ctx, func(sess *sqlstore.DBSession) error {
type dashDataQueryResult struct {
Id int64
UID string `xorm:"uid"`
IsFolder bool `xorm:"is_folder"`
FolderID int64 `xorm:"folder_id"`
Slug string `xorm:"slug"`
Data []byte
Created time.Time
Updated time.Time
}
rows := make([]*dashDataQueryResult, 0)
sess.Table("dashboard").
Where("org_id = ?", helper.orgID).
Cols("id", "is_folder", "folder_id", "data", "slug", "created", "updated", "uid")
err := sess.Find(&rows)
if err != nil {
return err
}
// Process all folders
for _, row := range rows {
if !row.IsFolder {
continue
}
dash, err := extract.ReadDashboard(bytes.NewReader(row.Data), lookup)
if err != nil {
return err
}
dash.UID = row.UID
slug := cleanFileName(dash.Title)
folder := map[string]string{
"title": dash.Title,
}
folderStructure.body = append(folderStructure.body, commitBody{
fpath: path.Join(rootDir, slug, "__folder.json"),
body: prettyJSON(folder),
})
alias[dash.UID] = slug
folders[row.Id] = slug
if row.Created.Before(folderStructure.when) {
folderStructure.when = row.Created
}
}
// Now process the dashboards in each folder
for _, row := range rows {
if row.IsFolder {
continue
}
fname := row.Slug + "-dash.json"
fpath, ok := folders[row.FolderID]
if ok {
fpath = path.Join(fpath, fname)
} else {
fpath = fname
}
alias[row.UID] = fpath
ids[row.Id] = fpath
}
return err
})
if err != nil {
return err
}
err = helper.add(folderStructure)
if err != nil {
return err
}
err = helper.add(commitOptions{
body: []commitBody{
{
fpath: filepath.Join(helper.orgDir, "root-alias.json"),
body: prettyJSON(alias),
},
{
fpath: filepath.Join(helper.orgDir, "root-ids.json"),
body: prettyJSON(ids),
},
},
when: folderStructure.when,
comment: "adding UID alias structure",
})
if err != nil {
return err
}
// Now walk the history
err = job.sql.WithDbSession(helper.ctx, func(sess *sqlstore.DBSession) error {
type dashVersionResult struct {
DashId int64 `xorm:"dashboard_id"`
Version int64 `xorm:"version"`
Created time.Time `xorm:"created"`
CreatedBy int64 `xorm:"created_by"`
Message string `xorm:"message"`
Data []byte
}
rows := make([]*dashVersionResult, 0, len(ids))
sess.Table("dashboard_version").
Join("INNER", "dashboard", "dashboard.id = dashboard_version.dashboard_id").
Where("org_id = ?", job.orgID).
Cols("dashboard_version.dashboard_id",
"dashboard_version.version",
"dashboard_version.created",
"dashboard_version.created_by",
"dashboard_version.message",
"dashboard_version.data").
Asc("dashboard_version.created")
err := sess.Find(&rows)
if err != nil {
return err
}
count := int64(0)
// Process all folders (only one level deep!!!)
for _, row := range rows {
fpath, ok := ids[row.DashId]
if !ok {
continue
}
msg := row.Message
if msg == "" {
msg = fmt.Sprintf("Version: %d", row.Version)
}
err = helper.add(commitOptions{
body: []commitBody{
{
fpath: filepath.Join(rootDir, fpath),
body: cleanDashboardJSON(row.Data),
},
},
userID: row.CreatedBy,
when: row.Created,
comment: msg,
})
if err != nil {
return err
}
count++
fmt.Printf("COMMIT: %d // %s (%d)\n", count, fpath, row.Version)
job.status.Current = count
job.status.Last = fpath
job.status.Changed = time.Now().UnixMilli()
job.broadcaster(job.status)
}
return nil
})
return err
}
func cleanDashboardJSON(data []byte) []byte {
var dash map[string]interface{}
err := json.Unmarshal(data, &dash)
if err != nil {
return nil
}
delete(dash, "id")
delete(dash, "uid")
delete(dash, "version")
clean, _ := json.MarshalIndent(dash, "", " ")
return clean
}
// replace any unsafe file name characters... TODO, but be a standard way to do this cleanly!!!
func cleanFileName(name string) string {
name = strings.ReplaceAll(name, "/", "-")
name = strings.ReplaceAll(name, "\\", "-")
name = strings.ReplaceAll(name, ":", "-")
if err := filestorage.ValidatePath(filestorage.Delimiter + name); err != nil {
randomName, _ := uuid.NewRandom()
return randomName.String()
}
return name
}
pkg/services/export/export_ds.go
+72
View File
@@ -0,0 +1,72 @@
package export
import (
"fmt"
"path/filepath"
"sort"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/searchV2/extract"
)
type dsLookup func(ref *extract.DataSourceRef) *extract.DataSourceRef
func exportDataSources(helper *commitHelper, job *gitExportJob) (dsLookup, error) {
cmd := &datasources.GetDataSourcesQuery{
OrgId: job.orgID,
}
err := job.sql.GetDataSources(helper.ctx, cmd)
if err != nil {
return nil, err
}
sort.SliceStable(cmd.Result, func(i, j int) bool {
return cmd.Result[i].Created.After(cmd.Result[j].Created)
})
byUID := make(map[string]*extract.DataSourceRef, len(cmd.Result))
byName := make(map[string]*extract.DataSourceRef, len(cmd.Result))
for _, ds := range cmd.Result {
ref := &extract.DataSourceRef{
UID: ds.Uid,
Type: ds.Type,
}
byUID[ds.Uid] = ref
byName[ds.Name] = ref
ds.OrgId = 0
ds.Version = 0
err := helper.add(commitOptions{
body: []commitBody{
{
fpath: filepath.Join(helper.orgDir, "datasources", fmt.Sprintf("%s-ds.json", ds.Uid)),
body: prettyJSON(ds),
},
},
when: ds.Created,
comment: fmt.Sprintf("Add datasource: %s", ds.Name),
})
if err != nil {
return nil, err
}
}
// Return the lookup function
return func(ref *extract.DataSourceRef) *extract.DataSourceRef {
if ref == nil || ref.UID == "" {
return &extract.DataSourceRef{
UID: "default.uid",
Type: "default.type",
}
}
v, ok := byUID[ref.UID]
if ok {
return v
}
v, ok = byName[ref.UID]
if ok {
return v
}
return nil
}, nil
}
pkg/services/export/export_snapshots.go
+43
View File
@@ -0,0 +1,43 @@
package export
import (
"fmt"
"path/filepath"
"time"
"github.com/grafana/grafana/pkg/services/dashboardsnapshots"
)
func exportSnapshots(helper *commitHelper, job *gitExportJob) error {
cmd := &dashboardsnapshots.GetDashboardSnapshotsQuery{
OrgId: job.orgID,
Limit: 500000,
SignedInUser: nil,
}
if cmd.SignedInUser == nil {
return fmt.Errorf("snapshots requires an admin user")
}
err := job.dashboardsnapshotsService.SearchDashboardSnapshots(helper.ctx, cmd)
if err != nil {
return err
}
if len(cmd.Result) < 1 {
return nil // nothing
}
gitcmd := commitOptions{
when: time.Now(),
comment: "Export playlists",
}
for _, snapshot := range cmd.Result {
gitcmd.body = append(gitcmd.body, commitBody{
fpath: filepath.Join(helper.orgDir, "snapshot", fmt.Sprintf("%d-snapshot.json", snapshot.Id)),
body: prettyJSON(snapshot),
})
}
return helper.add(gitcmd)
}
pkg/services/export/export_sys_playlists.go
+40
View File
@@ -0,0 +1,40 @@
package export
import (
"fmt"
"path/filepath"
"time"
"github.com/grafana/grafana/pkg/models"
)
func exportSystemPlaylists(helper *commitHelper, job *gitExportJob) error {
cmd := &models.GetPlaylistsQuery{
OrgId: job.orgID,
Limit: 500000,
}
err := job.sql.SearchPlaylists(helper.ctx, cmd)
if err != nil {
return err
}
if len(cmd.Result) < 1 {
return nil // nothing
}
gitcmd := commitOptions{
when: time.Now(),
comment: "Export playlists",
}
for _, playlist := range cmd.Result {
// TODO: fix the playlist API so it returns the json we need :)
gitcmd.body = append(gitcmd.body, commitBody{
fpath: filepath.Join(helper.orgDir, "system", "playlists", fmt.Sprintf("%s-playlist.json", playlist.UID)),
body: prettyJSON(playlist),
})
}
return helper.add(gitcmd)
}
pkg/services/export/export_sys_preferences.go
+128
View File
@@ -0,0 +1,128 @@
package export
import (
"fmt"
"path"
"path/filepath"
"time"
"github.com/grafana/grafana/pkg/services/sqlstore"
)
func exportSystemPreferences(helper *commitHelper, job *gitExportJob) error {
type preferences struct {
UserID int64 `json:"-" xorm:"user_id"`
TeamID int64 `json:"-" xorm:"team_id"`
HomeDashboardID int64 `json:"-" xorm:"home_dashboard_id"`
Updated time.Time `json:"-" xorm:"updated"`
JSONData map[string]interface{} `json:"-" xorm:"json_data"`
Theme string `json:"theme"`
Locale string `json:"locale"`
Timezone string `json:"timezone"`
WeekStart string `json:"week_start,omitempty"`
HomeDashboard string `json:"home,omitempty" xorm:"uid"` // dashboard
NavBar interface{} `json:"navbar,omitempty"`
QueryHistory interface{} `json:"queryHistory,omitempty"`
}
prefsDir := path.Join(helper.orgDir, "system", "preferences")
users := make(map[int64]*userInfo, len(helper.users))
for _, user := range helper.users {
users[user.ID] = user
}
return job.sql.WithDbSession(helper.ctx, func(sess *sqlstore.DBSession) error {
rows := make([]*preferences, 0)
sess.Table("preferences").
Join("LEFT", "dashboard", "dashboard.id = preferences.home_dashboard_id").
Cols("preferences.*", "dashboard.uid").
Where("preferences.org_id = ?", helper.orgID)
err := sess.Find(&rows)
if err != nil {
return err
}
var comment string
var fpath string
for _, row := range rows {
if row.TeamID > 0 {
fpath = filepath.Join(prefsDir, "team", fmt.Sprintf("%d.json", row.TeamID))
comment = fmt.Sprintf("Team preferences: %d", row.TeamID)
} else if row.UserID == 0 {
fpath = filepath.Join(prefsDir, "default.json")
comment = "Default preferences"
} else {
user, ok := users[row.UserID]
if ok {
delete(users, row.UserID)
} else {
user = &userInfo{
Login: fmt.Sprintf("__%d__", row.UserID),
}
}
fpath = filepath.Join(prefsDir, "user", fmt.Sprintf("%s.json", user.Login))
comment = fmt.Sprintf("User preferences: %s", user.getAuthor().Name)
}
if row.JSONData != nil {
v, ok := row.JSONData["locale"]
if ok && row.Locale == "" {
s, ok := v.(string)
if ok {
row.Locale = s
}
}
v, ok = row.JSONData["navbar"]
if ok && row.NavBar == nil {
row.NavBar = v
}
v, ok = row.JSONData["queryHistory"]
if ok && row.QueryHistory == nil {
row.QueryHistory = v
}
}
err := helper.add(commitOptions{
body: []commitBody{
{
fpath: fpath,
body: prettyJSON(row),
},
},
when: row.Updated,
comment: comment,
userID: row.UserID,
})
if err != nil {
return err
}
}
// add a file for all useres that may not be in the system
for _, user := range users {
row := preferences{
Theme: user.Theme, // never set?
}
err := helper.add(commitOptions{
body: []commitBody{
{
fpath: filepath.Join(prefsDir, "user", fmt.Sprintf("%s.json", user.Login)),
body: prettyJSON(row),
},
},
when: user.Updated,
comment: "user preferences",
userID: row.UserID,
})
if err != nil {
return err
}
}
return err
})
}
pkg/services/export/export_sys_stars.go
+65
View File
@@ -0,0 +1,65 @@
package export
import (
"fmt"
"path/filepath"
"github.com/grafana/grafana/pkg/services/sqlstore"
)
func exportSystemStars(helper *commitHelper, job *gitExportJob) error {
byUser := make(map[int64][]string, 50)
err := job.sql.WithDbSession(helper.ctx, func(sess *sqlstore.DBSession) error {
type starResult struct {
User int64 `xorm:"user_id"`
UID string `xorm:"uid"`
}
rows := make([]*starResult, 0)
sess.Table("star").
Join("INNER", "dashboard", "dashboard.id = star.dashboard_id").
Cols("star.user_id", "dashboard.uid").
Where("dashboard.org_id = ?", helper.orgID)
err := sess.Find(&rows)
if err != nil {
return err
}
for _, row := range rows {
stars := append(byUser[row.User], fmt.Sprintf("dashboard/%s", row.UID))
byUser[row.User] = stars
}
return err
})
if err != nil {
return err
}
for userID, stars := range byUser {
user, ok := helper.users[userID]
if !ok {
user = &userInfo{
Login: fmt.Sprintf("__unknown_%d", userID),
}
}
err := helper.add(commitOptions{
body: []commitBody{
{
fpath: filepath.Join(helper.orgDir, "system", "stars", fmt.Sprintf("%s.json", user.Login)),
body: prettyJSON(stars),
},
},
when: user.Updated,
comment: "user preferences",
userID: userID,
})
if err != nil {
return err
}
}
return nil
}
pkg/services/export/frame_helper.go
+138
View File
@@ -0,0 +1,138 @@
package export
import (
"encoding/json"
"fmt"
"strings"
"github.com/grafana/grafana-plugin-sdk-go/data"
)
type fieldInfo struct {
Name string
Conv data.FieldConverter
}
type frameOpts struct {
schema []fieldInfo
skip []string
}
func prettyJSON(v interface{}) []byte {
b, _ := json.MarshalIndent(v, "", " ")
return b
}
func queryResultToDataFrame(rows []map[string]interface{}, opts frameOpts) (*data.Frame, error) {
count := len(rows)
if count < 1 {
return nil, nil // empty frame
}
schema := opts.schema
if len(schema) < 1 {
skip := make(map[string]bool, len(opts.skip))
for _, k := range opts.skip {
skip[k] = true
}
for k, v := range rows[0] {
if skip[k] {
continue
}
field := fieldInfo{
Name: k,
Conv: data.FieldConverter{
OutputFieldType: data.FieldTypeFor(v),
},
}
if field.Conv.OutputFieldType == data.FieldTypeUnknown {
fmt.Printf("UNKNOWN type: %s / %v\n", k, v)
continue
}
// Don't write passwords to disk for now!!!!
if k == "password" || k == "salt" {
field.Conv.Converter = func(v interface{}) (interface{}, error) {
return fmt.Sprintf("<%s>", k), nil
}
}
schema = append(schema, field)
}
}
fields := make([]*data.Field, len(schema))
for i, s := range schema {
fields[i] = data.NewFieldFromFieldType(s.Conv.OutputFieldType, count)
fields[i].Name = s.Name
}
var err error
for i, row := range rows {
for j, s := range schema {
v, ok := row[s.Name]
if ok && v != nil {
if s.Conv.Converter != nil {
v, err = s.Conv.Converter(v)
if err != nil {
return nil, fmt.Errorf("converting field: %s // %s", s.Name, err.Error())
}
}
fields[j].Set(i, v)
}
}
}
// Fields are in random order
if len(opts.schema) < 1 {
last := []*data.Field{}
frame := data.NewFrame("")
lookup := make(map[string]*data.Field, len(fields))
for _, f := range fields {
if f.Name == "id" {
frame.Fields = append(frame.Fields, f) // first
continue
}
lookup[f.Name] = f
}
// First items
for _, name := range []string{"name", "login", "email", "role", "description", "uid"} {
f, ok := lookup[name]
if ok {
frame.Fields = append(frame.Fields, f) // first
delete(lookup, name)
}
}
// IDs
for k, f := range lookup {
if strings.HasSuffix(k, "_id") {
frame.Fields = append(frame.Fields, f) // first
delete(lookup, k)
} else if strings.HasPrefix(k, "is_") {
last = append(last, f) // first
delete(lookup, k)
}
}
// Last items
for _, name := range []string{"created", "updated"} {
f, ok := lookup[name]
if ok {
last = append(last, f) // first
delete(lookup, name)
}
}
// Rest
for _, f := range lookup {
frame.Fields = append(frame.Fields, f)
}
frame.Fields = append(frame.Fields, last...)
return frame, nil
}
return data.NewFrame("", fields...), nil
}
pkg/services/export/git_export_job.go
+205
View File
@@ -0,0 +1,205 @@
package export
import (
"context"
"fmt"
"path"
"sync"
"time"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/dashboardsnapshots"
"github.com/grafana/grafana/pkg/services/sqlstore"
)
var _ Job = new(gitExportJob)
type gitExportJob struct {
logger log.Logger
sql *sqlstore.SQLStore
dashboardsnapshotsService dashboardsnapshots.Service
orgID int64
rootDir string
statusMu sync.Mutex
status ExportStatus
cfg ExportConfig
broadcaster statusBroadcaster
}
type simpleExporter = func(helper *commitHelper, job *gitExportJob) error
func startGitExportJob(cfg ExportConfig, sql *sqlstore.SQLStore, dashboardsnapshotsService dashboardsnapshots.Service, rootDir string, orgID int64, broadcaster statusBroadcaster) (Job, error) {
job := &gitExportJob{
logger: log.New("git_export_job"),
cfg: cfg,
sql: sql,
dashboardsnapshotsService: dashboardsnapshotsService,
orgID: orgID,
rootDir: rootDir,
broadcaster: broadcaster,
status: ExportStatus{
Running: true,
Target: "git export",
Started: time.Now().UnixMilli(),
Current: 0,
},
}
broadcaster(job.status)
go job.start()
return job, nil
}
func (e *gitExportJob) getStatus() ExportStatus {
e.statusMu.Lock()
defer e.statusMu.Unlock()
return e.status
}
func (e *gitExportJob) getConfig() ExportConfig {
e.statusMu.Lock()
defer e.statusMu.Unlock()
return e.cfg
}
// Utility function to export dashboards
func (e *gitExportJob) start() {
defer func() {
e.logger.Info("Finished git export job")
e.statusMu.Lock()
defer e.statusMu.Unlock()
s := e.status
if err := recover(); err != nil {
e.logger.Error("export panic", "error", err)
s.Status = fmt.Sprintf("ERROR: %v", err)
}
// Make sure it finishes OK
if s.Finished < 10 {
s.Finished = time.Now().UnixMilli()
}
s.Running = false
if s.Status == "" {
s.Status = "done"
}
s.Target = e.rootDir
e.status = s
e.broadcaster(s)
}()
err := e.doExportWithHistory()
if err != nil {
e.logger.Error("ERROR", "e", err)
e.status.Status = "ERROR"
e.status.Last = err.Error()
e.broadcaster(e.status)
}
}
func (e *gitExportJob) doExportWithHistory() error {
r, err := git.PlainInit(e.rootDir, false)
if err != nil {
return err
}
// default to "main" branch
h := plumbing.NewSymbolicReference(plumbing.HEAD, plumbing.ReferenceName("refs/heads/main"))
err = r.Storer.SetReference(h)
if err != nil {
return err
}
w, err := r.Worktree()
if err != nil {
return err
}
helper := &commitHelper{
repo: r,
work: w,
ctx: context.Background(),
workDir: e.rootDir,
orgDir: e.rootDir,
}
cmd := &models.SearchOrgsQuery{}
err = e.sql.SearchOrgs(helper.ctx, cmd)
if err != nil {
return err
}
// Export each org
for _, org := range cmd.Result {
if len(cmd.Result) > 1 {
helper.orgDir = path.Join(e.rootDir, fmt.Sprintf("org_%d", org.Id))
}
err = helper.initOrg(e.sql, org.Id)
if err != nil {
return err
}
err = e.doOrgExportWithHistory(helper)
if err != nil {
return err
}
}
// cleanup the folder
e.status.Target = "pruning..."
e.broadcaster(e.status)
err = r.Prune(git.PruneOptions{})
// TODO
// git gc --prune=now --aggressive
return err
}
func (e *gitExportJob) doOrgExportWithHistory(helper *commitHelper) error {
lookup, err := exportDataSources(helper, e)
if err != nil {
return err
}
if true {
err = exportDashboards(helper, e, lookup)
if err != nil {
return err
}
}
// Run all the simple exporters
exporters := []simpleExporter{
dumpAuthTables,
exportSystemPreferences,
exportSystemStars,
exportSystemPlaylists,
exportAnnotations,
}
// This needs a real admin user to use the interfaces (and decrypt)
if false {
exporters = append(exporters, exportSnapshots)
}
for _, fn := range exporters {
err = fn(helper, e)
if err != nil {
return err
}
}
return err
}
/**
git remote add origin git@github.com:ryantxu/test-dash-repo.git
git branch -M main
git push -u origin main
**/
pkg/services/export/service.go
+37 -11
View File
@@ -2,15 +2,21 @@ package export
import (
"encoding/json"
"fmt"
"net/http"
"os"
"path/filepath"
"sync"
"time"
"github.com/grafana/grafana/pkg/api/response"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/dashboardsnapshots"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/live"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/setting"
)
type ExportService interface {
@@ -22,25 +28,31 @@ type ExportService interface {
}
type StandardExport struct {
logger log.Logger
sql *sqlstore.SQLStore
glive *live.GrafanaLive
mutex sync.Mutex
logger log.Logger
glive *live.GrafanaLive
mutex sync.Mutex
dataDir string
// Services
sql *sqlstore.SQLStore
dashboardsnapshotsService dashboardsnapshots.Service
// updated with mutex
exportJob Job
}
func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles, gl *live.GrafanaLive) ExportService {
func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles, gl *live.GrafanaLive, cfg *setting.Cfg, dashboardsnapshotsService dashboardsnapshots.Service) ExportService {
if !features.IsEnabled(featuremgmt.FlagExport) {
return &StubExport{}
}
return &StandardExport{
sql: sql,
glive: gl,
logger: log.New("export_service"),
exportJob: &stoppedJob{},
sql: sql,
glive: gl,
logger: log.New("export_service"),
dashboardsnapshotsService: dashboardsnapshotsService,
exportJob: &stoppedJob{},
dataDir: cfg.DataPath,
}
}
@@ -67,9 +79,23 @@ func (ex *StandardExport) HandleRequestExport(c *models.ReqContext) response.Res
return response.Error(http.StatusLocked, "export already running", nil)
}
job, err := startDummyExportJob(cfg, func(s ExportStatus) {
var job Job
broadcast := func(s ExportStatus) {
ex.broadcastStatus(c.OrgId, s)
})
}
switch cfg.Format {
case "dummy":
job, err = startDummyExportJob(cfg, broadcast)
case "git":
dir := filepath.Join(ex.dataDir, "export_git", fmt.Sprintf("git_%d", time.Now().Unix()))
if err := os.MkdirAll(dir, os.ModePerm); err != nil {
return response.Error(http.StatusBadRequest, "Error creating export folder", nil)
}
job, err = startGitExportJob(cfg, ex.sql, ex.dashboardsnapshotsService, dir, c.OrgId, broadcast)
default:
return response.Error(http.StatusBadRequest, "Unsupported job format", nil)
}
if err != nil {
ex.logger.Error("failed to start export job", "err", err)
return response.Error(http.StatusBadRequest, "failed to start export job", err)