public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Export: support stopping exports (#51769)

Browse Source
This commit is contained in:
Ryan McKinley
2022-07-07 11:02:01 -07:00
committed by GitHub
parent 8deb17fdc4
commit 5cb8010440
16 changed files with 385 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/api/api.go
+1
View File
@@ -559,6 +559,7 @@ func (hs *HTTPServer) registerRoutes() {
if hs.Features.IsEnabled(featuremgmt.FlagExport) {
adminRoute.Get("/export", reqGrafanaAdmin, routing.Wrap(hs.ExportService.HandleGetStatus))
adminRoute.Post("/export", reqGrafanaAdmin, routing.Wrap(hs.ExportService.HandleRequestExport))
adminRoute.Post("/export/stop", reqGrafanaAdmin, routing.Wrap(hs.ExportService.HandleRequestStop))
}
adminRoute.Post("/encryption/rotate-data-keys", reqGrafanaAdmin, routing.Wrap(hs.AdminRotateDataEncryptionKeys))
pkg/services/export/commit_helper.go
+35 -19
View File
@@ -17,13 +17,15 @@ import (
)
type commitHelper struct {
ctx context.Context
repo *git.Repository
work *git.Worktree
orgDir string // includes the orgID
workDir string // same as the worktree root
orgID int64
users map[int64]*userInfo
ctx context.Context
repo *git.Repository
work *git.Worktree
orgDir string // includes the orgID
workDir string // same as the worktree root
orgID int64
users map[int64]*userInfo
stopRequested bool
broadcast func(path string)
}
type commitBody struct {
@@ -64,6 +66,26 @@ func (ch *commitHelper) initOrg(sql *sqlstore.SQLStore, orgID int64) error {
}
func (ch *commitHelper) add(opts commitOptions) error {
if ch.stopRequested {
return fmt.Errorf("stop requested")
}
if len(opts.body) < 1 {
return nil // nothing to commit
}
user, ok := ch.users[opts.userID]
if !ok {
user = &userInfo{
Name: "admin",
Email: "admin@unknown.org",
}
}
sig := user.getAuthor()
if opts.when.Unix() > 100 {
sig.When = opts.when
}
for _, b := range opts.body {
if !strings.HasPrefix(b.fpath, ch.orgDir) {
return fmt.Errorf("invalid path, must be within the root folder")
@@ -87,6 +109,10 @@ func (ch *commitHelper) add(opts commitOptions) error {
if err != nil {
return err
}
err = os.Chtimes(b.fpath, sig.When, sig.When)
if err != nil {
return err
}
sub := b.fpath[len(ch.workDir)+1:]
_, err = ch.work.Add(sub)
@@ -100,22 +126,11 @@ func (ch *commitHelper) add(opts commitOptions) error {
}
}
user, ok := ch.users[opts.userID]
if !ok {
user = &userInfo{
Name: "admin",
Email: "admin@unknown.org",
}
}
sig := user.getAuthor()
if opts.when.Unix() > 10 {
sig.When = opts.when
}
copts := &git.CommitOptions{
Author: &sig,
}
ch.broadcast(opts.body[0].fpath)
_, err := ch.work.Commit(opts.comment, copts)
return err
}
@@ -140,6 +155,7 @@ func (u *userInfo) getAuthor() object.Signature {
return object.Signature{
Name: firstRealStringX(u.Name, u.Login, u.Email, "?"),
Email: firstRealStringX(u.Email, u.Login, u.Name, "?"),
When: time.Now(),
}
}
pkg/services/export/dummy_job.go
+10 -5
View File
@@ -15,10 +15,11 @@ var _ Job = new(dummyExportJob)
type dummyExportJob struct {
logger log.Logger
statusMu sync.Mutex
status ExportStatus
cfg ExportConfig
broadcaster statusBroadcaster
statusMu sync.Mutex
status ExportStatus
cfg ExportConfig
broadcaster statusBroadcaster
stopRequested bool
}
func startDummyExportJob(cfg ExportConfig, broadcaster statusBroadcaster) (Job, error) {
@@ -40,6 +41,10 @@ func startDummyExportJob(cfg ExportConfig, broadcaster statusBroadcaster) (Job,
return job, nil
}
func (e *dummyExportJob) requestStop() {
e.stopRequested = true
}
func (e *dummyExportJob) start() {
defer func() {
e.logger.Info("Finished dummy export job")
@@ -74,7 +79,7 @@ func (e *dummyExportJob) start() {
e.statusMu.Unlock()
// Wait till we are done
shouldStop := e.status.Current >= e.status.Count
shouldStop := e.stopRequested || e.status.Current >= e.status.Count
e.broadcaster(e.status)
if shouldStop {
pkg/services/export/export_dashboards.go
+24 -19
View File
@@ -21,8 +21,8 @@ func exportDashboards(helper *commitHelper, job *gitExportJob, lookup dsLookup)
folders := make(map[int64]string, 100)
// Should root files be at the root or in a subfolder called "general"?
if true {
folders[0] = "general"
if len(job.cfg.GeneralFolderPath) > 0 {
folders[0] = job.cfg.GeneralFolderPath // "general"
}
rootDir := path.Join(helper.orgDir, "root")
@@ -132,7 +132,7 @@ func exportDashboards(helper *commitHelper, job *gitExportJob, lookup dsLookup)
// Now walk the history
err = job.sql.WithDbSession(helper.ctx, func(sess *sqlstore.DBSession) error {
type dashVersionResult struct {
DashId int64 `xorm:"dashboard_id"`
DashId int64 `xorm:"id"`
Version int64 `xorm:"version"`
Created time.Time `xorm:"created"`
CreatedBy int64 `xorm:"created_by"`
@@ -142,16 +142,27 @@ func exportDashboards(helper *commitHelper, job *gitExportJob, lookup dsLookup)
rows := make([]*dashVersionResult, 0, len(ids))
sess.Table("dashboard_version").
Join("INNER", "dashboard", "dashboard.id = dashboard_version.dashboard_id").
Where("org_id = ?", job.orgID).
Cols("dashboard_version.dashboard_id",
"dashboard_version.version",
"dashboard_version.created",
"dashboard_version.created_by",
"dashboard_version.message",
"dashboard_version.data").
Asc("dashboard_version.created")
if job.cfg.KeepHistory {
sess.Table("dashboard_version").
Join("INNER", "dashboard", "dashboard.id = dashboard_version.dashboard_id").
Where("org_id = ?", job.orgID).
Cols("dashboard.id",
"dashboard_version.version",
"dashboard_version.created",
"dashboard_version.created_by",
"dashboard_version.message",
"dashboard_version.data").
Asc("dashboard_version.created")
} else {
sess.Table("dashboard").
Where("org_id = ?", job.orgID).
Cols("id",
"version",
"created",
"created_by",
"data").
Asc("created")
}
err := sess.Find(&rows)
if err != nil {
@@ -186,14 +197,8 @@ func exportDashboards(helper *commitHelper, job *gitExportJob, lookup dsLookup)
if err != nil {
return err
}
count++
fmt.Printf("COMMIT: %d // %s (%d)\n", count, fpath, row.Version)
job.status.Current = count
job.status.Last = fpath
job.status.Changed = time.Now().UnixMilli()
job.broadcaster(job.status)
}
return nil
pkg/services/export/export_ds.go
+8 -1
View File
@@ -11,7 +11,7 @@ import (
type dsLookup func(ref *extract.DataSourceRef) *extract.DataSourceRef
func exportDataSources(helper *commitHelper, job *gitExportJob) (dsLookup, error) {
func exportDataSources(helper *commitHelper, job *gitExportJob, save bool) (dsLookup, error) {
cmd := &datasources.GetDataSourcesQuery{
OrgId: job.orgID,
}
@@ -33,8 +33,15 @@ func exportDataSources(helper *commitHelper, job *gitExportJob) (dsLookup, error
}
byUID[ds.Uid] = ref
byName[ds.Name] = ref
if !save {
continue
}
ds.OrgId = 0
ds.Version = 0
ds.SecureJsonData = map[string][]byte{
"TODO": []byte("secret store lookup"),
}
err := helper.add(commitOptions{
body: []commitBody{
pkg/services/export/export_files.go
+48
View File
@@ -0,0 +1,48 @@
package export
import (
"fmt"
"path"
"github.com/grafana/grafana/pkg/infra/filestorage"
"github.com/grafana/grafana/pkg/infra/log"
)
func exportFiles(helper *commitHelper, job *gitExportJob) error {
fs := filestorage.NewDbStorage(log.New("grafanaStorageLogger"), job.sql, nil, fmt.Sprintf("/%d/", helper.orgID))
paging := &filestorage.Paging{}
for {
rsp, err := fs.List(helper.ctx, "/resources", paging, &filestorage.ListOptions{
WithFolders: false, // ????
Recursive: true,
WithContents: true,
})
if err != nil {
return err
}
for _, f := range rsp.Files {
if f.Size < 1 {
continue
}
err = helper.add(commitOptions{
body: []commitBody{{
body: f.Contents,
fpath: path.Join(helper.orgDir, f.FullPath),
}},
comment: fmt.Sprintf("Adding: %s", path.Base(f.FullPath)),
when: f.Created,
})
if err != nil {
return err
}
}
paging.After = rsp.LastPath
if !rsp.HasMore {
break
}
}
return nil
}
pkg/services/export/export_kv_store.go
+46
View File
@@ -0,0 +1,46 @@
package export
import (
"fmt"
"path"
"time"
"github.com/grafana/grafana/pkg/services/sqlstore"
)
func exportKVStore(helper *commitHelper, job *gitExportJob) error {
kvdir := path.Join(helper.orgDir, "system", "kv_store")
return job.sql.WithDbSession(helper.ctx, func(sess *sqlstore.DBSession) error {
type kvResult struct {
Namespace string `xorm:"namespace"`
Key string `xorm:"key"`
Value string `xorm:"value"`
Updated time.Time `xorm:"updated"`
}
rows := make([]*kvResult, 0)
sess.Table("kv_store").Where("org_id = ? OR org_id = 0", helper.orgID)
err := sess.Find(&rows)
if err != nil {
return err
}
for _, row := range rows {
err = helper.add(commitOptions{
body: []commitBody{{
body: []byte(row.Value),
fpath: path.Join(kvdir, row.Namespace, row.Key),
}},
comment: fmt.Sprintf("Exporting: %s/%s", row.Namespace, row.Key),
when: row.Updated,
})
if err != nil {
return err
}
}
return err
})
}
pkg/services/export/export_live.go
+47
View File
@@ -0,0 +1,47 @@
package export
import (
"fmt"
"path"
"time"
"github.com/grafana/grafana/pkg/services/sqlstore"
)
func exportLive(helper *commitHelper, job *gitExportJob) error {
messagedir := path.Join(helper.orgDir, "system", "live", "message")
return job.sql.WithDbSession(helper.ctx, func(sess *sqlstore.DBSession) error {
type msgResult struct {
Channel string `xorm:"channel"`
Data string `xorm:"data"`
CreatedBy int64 `xorm:"created_by"`
Created time.Time `xorm:"created"`
}
rows := make([]*msgResult, 0)
sess.Table("live_message").Where("org_id = ?", helper.orgID)
err := sess.Find(&rows)
if err != nil {
return err
}
for _, row := range rows {
err = helper.add(commitOptions{
body: []commitBody{{
body: []byte(row.Data),
fpath: path.Join(messagedir, row.Channel) + ".json", // must be JSON files
}},
comment: fmt.Sprintf("Exporting: %s", row.Channel),
when: row.Created,
userID: row.CreatedBy,
})
if err != nil {
return err
}
}
return err
})
}
pkg/services/export/export_snapshots.go
+1 -1
View File
@@ -29,7 +29,7 @@ func exportSnapshots(helper *commitHelper, job *gitExportJob) error {
gitcmd := commitOptions{
when: time.Now(),
comment: "Export playlists",
comment: "Export snapshots",
}
for _, snapshot := range cmd.Result {
pkg/services/export/export_sys_preferences.go
+8
View File
@@ -58,11 +58,15 @@ func exportSystemPreferences(helper *commitHelper, job *gitExportJob) error {
user, ok := users[row.UserID]
if ok {
delete(users, row.UserID)
if user.IsServiceAccount {
continue // don't write preferences for service account
}
} else {
user = &userInfo{
Login: fmt.Sprintf("__%d__", row.UserID),
}
}
fpath = filepath.Join(prefsDir, "user", fmt.Sprintf("%s.json", user.Login))
comment = fmt.Sprintf("User preferences: %s", user.getAuthor().Name)
}
@@ -105,6 +109,10 @@ func exportSystemPreferences(helper *commitHelper, job *gitExportJob) error {
// add a file for all useres that may not be in the system
for _, user := range users {
if user.IsServiceAccount {
continue
}
row := preferences{
Theme: user.Theme, // never set?
}
pkg/services/export/git_export_job.go
+35 -16
View File
@@ -29,6 +29,7 @@ type gitExportJob struct {
status ExportStatus
cfg ExportConfig
broadcaster statusBroadcaster
helper *commitHelper
}
type simpleExporter = func(helper *commitHelper, job *gitExportJob) error
@@ -69,6 +70,10 @@ func (e *gitExportJob) getConfig() ExportConfig {
return e.cfg
}
func (e *gitExportJob) requestStop() {
e.helper.stopRequested = true // will error on the next write
}
// Utility function to export dashboards
func (e *gitExportJob) start() {
defer func() {
@@ -119,16 +124,21 @@ func (e *gitExportJob) doExportWithHistory() error {
if err != nil {
return err
}
helper := &commitHelper{
e.helper = &commitHelper{
repo: r,
work: w,
ctx: context.Background(),
workDir: e.rootDir,
orgDir: e.rootDir,
broadcast: func(p string) {
e.status.Last = p[len(e.rootDir):]
e.status.Changed = time.Now().UnixMilli()
e.broadcaster(e.status)
},
}
cmd := &models.SearchOrgsQuery{}
err = e.sql.SearchOrgs(helper.ctx, cmd)
err = e.sql.SearchOrgs(e.helper.ctx, cmd)
if err != nil {
return err
}
@@ -136,14 +146,14 @@ func (e *gitExportJob) doExportWithHistory() error {
// Export each org
for _, org := range cmd.Result {
if len(cmd.Result) > 1 {
helper.orgDir = path.Join(e.rootDir, fmt.Sprintf("org_%d", org.Id))
e.helper.orgDir = path.Join(e.rootDir, fmt.Sprintf("org_%d", org.Id))
}
err = helper.initOrg(e.sql, org.Id)
err = e.helper.initOrg(e.sql, org.Id)
if err != nil {
return err
}
err = e.doOrgExportWithHistory(helper)
err = e.doOrgExportWithHistory(e.helper)
if err != nil {
return err
}
@@ -161,29 +171,38 @@ func (e *gitExportJob) doExportWithHistory() error {
}
func (e *gitExportJob) doOrgExportWithHistory(helper *commitHelper) error {
lookup, err := exportDataSources(helper, e)
include := e.cfg.Include
lookup, err := exportDataSources(helper, e, include.DS)
if err != nil {
return err
}
if true {
if include.Dash {
err = exportDashboards(helper, e, lookup)
if err != nil {
return err
}
}
// Run all the simple exporters
exporters := []simpleExporter{
dumpAuthTables,
exportSystemPreferences,
exportSystemStars,
exportSystemPlaylists,
exportAnnotations,
exporters := []simpleExporter{}
if include.Auth {
exporters = append(exporters, dumpAuthTables)
}
// This needs a real admin user to use the interfaces (and decrypt)
if false {
if include.Services {
exporters = append(exporters, exportFiles,
exportSystemPreferences,
exportSystemStars,
exportSystemPlaylists,
exportKVStore,
exportLive)
}
if include.Anno {
exporters = append(exporters, exportAnnotations)
}
if include.Snapshots {
exporters = append(exporters, exportSnapshots)
}
pkg/services/export/service.go
+12
View File
@@ -25,6 +25,9 @@ type ExportService interface {
// Read raw file contents out of the store
HandleRequestExport(c *models.ReqContext) response.Response
// Cancel any running export
HandleRequestStop(c *models.ReqContext) response.Response
}
type StandardExport struct {
@@ -63,6 +66,15 @@ func (ex *StandardExport) HandleGetStatus(c *models.ReqContext) response.Respons
return response.JSON(http.StatusOK, ex.exportJob.getStatus())
}
func (ex *StandardExport) HandleRequestStop(c *models.ReqContext) response.Response {
ex.mutex.Lock()
defer ex.mutex.Unlock()
ex.exportJob.requestStop()
return response.JSON(http.StatusOK, ex.exportJob.getStatus())
}
func (ex *StandardExport) HandleRequestExport(c *models.ReqContext) response.Response {
var cfg ExportConfig
err := json.NewDecoder(c.Req.Body).Decode(&cfg)
pkg/services/export/stopped_job.go
+2
View File
@@ -17,3 +17,5 @@ func (e *stoppedJob) getStatus() ExportStatus {
func (e *stoppedJob) getConfig() ExportConfig {
return ExportConfig{}
}
func (e *stoppedJob) requestStop() {}
pkg/services/export/stub.go
+4
View File
@@ -18,3 +18,7 @@ func (ex *StubExport) HandleGetStatus(c *models.ReqContext) response.Response {
func (ex *StubExport) HandleRequestExport(c *models.ReqContext) response.Response {
return response.Error(http.StatusForbidden, "feature not enabled", nil)
}
func (ex *StubExport) HandleRequestStop(c *models.ReqContext) response.Response {
return response.Error(http.StatusForbidden, "feature not enabled", nil)
}
pkg/services/export/types.go
+18 -9
View File
@@ -15,21 +15,30 @@ type ExportStatus struct {
// Basic export config (for now)
type ExportConfig struct {
Format string `json:"format"`
Git GitExportConfig `json:"git"`
Format string `json:"format"`
GeneralFolderPath string `json:"generalFolderPath"`
KeepHistory bool `json:"history"`
Include struct {
Auth bool `json:"auth"`
DS bool `json:"ds"`
Dash bool `json:"dash"`
Services bool `json:"services"`
Usage bool `json:"usage"`
Anno bool `json:"anno"`
Snapshots bool `json:"snapshots"`
} `json:"include"`
// Depends on the format
Git GitExportConfig `json:"git"`
}
type GitExportConfig struct {
// General folder is either at the root or as a subfolder
GeneralAtRoot bool `json:"generalAtRoot"`
// Keeping all history is nice, but much slower
ExcludeHistory bool `json:"excludeHistory"`
}
type GitExportConfig struct{}
type Job interface {
getStatus() ExportStatus
getConfig() ExportConfig
requestStop()
}
// Will broadcast the live status
public/app/features/storage/ExportView.tsx
+86 -69
View File
@@ -1,11 +1,14 @@
import React, { useEffect, useState } from 'react';
import { useLocalStorage } from 'react-use';
import { isLiveChannelMessageEvent, isLiveChannelStatusEvent, LiveChannelScope } from '@grafana/data';
import { getBackendSrv, getGrafanaLiveSrv } from '@grafana/runtime';
import { Button, CodeEditor, Modal } from '@grafana/ui';
import { Button, CodeEditor, HorizontalGroup, LinkButton } from '@grafana/ui';
import { StorageView } from './types';
export const EXPORT_LOCAL_STORAGE_KEY = 'grafana.export.config';
interface ExportStatusMessage {
running: boolean;
target: string;
@@ -18,25 +21,57 @@ interface ExportStatusMessage {
status: string;
}
interface ExportInclude {
auth: boolean;
ds: boolean;
dash: boolean;
services: boolean;
usage: boolean;
anno: boolean;
snapshots: boolean;
}
interface ExportJob {
format: 'git';
generalFolderPath: string;
history: boolean;
include: ExportInclude;
git?: {};
}
const includAll: ExportInclude = {
auth: true,
ds: true,
dash: true,
services: true,
usage: true,
anno: true,
snapshots: false, // will fail until we have a real user
};
const defaultJob: ExportJob = {
format: 'git',
generalFolderPath: 'general',
history: true,
include: includAll,
git: {},
};
interface Props {
onPathChange: (p: string, v?: StorageView) => void;
}
export const ExportView = ({ onPathChange }: Props) => {
const [status, setStatus] = useState<ExportStatusMessage>();
const [rawBody, setBody] = useLocalStorage<ExportJob>(EXPORT_LOCAL_STORAGE_KEY, defaultJob);
const body = { ...defaultJob, ...rawBody, include: { ...includAll, ...rawBody?.include } };
const [open, setOpen] = useState(false);
const [body, setBody] = useState({
format: 'git',
git: {},
});
const onDismiss = () => setOpen(false);
const doStart = () => {
getBackendSrv()
.post('/api/admin/export', body)
.then((v) => {
onDismiss();
});
getBackendSrv().post('/api/admin/export', body);
};
const doStop = () => {
getBackendSrv().post('/api/admin/export/stop');
};
useEffect(() => {
@@ -56,71 +91,53 @@ export const ExportView = ({ onPathChange }: Props) => {
},
});
// if not running, open the thread
setTimeout(() => {
if (!status) {
setOpen(true);
}
}, 500);
return () => {
subscription.unsubscribe();
};
// eslint-disable-next-line react-hooks/exhaustive-deps
}, []);
const renderButton = () => {
return (
<>
<Modal title={'Export grafana instance'} isOpen={open} onDismiss={onDismiss}>
<div>
<CodeEditor
height={200}
value={JSON.stringify(body, null, 2) ?? ''}
showLineNumbers={false}
readOnly={false}
language="json"
showMiniMap={false}
onBlur={(text: string) => {
setBody(JSON.parse(text)); // force JSON?
}}
/>
</div>
<Modal.ButtonRow>
<Button onClick={doStart}>Start</Button>
<Button variant="secondary" onClick={onDismiss}>
Cancel
</Button>
</Modal.ButtonRow>
</Modal>
<Button onClick={() => setOpen(true)} variant="primary">
Export
</Button>
<Button variant="secondary" onClick={() => onPathChange('/')}>
Cancel
</Button>
</>
);
};
if (!status) {
return <div>{renderButton()}</div>;
}
return (
<div>
<pre>{JSON.stringify(status, null, 2)}</pre>
{Boolean(!status.running) && renderButton()}
{Boolean(status.running) && (
<Button
variant="secondary"
onClick={() => {
getBackendSrv().post('/api/admin/export/stop');
}}
>
Stop
</Button>
{status && (
<div>
<h3>Status</h3>
<pre>{JSON.stringify(status, null, 2)}</pre>
{status.running && (
<div>
<Button variant="secondary" onClick={doStop}>
Stop
</Button>
</div>
)}
</div>
)}
{!Boolean(status?.running) && (
<div>
<h3>Export grafana instance</h3>
<CodeEditor
height={275}
value={JSON.stringify(body, null, 2) ?? ''}
showLineNumbers={false}
readOnly={false}
language="json"
showMiniMap={false}
onBlur={(text: string) => {
setBody(JSON.parse(text)); // force JSON?
}}
/>
<br />
<HorizontalGroup>
<Button onClick={doStart} variant="primary">
Export
</Button>
<LinkButton href="admin/storage/" variant="secondary">
Cancel
</LinkButton>
</HorizontalGroup>
</div>
)}
</div>
);