From 7858965117696b47c660bd835ae41686e3b604b8 Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 18 Jan 2018 12:04:12 +0100 Subject: [PATCH 01/12] codestyle: extract code into methods --- .../wrapper/datasource_plugin_wrapper.go | 12 ++-- .../provisioning/dashboards/file_reader.go | 59 ++++++++++++------- 2 files changed, 45 insertions(+), 26 deletions(-) diff --git a/pkg/plugins/datasource/wrapper/datasource_plugin_wrapper.go b/pkg/plugins/datasource/wrapper/datasource_plugin_wrapper.go index 490fa0f573a..0afa2a400e4 100644 --- a/pkg/plugins/datasource/wrapper/datasource_plugin_wrapper.go +++ b/pkg/plugins/datasource/wrapper/datasource_plugin_wrapper.go @@ -28,12 +28,12 @@ func (tw *DatasourcePluginWrapper) Query(ctx context.Context, ds *models.DataSou pbQuery := &datasource.DatasourceRequest{ Datasource: &datasource.DatasourceInfo{ - Name: ds.Name, - Type: ds.Type, - Url: ds.Url, - Id: ds.Id, - OrgId: ds.OrgId, - JsonData: string(jsonData), + Name: ds.Name, + Type: ds.Type, + Url: ds.Url, + Id: ds.Id, + OrgId: ds.OrgId, + JsonData: string(jsonData), DecryptedSecureJsonData: ds.SecureJsonData.Decrypt(), }, TimeRange: &datasource.TimeRange{ diff --git a/pkg/services/provisioning/dashboards/file_reader.go b/pkg/services/provisioning/dashboards/file_reader.go index fbe1a03e287..6b9eeec9e48 100644 --- a/pkg/services/provisioning/dashboards/file_reader.go +++ b/pkg/services/provisioning/dashboards/file_reader.go @@ -135,35 +135,39 @@ func getOrCreateFolderId(cfg *DashboardsAsConfig, repo dashboards.Repository) (i return cmd.Result.Id, nil } +func resolveSymlink(fileinfo os.FileInfo, path string) (os.FileInfo, error) { + checkFilepath, err := filepath.EvalSymlinks(path) + if path != checkFilepath { + path = checkFilepath + fi, err := os.Lstat(checkFilepath) + if err != nil { + return nil, err + } + + return fi, nil + } + + return fileinfo, err +} + func createWalkFn(fr *fileReader, folderId int64) filepath.WalkFunc { return func(path string, fileInfo os.FileInfo, err error) error { if err != nil { return err } - if fileInfo.IsDir() { - if strings.HasPrefix(fileInfo.Name(), ".") { - return filepath.SkipDir - } - return nil + + isValid, err := validateWalkablePath(fileInfo) + if !isValid { + return err } - if !strings.HasSuffix(fileInfo.Name(), ".json") { - return nil - } - - checkFilepath, err := filepath.EvalSymlinks(path) - - if path != checkFilepath { - path = checkFilepath - fi, err := os.Lstat(checkFilepath) - if err != nil { - return err - } - fileInfo = fi + resolvedFileInfo, err := resolveSymlink(fileInfo, path) + if err != nil { + return err } cachedDashboard, exist := fr.cache.getCache(path) - if exist && cachedDashboard.UpdatedAt == fileInfo.ModTime() { + if exist && cachedDashboard.UpdatedAt == resolvedFileInfo.ModTime() { return nil } @@ -194,7 +198,7 @@ func createWalkFn(fr *fileReader, folderId int64) filepath.WalkFunc { } // break if db version is newer then fil version - if cmd.Result.Updated.Unix() >= fileInfo.ModTime().Unix() { + if cmd.Result.Updated.Unix() >= resolvedFileInfo.ModTime().Unix() { return nil } @@ -205,6 +209,21 @@ func createWalkFn(fr *fileReader, folderId int64) filepath.WalkFunc { } } +func validateWalkablePath(fileInfo os.FileInfo) (bool, error) { + if fileInfo.IsDir() { + if strings.HasPrefix(fileInfo.Name(), ".") { + return false, filepath.SkipDir + } + return false, nil + } + + if !strings.HasSuffix(fileInfo.Name(), ".json") { + return false, nil + } + + return true, nil +} + func (fr *fileReader) readDashboardFromFile(path string, folderId int64) (*dashboards.SaveDashboardItem, error) { reader, err := os.Open(path) if err != nil { From 67a9e6a71d9c0cbcaf1343e5ef08dd9021c3a51b Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 23 Jan 2018 12:28:56 +0100 Subject: [PATCH 02/12] provisioing: add lookup table provisioned dashboards --- conf/provisioning/dashboards/sample.yaml | 2 +- pkg/api/dashboard.go | 2 +- pkg/api/dashboard_test.go | 14 +- pkg/models/dashboard_provisioning.go | 2 + pkg/models/dashboards.go | 21 ++ pkg/services/dashboards/dashboards.go | 94 ++++++-- .../dashboards/dashboard_cache.go | 6 +- .../provisioning/dashboards/file_reader.go | 47 ++-- .../dashboards/file_reader_test.go | 14 +- pkg/services/provisioning/dashboards/types.go | 4 +- pkg/services/sqlstore/dashboard.go | 221 +++++++++--------- .../sqlstore/dashboard_provisioning.go | 66 ++++++ .../sqlstore/dashboard_provisioning_test.go | 48 ++++ .../sqlstore/migrations/dashboard_mig.go | 17 ++ pkg/services/sqlstore/playlist.go | 4 - 15 files changed, 400 insertions(+), 162 deletions(-) create mode 100644 pkg/models/dashboard_provisioning.go create mode 100644 pkg/services/sqlstore/dashboard_provisioning.go create mode 100644 pkg/services/sqlstore/dashboard_provisioning_test.go diff --git a/conf/provisioning/dashboards/sample.yaml b/conf/provisioning/dashboards/sample.yaml index 40992d1461e..e40612af508 100644 --- a/conf/provisioning/dashboards/sample.yaml +++ b/conf/provisioning/dashboards/sample.yaml @@ -3,4 +3,4 @@ # folder: '' # type: file # options: -# folder: /var/lib/grafana/dashboards \ No newline at end of file +# folder: /var/lib/grafana/dashboards diff --git a/pkg/api/dashboard.go b/pkg/api/dashboard.go index 87c42884e31..28a955d5abf 100644 --- a/pkg/api/dashboard.go +++ b/pkg/api/dashboard.go @@ -182,7 +182,7 @@ func PostDashboard(c *middleware.Context, cmd m.SaveDashboardCommand) Response { } } - dashItem := &dashboards.SaveDashboardItem{ + dashItem := &dashboards.SaveDashboardDTO{ Dashboard: dash, Message: cmd.Message, OrgId: c.OrgId, diff --git a/pkg/api/dashboard_test.go b/pkg/api/dashboard_test.go index e6228878625..ed76544b9b8 100644 --- a/pkg/api/dashboard_test.go +++ b/pkg/api/dashboard_test.go @@ -21,15 +21,25 @@ import ( ) type fakeDashboardRepo struct { - inserted []*dashboards.SaveDashboardItem + inserted []*dashboards.SaveDashboardDTO + provisioned []*m.DashboardProvisioning getDashboard []*m.Dashboard } -func (repo *fakeDashboardRepo) SaveDashboard(json *dashboards.SaveDashboardItem) (*m.Dashboard, error) { +func (repo *fakeDashboardRepo) SaveDashboard(json *dashboards.SaveDashboardDTO) (*m.Dashboard, error) { repo.inserted = append(repo.inserted, json) return json.Dashboard, nil } +func (repo *fakeDashboardRepo) SaveProvisionedDashboard(dto *dashboards.SaveDashboardDTO, provisioning *m.DashboardProvisioning) (*m.Dashboard, error) { + repo.inserted = append(repo.inserted, dto) + return dto.Dashboard, nil +} + +func (repo *fakeDashboardRepo) GetProvisionedDashboardData(name string) ([]*m.DashboardProvisioning, error) { + return repo.provisioned, nil +} + var fakeRepo *fakeDashboardRepo func TestDashboardApiEndpoint(t *testing.T) { diff --git a/pkg/models/dashboard_provisioning.go b/pkg/models/dashboard_provisioning.go new file mode 100644 index 00000000000..d6a05a41090 --- /dev/null +++ b/pkg/models/dashboard_provisioning.go @@ -0,0 +1,2 @@ +package models + diff --git a/pkg/models/dashboards.go b/pkg/models/dashboards.go index 091f27ec413..577e97c657d 100644 --- a/pkg/models/dashboards.go +++ b/pkg/models/dashboards.go @@ -167,6 +167,21 @@ type SaveDashboardCommand struct { Result *Dashboard } +type DashboardProvisioning struct { + Id int64 + DashboardId int64 + Name string + ExternalId string + Updated time.Time +} + +type SaveProvisionedDashboardCommand struct { + DashboardCmd *SaveDashboardCommand + DashboardProvisioning *DashboardProvisioning + + Result *Dashboard +} + type DeleteDashboardCommand struct { Id int64 OrgId int64 @@ -209,3 +224,9 @@ type GetDashboardSlugByIdQuery struct { Id int64 Result string } + +type GetProvisionedDashboardDataQuery struct { + Name string + + Result []*DashboardProvisioning +} diff --git a/pkg/services/dashboards/dashboards.go b/pkg/services/dashboards/dashboards.go index 4bdba59b18e..ce60fde0454 100644 --- a/pkg/services/dashboards/dashboards.go +++ b/pkg/services/dashboards/dashboards.go @@ -9,7 +9,9 @@ import ( ) type Repository interface { - SaveDashboard(*SaveDashboardItem) (*models.Dashboard, error) + SaveDashboard(*SaveDashboardDTO) (*models.Dashboard, error) + SaveProvisionedDashboard(dto *SaveDashboardDTO, provisioning *models.DashboardProvisioning) (*models.Dashboard, error) + GetProvisionedDashboardData(name string) ([]*models.DashboardProvisioning, error) } var repositoryInstance Repository @@ -22,7 +24,7 @@ func SetRepository(rep Repository) { repositoryInstance = rep } -type SaveDashboardItem struct { +type SaveDashboardDTO struct { OrgId int64 UpdatedAt time.Time UserId int64 @@ -33,15 +35,25 @@ type SaveDashboardItem struct { type DashboardRepository struct{} -func (dr *DashboardRepository) SaveDashboard(json *SaveDashboardItem) (*models.Dashboard, error) { - dashboard := json.Dashboard +func (dr *DashboardRepository) GetProvisionedDashboardData(name string) ([]*models.DashboardProvisioning, error) { + cmd := &models.GetProvisionedDashboardDataQuery{Name: name} + err := bus.Dispatch(cmd) + if err != nil { + return nil, err + } + + return cmd.Result, nil +} + +func (dr *DashboardRepository) buildSaveDashboardCommand(dto *SaveDashboardDTO) (*models.SaveDashboardCommand, error) { + dashboard := dto.Dashboard if dashboard.Title == "" { return nil, models.ErrDashboardTitleEmpty } validateAlertsCmd := alerting.ValidateDashboardAlertsCommand{ - OrgId: json.OrgId, + OrgId: dto.OrgId, Dashboard: dashboard, } @@ -49,33 +61,77 @@ func (dr *DashboardRepository) SaveDashboard(json *SaveDashboardItem) (*models.D return nil, models.ErrDashboardContainsInvalidAlertData } - cmd := models.SaveDashboardCommand{ + cmd := &models.SaveDashboardCommand{ Dashboard: dashboard.Data, - Message: json.Message, - OrgId: json.OrgId, - Overwrite: json.Overwrite, - UserId: json.UserId, + Message: dto.Message, + OrgId: dto.OrgId, + Overwrite: dto.Overwrite, + UserId: dto.UserId, FolderId: dashboard.FolderId, IsFolder: dashboard.IsFolder, } - if !json.UpdatedAt.IsZero() { - cmd.UpdatedAt = json.UpdatedAt + if !dto.UpdatedAt.IsZero() { + cmd.UpdatedAt = dto.UpdatedAt } - err := bus.Dispatch(&cmd) - if err != nil { - return nil, err - } + return cmd, nil +} +func (dr *DashboardRepository) updateAlerting(cmd *models.SaveDashboardCommand, dto *SaveDashboardDTO) error { alertCmd := alerting.UpdateDashboardAlertsCommand{ - OrgId: json.OrgId, - UserId: json.UserId, + OrgId: dto.OrgId, + UserId: dto.UserId, Dashboard: cmd.Result, } if err := bus.Dispatch(&alertCmd); err != nil { - return nil, models.ErrDashboardFailedToUpdateAlertData + return models.ErrDashboardFailedToUpdateAlertData + } + + return nil +} + +func (dr *DashboardRepository) SaveProvisionedDashboard(dto *SaveDashboardDTO, provisioning *models.DashboardProvisioning) (*models.Dashboard, error) { + cmd, err := dr.buildSaveDashboardCommand(dto) + if err != nil { + return nil, err + } + + saveCmd := &models.SaveProvisionedDashboardCommand{ + DashboardCmd: cmd, + DashboardProvisioning: provisioning, + } + + // dashboard + err = bus.Dispatch(saveCmd) + if err != nil { + return nil, err + } + + //alerts + err = dr.updateAlerting(cmd, dto) + if err != nil { + return nil, err + } + + return cmd.Result, nil +} + +func (dr *DashboardRepository) SaveDashboard(dto *SaveDashboardDTO) (*models.Dashboard, error) { + cmd, err := dr.buildSaveDashboardCommand(dto) + if err != nil { + return nil, err + } + + err = bus.Dispatch(&cmd) + if err != nil { + return nil, err + } + + err = dr.updateAlerting(cmd, dto) + if err != nil { + return nil, err } return cmd.Result, nil diff --git a/pkg/services/provisioning/dashboards/dashboard_cache.go b/pkg/services/provisioning/dashboards/dashboard_cache.go index da6b7e8a5e8..7f8f6fe490f 100644 --- a/pkg/services/provisioning/dashboards/dashboard_cache.go +++ b/pkg/services/provisioning/dashboards/dashboard_cache.go @@ -14,17 +14,17 @@ func NewDashboardCache() *dashboardCache { return &dashboardCache{internalCache: gocache.New(5*time.Minute, 30*time.Minute)} } -func (fr *dashboardCache) addDashboardCache(key string, json *dashboards.SaveDashboardItem) { +func (fr *dashboardCache) addDashboardCache(key string, json *dashboards.SaveDashboardDTO) { fr.internalCache.Add(key, json, time.Minute*10) } -func (fr *dashboardCache) getCache(key string) (*dashboards.SaveDashboardItem, bool) { +func (fr *dashboardCache) getCache(key string) (*dashboards.SaveDashboardDTO, bool) { obj, exist := fr.internalCache.Get(key) if !exist { return nil, exist } - dash, ok := obj.(*dashboards.SaveDashboardItem) + dash, ok := obj.(*dashboards.SaveDashboardDTO) if !ok { return nil, ok } diff --git a/pkg/services/provisioning/dashboards/file_reader.go b/pkg/services/provisioning/dashboards/file_reader.go index 6b9eeec9e48..91aa6169390 100644 --- a/pkg/services/provisioning/dashboards/file_reader.go +++ b/pkg/services/provisioning/dashboards/file_reader.go @@ -25,12 +25,12 @@ var ( ) type fileReader struct { - Cfg *DashboardsAsConfig - Path string - log log.Logger - dashboardRepo dashboards.Repository - cache *dashboardCache - createWalk func(fr *fileReader, folderId int64) filepath.WalkFunc + Cfg *DashboardsAsConfig + Path string + log log.Logger + dashboardRepo dashboards.Repository + cache *dashboardCache + createWalk func(fr *fileReader, folderId int64) filepath.WalkFunc } func NewDashboardFileReader(cfg *DashboardsAsConfig, log log.Logger) (*fileReader, error) { @@ -50,28 +50,28 @@ func NewDashboardFileReader(cfg *DashboardsAsConfig, log log.Logger) (*fileReade } return &fileReader{ - Cfg: cfg, - Path: path, - log: log, - dashboardRepo: dashboards.GetRepository(), - cache: NewDashboardCache(), - createWalk: createWalkFn, + Cfg: cfg, + Path: path, + log: log, + dashboardRepo: dashboards.GetRepository(), + cache: NewDashboardCache(), + createWalk: createWalkFn, }, nil } func (fr *fileReader) ReadAndListen(ctx context.Context) error { - ticker := time.NewTicker(checkDiskForChangesInterval) - if err := fr.startWalkingDisk(); err != nil { fr.log.Error("failed to search for dashboards", "error", err) } + ticker := time.NewTicker(checkDiskForChangesInterval) + running := false for { select { case <-ticker.C: - if !running { // avoid walking the filesystem in parallel. incase fs is very slow. + if !running { // avoid walking the filesystem in parallel. in-case fs is very slow. running = true go func() { if err := fr.startWalkingDisk(); err != nil { @@ -115,7 +115,7 @@ func getOrCreateFolderId(cfg *DashboardsAsConfig, repo dashboards.Repository) (i // dashboard folder not found. create one. if err == models.ErrDashboardNotFound { - dash := &dashboards.SaveDashboardItem{} + dash := &dashboards.SaveDashboardDTO{} dash.Dashboard = models.NewDashboard(cfg.Folder) dash.Dashboard.IsFolder = true dash.Overwrite = true @@ -129,7 +129,7 @@ func getOrCreateFolderId(cfg *DashboardsAsConfig, repo dashboards.Repository) (i } if !cmd.Result.IsFolder { - return 0, fmt.Errorf("Got invalid response. Expected folder, found dashboard") + return 0, fmt.Errorf("got invalid response. expected folder, found dashboard") } return cmd.Result.Id, nil @@ -188,7 +188,7 @@ func createWalkFn(fr *fileReader, folderId int64) filepath.WalkFunc { // if we don't have the dashboard in the db, save it! if err == models.ErrDashboardNotFound { fr.log.Debug("saving new dashboard", "file", path) - _, err = fr.dashboardRepo.SaveDashboard(dash) + err = saveDashboard(fr, path, dash, fileInfo.ModTime()) return err } @@ -203,11 +203,18 @@ func createWalkFn(fr *fileReader, folderId int64) filepath.WalkFunc { } fr.log.Debug("loading dashboard from disk into database.", "file", path) - _, err = fr.dashboardRepo.SaveDashboard(dash) + err = saveDashboard(fr, path, dash, fileInfo.ModTime()) return err } } +func saveDashboard(fr *fileReader, path string, dash *dashboards.SaveDashboardDTO, modTime time.Time) error { + //dash.Extras["provisioning.filepath"] = path + _, err := fr.dashboardRepo.SaveDashboard(dash) + + + return err +} func validateWalkablePath(fileInfo os.FileInfo) (bool, error) { if fileInfo.IsDir() { @@ -224,7 +231,7 @@ func validateWalkablePath(fileInfo os.FileInfo) (bool, error) { return true, nil } -func (fr *fileReader) readDashboardFromFile(path string, folderId int64) (*dashboards.SaveDashboardItem, error) { +func (fr *fileReader) readDashboardFromFile(path string, folderId int64) (*dashboards.SaveDashboardDTO, error) { reader, err := os.Open(path) if err != nil { return nil, err diff --git a/pkg/services/provisioning/dashboards/file_reader_test.go b/pkg/services/provisioning/dashboards/file_reader_test.go index f2805196dde..5c90e573f80 100644 --- a/pkg/services/provisioning/dashboards/file_reader_test.go +++ b/pkg/services/provisioning/dashboards/file_reader_test.go @@ -241,15 +241,25 @@ func (ffi FakeFileInfo) Sys() interface{} { } type fakeDashboardRepo struct { - inserted []*dashboards.SaveDashboardItem + inserted []*dashboards.SaveDashboardDTO + provisioned []*models.DashboardProvisioning getDashboard []*models.Dashboard } -func (repo *fakeDashboardRepo) SaveDashboard(json *dashboards.SaveDashboardItem) (*models.Dashboard, error) { +func (repo *fakeDashboardRepo) SaveDashboard(json *dashboards.SaveDashboardDTO) (*models.Dashboard, error) { repo.inserted = append(repo.inserted, json) return json.Dashboard, nil } +func (repo *fakeDashboardRepo) GetProvisionedDashboardData(name string) ([]*models.DashboardProvisioning, error) { + return repo.provisioned, nil +} + +func (repo *fakeDashboardRepo) SaveProvisionedDashboard(dto *dashboards.SaveDashboardDTO, provisioning *models.DashboardProvisioning) (*models.Dashboard, error) { + repo.inserted = append(repo.inserted, dto) + return dto.Dashboard, nil +} + func mockGetDashboardQuery(cmd *models.GetDashboardQuery) error { for _, d := range fakeRepo.getDashboard { if d.Slug == cmd.Slug { diff --git a/pkg/services/provisioning/dashboards/types.go b/pkg/services/provisioning/dashboards/types.go index cf65c65348c..91379b33148 100644 --- a/pkg/services/provisioning/dashboards/types.go +++ b/pkg/services/provisioning/dashboards/types.go @@ -18,8 +18,8 @@ type DashboardsAsConfig struct { Options map[string]interface{} `json:"options" yaml:"options"` } -func createDashboardJson(data *simplejson.Json, lastModified time.Time, cfg *DashboardsAsConfig, folderId int64) (*dashboards.SaveDashboardItem, error) { - dash := &dashboards.SaveDashboardItem{} +func createDashboardJson(data *simplejson.Json, lastModified time.Time, cfg *DashboardsAsConfig, folderId int64) (*dashboards.SaveDashboardDTO, error) { + dash := &dashboards.SaveDashboardDTO{} dash.Dashboard = models.NewDashboardFromJson(data) dash.UpdatedAt = lastModified dash.Overwrite = true diff --git a/pkg/services/sqlstore/dashboard.go b/pkg/services/sqlstore/dashboard.go index 0b6b60a5e11..4c72e3423be 100644 --- a/pkg/services/sqlstore/dashboard.go +++ b/pkg/services/sqlstore/dashboard.go @@ -22,121 +22,126 @@ func init() { func SaveDashboard(cmd *m.SaveDashboardCommand) error { return inTransaction(func(sess *DBSession) error { - dash := cmd.GetDashboardModel() + return saveDashboard(sess, cmd) + }) +} - // try get existing dashboard - var existing, sameTitle m.Dashboard +func saveDashboard(sess *DBSession, cmd *m.SaveDashboardCommand) error { + dash := cmd.GetDashboardModel() - if dash.Id > 0 { - dashWithIdExists, err := sess.Where("id=? AND org_id=?", dash.Id, dash.OrgId).Get(&existing) - if err != nil { + // try get existing dashboard + var existing, sameTitle m.Dashboard + + if dash.Id > 0 { + dashWithIdExists, err := sess.Where("id=? AND org_id=?", dash.Id, dash.OrgId).Get(&existing) + if err != nil { + return err + } + if !dashWithIdExists { + return m.ErrDashboardNotFound + } + + // check for is someone else has written in between + if dash.Version != existing.Version { + if cmd.Overwrite { + dash.Version = existing.Version + } else { + return m.ErrDashboardVersionMismatch + } + } + + // do not allow plugin dashboard updates without overwrite flag + if existing.PluginId != "" && cmd.Overwrite == false { + return m.UpdatePluginDashboardError{PluginId: existing.PluginId} + } + } + + sameTitleExists, err := sess.Where("org_id=? AND slug=?", dash.OrgId, dash.Slug).Get(&sameTitle) + if err != nil { + return err + } + + if sameTitleExists { + // another dashboard with same name + if dash.Id != sameTitle.Id { + if cmd.Overwrite { + dash.Id = sameTitle.Id + dash.Version = sameTitle.Version + } else { + return m.ErrDashboardWithSameNameExists + } + } + } + + err = setHasAcl(sess, dash) + if err != nil { + return err + } + + parentVersion := dash.Version + affectedRows := int64(0) + + if dash.Id == 0 { + dash.Version = 1 + metrics.M_Api_Dashboard_Insert.Inc() + dash.Data.Set("version", dash.Version) + affectedRows, err = sess.Insert(dash) + } else { + dash.Version++ + dash.Data.Set("version", dash.Version) + + if !cmd.UpdatedAt.IsZero() { + dash.Updated = cmd.UpdatedAt + } + + affectedRows, err = sess.MustCols("folder_id", "has_acl").Id(dash.Id).Update(dash) + } + + if err != nil { + return err + } + + if affectedRows == 0 { + return m.ErrDashboardNotFound + } + + dashVersion := &m.DashboardVersion{ + DashboardId: dash.Id, + ParentVersion: parentVersion, + RestoredFrom: cmd.RestoredFrom, + Version: dash.Version, + Created: time.Now(), + CreatedBy: dash.UpdatedBy, + Message: cmd.Message, + Data: dash.Data, + } + + // insert version entry + if affectedRows, err = sess.Insert(dashVersion); err != nil { + return err + } else if affectedRows == 0 { + return m.ErrDashboardNotFound + } + + // delete existing tags + _, err = sess.Exec("DELETE FROM dashboard_tag WHERE dashboard_id=?", dash.Id) + if err != nil { + return err + } + + // insert new tags + tags := dash.GetTags() + if len(tags) > 0 { + for _, tag := range tags { + if _, err := sess.Insert(&DashboardTag{DashboardId: dash.Id, Term: tag}); err != nil { return err } - if !dashWithIdExists { - return m.ErrDashboardNotFound - } - - // check for is someone else has written in between - if dash.Version != existing.Version { - if cmd.Overwrite { - dash.Version = existing.Version - } else { - return m.ErrDashboardVersionMismatch - } - } - - // do not allow plugin dashboard updates without overwrite flag - if existing.PluginId != "" && cmd.Overwrite == false { - return m.UpdatePluginDashboardError{PluginId: existing.PluginId} - } } + } - sameTitleExists, err := sess.Where("org_id=? AND slug=?", dash.OrgId, dash.Slug).Get(&sameTitle) - if err != nil { - return err - } + cmd.Result = dash - if sameTitleExists { - // another dashboard with same name - if dash.Id != sameTitle.Id { - if cmd.Overwrite { - dash.Id = sameTitle.Id - dash.Version = sameTitle.Version - } else { - return m.ErrDashboardWithSameNameExists - } - } - } - - err = setHasAcl(sess, dash) - if err != nil { - return err - } - - parentVersion := dash.Version - affectedRows := int64(0) - - if dash.Id == 0 { - dash.Version = 1 - metrics.M_Api_Dashboard_Insert.Inc() - dash.Data.Set("version", dash.Version) - affectedRows, err = sess.Insert(dash) - } else { - dash.Version++ - dash.Data.Set("version", dash.Version) - - if !cmd.UpdatedAt.IsZero() { - dash.Updated = cmd.UpdatedAt - } - - affectedRows, err = sess.MustCols("folder_id", "has_acl").Id(dash.Id).Update(dash) - } - - if err != nil { - return err - } - - if affectedRows == 0 { - return m.ErrDashboardNotFound - } - - dashVersion := &m.DashboardVersion{ - DashboardId: dash.Id, - ParentVersion: parentVersion, - RestoredFrom: cmd.RestoredFrom, - Version: dash.Version, - Created: time.Now(), - CreatedBy: dash.UpdatedBy, - Message: cmd.Message, - Data: dash.Data, - } - - // insert version entry - if affectedRows, err = sess.Insert(dashVersion); err != nil { - return err - } else if affectedRows == 0 { - return m.ErrDashboardNotFound - } - - // delete existing tags - _, err = sess.Exec("DELETE FROM dashboard_tag WHERE dashboard_id=?", dash.Id) - if err != nil { - return err - } - - // insert new tags - tags := dash.GetTags() - if len(tags) > 0 { - for _, tag := range tags { - if _, err := sess.Insert(&DashboardTag{DashboardId: dash.Id, Term: tag}); err != nil { - return err - } - } - } - cmd.Result = dash - - return err - }) + return err } func setHasAcl(sess *DBSession, dash *m.Dashboard) error { diff --git a/pkg/services/sqlstore/dashboard_provisioning.go b/pkg/services/sqlstore/dashboard_provisioning.go new file mode 100644 index 00000000000..91d63e82a28 --- /dev/null +++ b/pkg/services/sqlstore/dashboard_provisioning.go @@ -0,0 +1,66 @@ +package sqlstore + +import ( + "time" + + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/models" +) + +func init() { + bus.AddHandler("sql", GetProvisionedDashboardDataQuery) + bus.AddHandler("sql", SaveProvisionedDashboard) +} + +type DashboardExtras struct { + Id int64 + DashboardId int64 + Key string + Value string +} + +func SaveProvisionedDashboard(cmd *models.SaveProvisionedDashboardCommand) error { + return inTransaction(func(sess *DBSession) error { + err := saveDashboard(sess, cmd.DashboardCmd) + + if err != nil { + return err + } + + cmd.Result = cmd.DashboardCmd.Result + return saveProvionedData(sess, cmd.DashboardProvisioning) + }) +} + +func saveProvionedData(sess *DBSession, cmd *models.DashboardProvisioning) error { + results := &models.DashboardProvisioning{} + + exist, err := sess.Where("dashboard_id=?", cmd.DashboardId).Get(results) + if err != nil { + return err + } + + cmd.Id = results.Id + cmd.Updated = time.Now() + + println("exists", exist) + if exist { + + _, err = sess.ID(results.Id).Update(cmd) + } else { + _, err = sess.Insert(cmd) + } + + return err +} + +func GetProvisionedDashboardDataQuery(cmd *models.GetProvisionedDashboardDataQuery) error { + var result []*models.DashboardProvisioning + + if err := x.Where("name = ?", cmd.Name).Find(&result); err != nil { + return err + } + + cmd.Result = result + return nil +} diff --git a/pkg/services/sqlstore/dashboard_provisioning_test.go b/pkg/services/sqlstore/dashboard_provisioning_test.go new file mode 100644 index 00000000000..96e63d58933 --- /dev/null +++ b/pkg/services/sqlstore/dashboard_provisioning_test.go @@ -0,0 +1,48 @@ +package sqlstore + +import ( + "testing" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/models" + . "github.com/smartystreets/goconvey/convey" +) + +func TestDashboardProvisioningTest(t *testing.T) { + Convey("Testing Dashboard provisioning", t, func() { + InitTestDB(t) + + saveDashboardCmd := &models.SaveDashboardCommand{ + OrgId: 1, + FolderId: 0, + IsFolder: false, + Dashboard: simplejson.NewFromAny(map[string]interface{}{ + "id": nil, + "title": "test dashboard", + }), + } + + Convey("Saving dashboards with extras", func() { + cmd := &models.SaveProvisionedDashboardCommand{ + DashboardCmd: saveDashboardCmd, + DashboardProvisioning: &models.DashboardProvisioning{ + Name: "default", + ExternalId: "/var/grafana.json", + }, + } + + err := SaveProvisionedDashboard(cmd) + So(err, ShouldBeNil) + So(cmd.Result, ShouldNotBeNil) + So(cmd.Result.Id, ShouldNotEqual, 0) + + Convey("Can query for provisioned dashboards", func() { + query := &models.GetProvisionedDashboardDataQuery{Name: "default"} + err := GetProvisionedDashboardDataQuery(query) + So(err, ShouldBeNil) + + So(len(query.Result), ShouldEqual, 1) + }) + }) + }) +} diff --git a/pkg/services/sqlstore/migrations/dashboard_mig.go b/pkg/services/sqlstore/migrations/dashboard_mig.go index 4f1602be931..ccaa4d013d3 100644 --- a/pkg/services/sqlstore/migrations/dashboard_mig.go +++ b/pkg/services/sqlstore/migrations/dashboard_mig.go @@ -150,4 +150,21 @@ func addDashboardMigration(mg *Migrator) { mg.AddMigration("Add column has_acl in dashboard", NewAddColumnMigration(dashboardV2, &Column{ Name: "has_acl", Type: DB_Bool, Nullable: false, Default: "0", })) + + dashboardExtrasTable := Table{ + Name: "dashboard_provisioning", + Columns: []*Column{ + {Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true}, + {Name: "dashboard_id", Type: DB_BigInt, Nullable: true}, + {Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false}, + {Name: "external_id", Type: DB_Text, Nullable: false}, + {Name: "updated", Type: DB_Int, Nullable: false}, + }, + Indices: []*Index{ + {Cols: []string{"dashboard_id"}}, + {Cols: []string{"dashboard_id", "name"}, Type: IndexType}, + }, + } + + mg.AddMigration("create dashboard_provisioning", NewAddTableMigration(dashboardExtrasTable)) } diff --git a/pkg/services/sqlstore/playlist.go b/pkg/services/sqlstore/playlist.go index 72f3079db8d..4b4536455da 100644 --- a/pkg/services/sqlstore/playlist.go +++ b/pkg/services/sqlstore/playlist.go @@ -1,8 +1,6 @@ package sqlstore import ( - "fmt" - "github.com/grafana/grafana/pkg/bus" m "github.com/grafana/grafana/pkg/models" ) @@ -27,8 +25,6 @@ func CreatePlaylist(cmd *m.CreatePlaylistCommand) error { _, err = x.Insert(&playlist) - fmt.Printf("%v", playlist.Id) - playlistItems := make([]m.PlaylistItem, 0) for _, item := range cmd.Items { playlistItems = append(playlistItems, m.PlaylistItem{ From 77a4ccb82269cc55439c14636c9c8e2257df211a Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 23 Jan 2018 20:22:04 +0100 Subject: [PATCH 03/12] dashboards: save provisioning meta data --- pkg/models/dashboard_provisioning.go | 1 - .../provisioning/dashboards/file_reader.go | 32 ++++++++++--------- .../sqlstore/dashboard_provisioning.go | 16 +++++----- .../sqlstore/dashboard_provisioning_test.go | 2 ++ 4 files changed, 27 insertions(+), 24 deletions(-) diff --git a/pkg/models/dashboard_provisioning.go b/pkg/models/dashboard_provisioning.go index d6a05a41090..2640e7f93ea 100644 --- a/pkg/models/dashboard_provisioning.go +++ b/pkg/models/dashboard_provisioning.go @@ -1,2 +1 @@ package models - diff --git a/pkg/services/provisioning/dashboards/file_reader.go b/pkg/services/provisioning/dashboards/file_reader.go index 91aa6169390..3005de9a390 100644 --- a/pkg/services/provisioning/dashboards/file_reader.go +++ b/pkg/services/provisioning/dashboards/file_reader.go @@ -25,12 +25,12 @@ var ( ) type fileReader struct { - Cfg *DashboardsAsConfig - Path string - log log.Logger - dashboardRepo dashboards.Repository - cache *dashboardCache - createWalk func(fr *fileReader, folderId int64) filepath.WalkFunc + Cfg *DashboardsAsConfig + Path string + log log.Logger + dashboardRepo dashboards.Repository + cache *dashboardCache + createWalk func(fr *fileReader, folderId int64) filepath.WalkFunc } func NewDashboardFileReader(cfg *DashboardsAsConfig, log log.Logger) (*fileReader, error) { @@ -50,12 +50,12 @@ func NewDashboardFileReader(cfg *DashboardsAsConfig, log log.Logger) (*fileReade } return &fileReader{ - Cfg: cfg, - Path: path, - log: log, - dashboardRepo: dashboards.GetRepository(), - cache: NewDashboardCache(), - createWalk: createWalkFn, + Cfg: cfg, + Path: path, + log: log, + dashboardRepo: dashboards.GetRepository(), + cache: NewDashboardCache(), + createWalk: createWalkFn, }, nil } @@ -209,9 +209,11 @@ func createWalkFn(fr *fileReader, folderId int64) filepath.WalkFunc { } } func saveDashboard(fr *fileReader, path string, dash *dashboards.SaveDashboardDTO, modTime time.Time) error { - //dash.Extras["provisioning.filepath"] = path - _, err := fr.dashboardRepo.SaveDashboard(dash) - + d := &models.DashboardProvisioning{ + ExternalId: path, + Name: fr.Cfg.Name, + } + _, err := fr.dashboardRepo.SaveProvisionedDashboard(dash, d) return err } diff --git a/pkg/services/sqlstore/dashboard_provisioning.go b/pkg/services/sqlstore/dashboard_provisioning.go index 91d63e82a28..c064f78ce67 100644 --- a/pkg/services/sqlstore/dashboard_provisioning.go +++ b/pkg/services/sqlstore/dashboard_provisioning.go @@ -28,25 +28,25 @@ func SaveProvisionedDashboard(cmd *models.SaveProvisionedDashboardCommand) error } cmd.Result = cmd.DashboardCmd.Result - return saveProvionedData(sess, cmd.DashboardProvisioning) + + return saveProvionedData(sess, cmd.DashboardProvisioning, cmd.Result) }) } -func saveProvionedData(sess *DBSession, cmd *models.DashboardProvisioning) error { - results := &models.DashboardProvisioning{} +func saveProvionedData(sess *DBSession, cmd *models.DashboardProvisioning, dashboard *models.Dashboard) error { + result := &models.DashboardProvisioning{} - exist, err := sess.Where("dashboard_id=?", cmd.DashboardId).Get(results) + exist, err := sess.Where("dashboard_id=?", dashboard.Id).Get(result) if err != nil { return err } - cmd.Id = results.Id + cmd.Id = result.Id cmd.Updated = time.Now() + cmd.DashboardId = dashboard.Id - println("exists", exist) if exist { - - _, err = sess.ID(results.Id).Update(cmd) + _, err = sess.ID(result.Id).Update(cmd) } else { _, err = sess.Insert(cmd) } diff --git a/pkg/services/sqlstore/dashboard_provisioning_test.go b/pkg/services/sqlstore/dashboard_provisioning_test.go index 96e63d58933..c42d95e7495 100644 --- a/pkg/services/sqlstore/dashboard_provisioning_test.go +++ b/pkg/services/sqlstore/dashboard_provisioning_test.go @@ -35,6 +35,7 @@ func TestDashboardProvisioningTest(t *testing.T) { So(err, ShouldBeNil) So(cmd.Result, ShouldNotBeNil) So(cmd.Result.Id, ShouldNotEqual, 0) + dashId := cmd.Result.Id Convey("Can query for provisioned dashboards", func() { query := &models.GetProvisionedDashboardDataQuery{Name: "default"} @@ -42,6 +43,7 @@ func TestDashboardProvisioningTest(t *testing.T) { So(err, ShouldBeNil) So(len(query.Result), ShouldEqual, 1) + So(query.Result[0].DashboardId, ShouldEqual, dashId) }) }) }) From 57e7048b8ffcee67d241c930a4f43ff001e63b35 Mon Sep 17 00:00:00 2001 From: bergquist Date: Wed, 24 Jan 2018 15:22:03 +0100 Subject: [PATCH 04/12] provisioning: enables title changes for dashboards --- .../provisioning/dashboards/file_reader.go | 64 +++++++++++++++---- .../dashboards/file_reader_test.go | 6 +- pkg/services/sqlstore/dashboard.go | 1 - 3 files changed, 55 insertions(+), 16 deletions(-) diff --git a/pkg/services/provisioning/dashboards/file_reader.go b/pkg/services/provisioning/dashboards/file_reader.go index 3005de9a390..dbbb44f15b8 100644 --- a/pkg/services/provisioning/dashboards/file_reader.go +++ b/pkg/services/provisioning/dashboards/file_reader.go @@ -30,7 +30,7 @@ type fileReader struct { log log.Logger dashboardRepo dashboards.Repository cache *dashboardCache - createWalk func(fr *fileReader, folderId int64) filepath.WalkFunc + createWalk func(fr *fileReader, folderId int64, provisionedDashboards map[string]*models.DashboardProvisioning) filepath.WalkFunc } func NewDashboardFileReader(cfg *DashboardsAsConfig, log log.Logger) (*fileReader, error) { @@ -98,7 +98,26 @@ func (fr *fileReader) startWalkingDisk() error { return err } - return filepath.Walk(fr.Path, fr.createWalk(fr, folderId)) + byPath, err := getProvisionedDashboardByPath(fr.dashboardRepo, fr.Cfg.Name) + if err != nil { + return err + } + + return filepath.Walk(fr.Path, fr.createWalk(fr, folderId, byPath)) +} + +func getProvisionedDashboardByPath(repo dashboards.Repository, name string) (map[string]*models.DashboardProvisioning, error) { + arr, err := repo.GetProvisionedDashboardData(name) + if err != nil { + return nil, err + } + + byPath := map[string]*models.DashboardProvisioning{} + for _, pd := range arr { + byPath[pd.ExternalId] = pd + } + + return byPath, nil } func getOrCreateFolderId(cfg *DashboardsAsConfig, repo dashboards.Repository) (int64, error) { @@ -150,7 +169,7 @@ func resolveSymlink(fileinfo os.FileInfo, path string) (os.FileInfo, error) { return fileinfo, err } -func createWalkFn(fr *fileReader, folderId int64) filepath.WalkFunc { +func createWalkFn(fr *fileReader, folderId int64, provisionedDashboards map[string]*models.DashboardProvisioning) filepath.WalkFunc { return func(path string, fileInfo os.FileInfo, err error) error { if err != nil { return err @@ -177,18 +196,31 @@ func createWalkFn(fr *fileReader, folderId int64) filepath.WalkFunc { return nil } - if dash.Dashboard.Id != 0 { - fr.log.Error("Cannot provision dashboard. Please remove the id property from the json file") - return nil + var dbDashboard *models.Dashboard + cmd := &models.GetDashboardQuery{} + provisionedData, allReadyProvisioned := provisionedDashboards[path] + + // see if the + if allReadyProvisioned { + dash.Dashboard.Id = provisionedData.DashboardId + dash.Dashboard.Data.Set("id", provisionedData.DashboardId) + cmd.Id = provisionedData.DashboardId + } else { + if dash.Dashboard.Id != 0 { + fr.log.Error("Cannot provision dashboard. Please remove the id property from the json file") + return nil + } + + cmd.Slug = dash.Dashboard.Slug } - cmd := &models.GetDashboardQuery{Slug: dash.Dashboard.Slug} err = bus.Dispatch(cmd) + dbDashboard = cmd.Result // if we don't have the dashboard in the db, save it! if err == models.ErrDashboardNotFound { fr.log.Debug("saving new dashboard", "file", path) - err = saveDashboard(fr, path, dash, fileInfo.ModTime()) + err = saveDashboard(fr, path, dash) return err } @@ -198,24 +230,30 @@ func createWalkFn(fr *fileReader, folderId int64) filepath.WalkFunc { } // break if db version is newer then fil version - if cmd.Result.Updated.Unix() >= resolvedFileInfo.ModTime().Unix() { + if dbDashboard.Updated.Unix() >= resolvedFileInfo.ModTime().Unix() { return nil } fr.log.Debug("loading dashboard from disk into database.", "file", path) - err = saveDashboard(fr, path, dash, fileInfo.ModTime()) + err = saveDashboard(fr, path, dash) return err } } -func saveDashboard(fr *fileReader, path string, dash *dashboards.SaveDashboardDTO, modTime time.Time) error { + +func saveDashboard(fr *fileReader, path string, dash *dashboards.SaveDashboardDTO) error { d := &models.DashboardProvisioning{ ExternalId: path, Name: fr.Cfg.Name, } - _, err := fr.dashboardRepo.SaveProvisionedDashboard(dash, d) - return err + _, err := fr.dashboardRepo.SaveProvisionedDashboard(dash, d) + if err != nil { + return err + } + + fr.cache.addDashboardCache(path, dash) + return nil } func validateWalkablePath(fileInfo os.FileInfo) (bool, error) { diff --git a/pkg/services/provisioning/dashboards/file_reader_test.go b/pkg/services/provisioning/dashboards/file_reader_test.go index 5c90e573f80..390190e29af 100644 --- a/pkg/services/provisioning/dashboards/file_reader_test.go +++ b/pkg/services/provisioning/dashboards/file_reader_test.go @@ -174,13 +174,15 @@ func TestDashboardFileReader(t *testing.T) { reader, err := NewDashboardFileReader(cfg, log.New("test-logger")) So(err, ShouldBeNil) + emptyProvisioned := map[string]*models.DashboardProvisioning{} + Convey("should skip dirs that starts with .", func() { - shouldSkip := reader.createWalk(reader, 0)("path", &FakeFileInfo{isDirectory: true, name: ".folder"}, nil) + shouldSkip := reader.createWalk(reader, 0, emptyProvisioned)("path", &FakeFileInfo{isDirectory: true, name: ".folder"}, nil) So(shouldSkip, ShouldEqual, filepath.SkipDir) }) Convey("should keep walking if file is not .json", func() { - shouldSkip := reader.createWalk(reader, 0)("path", &FakeFileInfo{isDirectory: true, name: "folder"}, nil) + shouldSkip := reader.createWalk(reader, 0, emptyProvisioned)("path", &FakeFileInfo{isDirectory: true, name: "folder"}, nil) So(shouldSkip, ShouldBeNil) }) }) diff --git a/pkg/services/sqlstore/dashboard.go b/pkg/services/sqlstore/dashboard.go index 4c72e3423be..2b94f0e089c 100644 --- a/pkg/services/sqlstore/dashboard.go +++ b/pkg/services/sqlstore/dashboard.go @@ -31,7 +31,6 @@ func saveDashboard(sess *DBSession, cmd *m.SaveDashboardCommand) error { // try get existing dashboard var existing, sameTitle m.Dashboard - if dash.Id > 0 { dashWithIdExists, err := sess.Where("id=? AND org_id=?", dash.Id, dash.OrgId).Get(&existing) if err != nil { From d62d5c74186ff3398660cf643c8bb176466c7381 Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 25 Jan 2018 15:25:07 +0100 Subject: [PATCH 05/12] provisioning: delete dashboards from db when file is missing --- pkg/models/dashboards.go | 5 ++ .../provisioning/dashboards/file_reader.go | 46 ++++++++++++++----- .../dashboards/file_reader_test.go | 5 +- pkg/services/sqlstore/dashboard.go | 1 + 4 files changed, 44 insertions(+), 13 deletions(-) diff --git a/pkg/models/dashboards.go b/pkg/models/dashboards.go index 577e97c657d..9e024d2db5e 100644 --- a/pkg/models/dashboards.go +++ b/pkg/models/dashboards.go @@ -58,6 +58,11 @@ type Dashboard struct { Data *simplejson.Json } +func (d *Dashboard) SetId(id int64) { + d.Id = id + d.Data.Set("id", id) +} + // NewDashboard creates a new dashboard func NewDashboard(title string) *Dashboard { dash := &Dashboard{} diff --git a/pkg/services/provisioning/dashboards/file_reader.go b/pkg/services/provisioning/dashboards/file_reader.go index dbbb44f15b8..085c6173df4 100644 --- a/pkg/services/provisioning/dashboards/file_reader.go +++ b/pkg/services/provisioning/dashboards/file_reader.go @@ -30,7 +30,7 @@ type fileReader struct { log log.Logger dashboardRepo dashboards.Repository cache *dashboardCache - createWalk func(fr *fileReader, folderId int64, provisionedDashboards map[string]*models.DashboardProvisioning) filepath.WalkFunc + createWalk func(fr *fileReader, folderId int64, provisionedDashboards map[string]*models.DashboardProvisioning, filesOnDisk map[string]bool) filepath.WalkFunc } func NewDashboardFileReader(cfg *DashboardsAsConfig, log log.Logger) (*fileReader, error) { @@ -103,7 +103,29 @@ func (fr *fileReader) startWalkingDisk() error { return err } - return filepath.Walk(fr.Path, fr.createWalk(fr, folderId, byPath)) + filesFoundOnDisk := map[string]bool{} + + err = filepath.Walk(fr.Path, fr.createWalk(fr, folderId, byPath, filesFoundOnDisk)) + + //delete dashboards without files + var dashboardToDelete []int64 + for path, provisioningData := range byPath { + _, existsInDatabase := filesFoundOnDisk[path] + if !existsInDatabase { + dashboardToDelete = append(dashboardToDelete, provisioningData.DashboardId) + } + } + + for _, dashboardId := range dashboardToDelete { + fr.log.Debug("deleting provisioned dashboard. missing on disk", "id", dashboardId) + cmd := &models.DeleteDashboardCommand{OrgId: fr.Cfg.OrgId, Id: dashboardId} + err := bus.Dispatch(cmd) + if err != nil { + return err + } + } + + return nil } func getProvisionedDashboardByPath(repo dashboards.Repository, name string) (map[string]*models.DashboardProvisioning, error) { @@ -169,7 +191,7 @@ func resolveSymlink(fileinfo os.FileInfo, path string) (os.FileInfo, error) { return fileinfo, err } -func createWalkFn(fr *fileReader, folderId int64, provisionedDashboards map[string]*models.DashboardProvisioning) filepath.WalkFunc { +func createWalkFn(fr *fileReader, folderId int64, provisionedDashboards map[string]*models.DashboardProvisioning, filesOnDisk map[string]bool) filepath.WalkFunc { return func(path string, fileInfo os.FileInfo, err error) error { if err != nil { return err @@ -185,6 +207,9 @@ func createWalkFn(fr *fileReader, folderId int64, provisionedDashboards map[stri return err } + // mark file as provisioned + filesOnDisk[path] = true + cachedDashboard, exist := fr.cache.getCache(path) if exist && cachedDashboard.UpdatedAt == resolvedFileInfo.ModTime() { return nil @@ -197,25 +222,24 @@ func createWalkFn(fr *fileReader, folderId int64, provisionedDashboards map[stri } var dbDashboard *models.Dashboard - cmd := &models.GetDashboardQuery{} + query := &models.GetDashboardQuery{} provisionedData, allReadyProvisioned := provisionedDashboards[path] - // see if the if allReadyProvisioned { - dash.Dashboard.Id = provisionedData.DashboardId - dash.Dashboard.Data.Set("id", provisionedData.DashboardId) - cmd.Id = provisionedData.DashboardId + dash.Dashboard.SetId(provisionedData.DashboardId) + + query.Id = provisionedData.DashboardId } else { if dash.Dashboard.Id != 0 { fr.log.Error("Cannot provision dashboard. Please remove the id property from the json file") return nil } - cmd.Slug = dash.Dashboard.Slug + query.Slug = dash.Dashboard.Slug } - err = bus.Dispatch(cmd) - dbDashboard = cmd.Result + err = bus.Dispatch(query) + dbDashboard = query.Result // if we don't have the dashboard in the db, save it! if err == models.ErrDashboardNotFound { diff --git a/pkg/services/provisioning/dashboards/file_reader_test.go b/pkg/services/provisioning/dashboards/file_reader_test.go index 390190e29af..cd7d9a03fd3 100644 --- a/pkg/services/provisioning/dashboards/file_reader_test.go +++ b/pkg/services/provisioning/dashboards/file_reader_test.go @@ -175,14 +175,15 @@ func TestDashboardFileReader(t *testing.T) { So(err, ShouldBeNil) emptyProvisioned := map[string]*models.DashboardProvisioning{} + noFiles := map[string]bool{} Convey("should skip dirs that starts with .", func() { - shouldSkip := reader.createWalk(reader, 0, emptyProvisioned)("path", &FakeFileInfo{isDirectory: true, name: ".folder"}, nil) + shouldSkip := reader.createWalk(reader, 0, emptyProvisioned, noFiles)("path", &FakeFileInfo{isDirectory: true, name: ".folder"}, nil) So(shouldSkip, ShouldEqual, filepath.SkipDir) }) Convey("should keep walking if file is not .json", func() { - shouldSkip := reader.createWalk(reader, 0, emptyProvisioned)("path", &FakeFileInfo{isDirectory: true, name: "folder"}, nil) + shouldSkip := reader.createWalk(reader, 0, emptyProvisioned, noFiles)("path", &FakeFileInfo{isDirectory: true, name: "folder"}, nil) So(shouldSkip, ShouldBeNil) }) }) diff --git a/pkg/services/sqlstore/dashboard.go b/pkg/services/sqlstore/dashboard.go index 2b94f0e089c..bc60165ca69 100644 --- a/pkg/services/sqlstore/dashboard.go +++ b/pkg/services/sqlstore/dashboard.go @@ -313,6 +313,7 @@ func DeleteDashboard(cmd *m.DeleteDashboardCommand) error { "DELETE FROM dashboard_version WHERE dashboard_id = ?", "DELETE FROM dashboard WHERE folder_id = ?", "DELETE FROM annotation WHERE dashboard_id = ?", + "DELETE FROM dashboard_provisioning WHERE dashboard_id = ?", } for _, sql := range deletes { From e0abd862e0714a060badd7c13e2a12c7b8b5915d Mon Sep 17 00:00:00 2001 From: Leonard Gram Date: Thu, 8 Feb 2018 16:36:53 +0100 Subject: [PATCH 06/12] provisioning: fixed bug in saving dashboards. --- pkg/services/dashboards/dashboards.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/dashboards/dashboards.go b/pkg/services/dashboards/dashboards.go index ce60fde0454..b0392f7944f 100644 --- a/pkg/services/dashboards/dashboards.go +++ b/pkg/services/dashboards/dashboards.go @@ -124,7 +124,7 @@ func (dr *DashboardRepository) SaveDashboard(dto *SaveDashboardDTO) (*models.Das return nil, err } - err = bus.Dispatch(&cmd) + err = bus.Dispatch(cmd) if err != nil { return nil, err } From 31f3fdcba39595340953508166f798b80956e621 Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 9 Feb 2018 12:17:58 +0100 Subject: [PATCH 07/12] provisioning: delete dashboards before insert/update --- .../dashboards/dashboard_cache.go | 6 +- .../provisioning/dashboards/file_reader.go | 145 ++++++++++-------- .../dashboards/file_reader_test.go | 10 +- .../folder-one/dashboard1.json | 3 +- .../folder-one/dashboard2.json | 3 +- 5 files changed, 89 insertions(+), 78 deletions(-) diff --git a/pkg/services/provisioning/dashboards/dashboard_cache.go b/pkg/services/provisioning/dashboards/dashboard_cache.go index 7f8f6fe490f..4135a18be39 100644 --- a/pkg/services/provisioning/dashboards/dashboard_cache.go +++ b/pkg/services/provisioning/dashboards/dashboard_cache.go @@ -18,7 +18,11 @@ func (fr *dashboardCache) addDashboardCache(key string, json *dashboards.SaveDas fr.internalCache.Add(key, json, time.Minute*10) } -func (fr *dashboardCache) getCache(key string) (*dashboards.SaveDashboardDTO, bool) { +func (fr *dashboardCache) deleteDashboard(key string) { + fr.internalCache.Delete(key) +} + +func (fr *dashboardCache) getDashboard(key string) (*dashboards.SaveDashboardDTO, bool) { obj, exist := fr.internalCache.Get(key) if !exist { return nil, exist diff --git a/pkg/services/provisioning/dashboards/file_reader.go b/pkg/services/provisioning/dashboards/file_reader.go index 085c6173df4..3dd35decc2c 100644 --- a/pkg/services/provisioning/dashboards/file_reader.go +++ b/pkg/services/provisioning/dashboards/file_reader.go @@ -30,7 +30,7 @@ type fileReader struct { log log.Logger dashboardRepo dashboards.Repository cache *dashboardCache - createWalk func(fr *fileReader, folderId int64, provisionedDashboards map[string]*models.DashboardProvisioning, filesOnDisk map[string]bool) filepath.WalkFunc + createWalk func(filesOnDisk map[string]os.FileInfo) filepath.WalkFunc } func NewDashboardFileReader(cfg *DashboardsAsConfig, log log.Logger) (*fileReader, error) { @@ -98,24 +98,26 @@ func (fr *fileReader) startWalkingDisk() error { return err } - byPath, err := getProvisionedDashboardByPath(fr.dashboardRepo, fr.Cfg.Name) + provisionedDashboardRefs, err := getProvisionedDashboardByPath(fr.dashboardRepo, fr.Cfg.Name) if err != nil { return err } - filesFoundOnDisk := map[string]bool{} + filesFoundOnDisk := map[string]os.FileInfo{} - err = filepath.Walk(fr.Path, fr.createWalk(fr, folderId, byPath, filesFoundOnDisk)) + err = filepath.Walk(fr.Path, fr.createWalk(filesFoundOnDisk)) - //delete dashboards without files + // find dashboards to delete since json file is missing var dashboardToDelete []int64 - for path, provisioningData := range byPath { + for path, provisioningData := range provisionedDashboardRefs { _, existsInDatabase := filesFoundOnDisk[path] if !existsInDatabase { dashboardToDelete = append(dashboardToDelete, provisioningData.DashboardId) + fr.cache.deleteDashboard(path) } } + // delete dashboard that are missing json file for _, dashboardId := range dashboardToDelete { fr.log.Debug("deleting provisioned dashboard. missing on disk", "id", dashboardId) cmd := &models.DeleteDashboardCommand{OrgId: fr.Cfg.OrgId, Id: dashboardId} @@ -125,9 +127,75 @@ func (fr *fileReader) startWalkingDisk() error { } } + // insert/update dashboards based on json files + for path, fileInfo := range filesFoundOnDisk { + err = fr.upsertDashboard(path, folderId, fileInfo, provisionedDashboardRefs) + if err != nil { + return err + } + } + return nil } +func (fr *fileReader) upsertDashboard(path string, folderId int64, fileInfo os.FileInfo, provisionedDashboardRefs map[string]*models.DashboardProvisioning) error { + resolvedFileInfo, err := resolveSymlink(fileInfo, path) + if err != nil { + return err + } + + cachedDashboard, exist := fr.cache.getDashboard(path) + if exist && cachedDashboard.UpdatedAt == resolvedFileInfo.ModTime() { + return nil + } + + dash, err := fr.readDashboardFromFile(path, folderId) + if err != nil { + fr.log.Error("failed to load dashboard from ", "file", path, "error", err) + return nil + } + + var dbDashboard *models.Dashboard + query := &models.GetDashboardQuery{} + provisionedData, allReadyProvisioned := provisionedDashboardRefs[path] + + if allReadyProvisioned { + dash.Dashboard.SetId(provisionedData.DashboardId) + + query.Id = provisionedData.DashboardId + } else { + if dash.Dashboard.Id != 0 { + fr.log.Error("Cannot provision dashboard. Please remove the id property from the json file") + return nil + } + + query.Slug = dash.Dashboard.Slug + } + + err = bus.Dispatch(query) + dbDashboard = query.Result + + // if we don't have the dashboard in the db, save it! + if err == models.ErrDashboardNotFound { + fr.log.Debug("saving new dashboard", "file", path) + err = saveDashboard(fr, path, dash) + return err + } + + if err != nil { + fr.log.Error("failed to query for dashboard", "slug", dash.Dashboard.Slug, "error", err) + return nil + } + + // break if db version is newer then fil version + if dbDashboard.Updated.Unix() >= resolvedFileInfo.ModTime().Unix() { + return nil + } + + fr.log.Debug("loading dashboard from disk into database.", "file", path) + return saveDashboard(fr, path, dash) +} + func getProvisionedDashboardByPath(repo dashboards.Repository, name string) (map[string]*models.DashboardProvisioning, error) { arr, err := repo.GetProvisionedDashboardData(name) if err != nil { @@ -191,7 +259,7 @@ func resolveSymlink(fileinfo os.FileInfo, path string) (os.FileInfo, error) { return fileinfo, err } -func createWalkFn(fr *fileReader, folderId int64, provisionedDashboards map[string]*models.DashboardProvisioning, filesOnDisk map[string]bool) filepath.WalkFunc { +func createWalkFn(filesOnDisk map[string]os.FileInfo) filepath.WalkFunc { return func(path string, fileInfo os.FileInfo, err error) error { if err != nil { return err @@ -202,66 +270,8 @@ func createWalkFn(fr *fileReader, folderId int64, provisionedDashboards map[stri return err } - resolvedFileInfo, err := resolveSymlink(fileInfo, path) - if err != nil { - return err - } - - // mark file as provisioned - filesOnDisk[path] = true - - cachedDashboard, exist := fr.cache.getCache(path) - if exist && cachedDashboard.UpdatedAt == resolvedFileInfo.ModTime() { - return nil - } - - dash, err := fr.readDashboardFromFile(path, folderId) - if err != nil { - fr.log.Error("failed to load dashboard from ", "file", path, "error", err) - return nil - } - - var dbDashboard *models.Dashboard - query := &models.GetDashboardQuery{} - provisionedData, allReadyProvisioned := provisionedDashboards[path] - - if allReadyProvisioned { - dash.Dashboard.SetId(provisionedData.DashboardId) - - query.Id = provisionedData.DashboardId - } else { - if dash.Dashboard.Id != 0 { - fr.log.Error("Cannot provision dashboard. Please remove the id property from the json file") - return nil - } - - query.Slug = dash.Dashboard.Slug - } - - err = bus.Dispatch(query) - dbDashboard = query.Result - - // if we don't have the dashboard in the db, save it! - if err == models.ErrDashboardNotFound { - fr.log.Debug("saving new dashboard", "file", path) - err = saveDashboard(fr, path, dash) - return err - } - - if err != nil { - fr.log.Error("failed to query for dashboard", "slug", dash.Dashboard.Slug, "error", err) - return nil - } - - // break if db version is newer then fil version - if dbDashboard.Updated.Unix() >= resolvedFileInfo.ModTime().Unix() { - return nil - } - - fr.log.Debug("loading dashboard from disk into database.", "file", path) - err = saveDashboard(fr, path, dash) - - return err + filesOnDisk[path] = fileInfo + return nil } } @@ -276,7 +286,6 @@ func saveDashboard(fr *fileReader, path string, dash *dashboards.SaveDashboardDT return err } - fr.cache.addDashboardCache(path, dash) return nil } diff --git a/pkg/services/provisioning/dashboards/file_reader_test.go b/pkg/services/provisioning/dashboards/file_reader_test.go index cd7d9a03fd3..1e1f2df1e78 100644 --- a/pkg/services/provisioning/dashboards/file_reader_test.go +++ b/pkg/services/provisioning/dashboards/file_reader_test.go @@ -62,8 +62,8 @@ func TestDashboardFileReader(t *testing.T) { } } - So(dashboards, ShouldEqual, 2) So(folders, ShouldEqual, 1) + So(dashboards, ShouldEqual, 2) }) Convey("Should not update dashboards when db is newer", func() { @@ -174,16 +174,15 @@ func TestDashboardFileReader(t *testing.T) { reader, err := NewDashboardFileReader(cfg, log.New("test-logger")) So(err, ShouldBeNil) - emptyProvisioned := map[string]*models.DashboardProvisioning{} - noFiles := map[string]bool{} + noFiles := map[string]os.FileInfo{} Convey("should skip dirs that starts with .", func() { - shouldSkip := reader.createWalk(reader, 0, emptyProvisioned, noFiles)("path", &FakeFileInfo{isDirectory: true, name: ".folder"}, nil) + shouldSkip := reader.createWalk(noFiles)("path", &FakeFileInfo{isDirectory: true, name: ".folder"}, nil) So(shouldSkip, ShouldEqual, filepath.SkipDir) }) Convey("should keep walking if file is not .json", func() { - shouldSkip := reader.createWalk(reader, 0, emptyProvisioned, noFiles)("path", &FakeFileInfo{isDirectory: true, name: "folder"}, nil) + shouldSkip := reader.createWalk(noFiles)("path", &FakeFileInfo{isDirectory: true, name: "folder"}, nil) So(shouldSkip, ShouldBeNil) }) }) @@ -260,6 +259,7 @@ func (repo *fakeDashboardRepo) GetProvisionedDashboardData(name string) ([]*mode func (repo *fakeDashboardRepo) SaveProvisionedDashboard(dto *dashboards.SaveDashboardDTO, provisioning *models.DashboardProvisioning) (*models.Dashboard, error) { repo.inserted = append(repo.inserted, dto) + repo.provisioned = append(repo.provisioned, provisioning) return dto.Dashboard, nil } diff --git a/pkg/services/provisioning/dashboards/test-dashboards/folder-one/dashboard1.json b/pkg/services/provisioning/dashboards/test-dashboards/folder-one/dashboard1.json index 5b6765a4ed6..febb98be0e8 100644 --- a/pkg/services/provisioning/dashboards/test-dashboards/folder-one/dashboard1.json +++ b/pkg/services/provisioning/dashboards/test-dashboards/folder-one/dashboard1.json @@ -1,5 +1,5 @@ { - "title": "Grafana", + "title": "Grafana1", "tags": [], "style": "dark", "timezone": "browser", @@ -170,4 +170,3 @@ }, "version": 5 } - \ No newline at end of file diff --git a/pkg/services/provisioning/dashboards/test-dashboards/folder-one/dashboard2.json b/pkg/services/provisioning/dashboards/test-dashboards/folder-one/dashboard2.json index 5b6765a4ed6..9291f16d9e7 100644 --- a/pkg/services/provisioning/dashboards/test-dashboards/folder-one/dashboard2.json +++ b/pkg/services/provisioning/dashboards/test-dashboards/folder-one/dashboard2.json @@ -1,5 +1,5 @@ { - "title": "Grafana", + "title": "Grafana2", "tags": [], "style": "dark", "timezone": "browser", @@ -170,4 +170,3 @@ }, "version": 5 } - \ No newline at end of file From 59cd2d5102dd6caf45279cce21407b2d63d3d237 Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 9 Feb 2018 15:23:38 +0100 Subject: [PATCH 08/12] provisioning: avoid caching and use updated field from db --- pkg/models/dashboard_provisioning.go | 1 - .../dashboards/dashboard_cache.go | 37 -------- .../provisioning/dashboards/file_reader.go | 84 ++++--------------- .../dashboards/file_reader_test.go | 17 ---- .../sqlstore/dashboard_provisioning.go | 6 +- .../sqlstore/migrations/dashboard_mig.go | 2 +- 6 files changed, 20 insertions(+), 127 deletions(-) delete mode 100644 pkg/models/dashboard_provisioning.go delete mode 100644 pkg/services/provisioning/dashboards/dashboard_cache.go diff --git a/pkg/models/dashboard_provisioning.go b/pkg/models/dashboard_provisioning.go deleted file mode 100644 index 2640e7f93ea..00000000000 --- a/pkg/models/dashboard_provisioning.go +++ /dev/null @@ -1 +0,0 @@ -package models diff --git a/pkg/services/provisioning/dashboards/dashboard_cache.go b/pkg/services/provisioning/dashboards/dashboard_cache.go deleted file mode 100644 index 4135a18be39..00000000000 --- a/pkg/services/provisioning/dashboards/dashboard_cache.go +++ /dev/null @@ -1,37 +0,0 @@ -package dashboards - -import ( - "github.com/grafana/grafana/pkg/services/dashboards" - gocache "github.com/patrickmn/go-cache" - "time" -) - -type dashboardCache struct { - internalCache *gocache.Cache -} - -func NewDashboardCache() *dashboardCache { - return &dashboardCache{internalCache: gocache.New(5*time.Minute, 30*time.Minute)} -} - -func (fr *dashboardCache) addDashboardCache(key string, json *dashboards.SaveDashboardDTO) { - fr.internalCache.Add(key, json, time.Minute*10) -} - -func (fr *dashboardCache) deleteDashboard(key string) { - fr.internalCache.Delete(key) -} - -func (fr *dashboardCache) getDashboard(key string) (*dashboards.SaveDashboardDTO, bool) { - obj, exist := fr.internalCache.Get(key) - if !exist { - return nil, exist - } - - dash, ok := obj.(*dashboards.SaveDashboardDTO) - if !ok { - return nil, ok - } - - return dash, ok -} diff --git a/pkg/services/provisioning/dashboards/file_reader.go b/pkg/services/provisioning/dashboards/file_reader.go index 3dd35decc2c..d753255744e 100644 --- a/pkg/services/provisioning/dashboards/file_reader.go +++ b/pkg/services/provisioning/dashboards/file_reader.go @@ -29,7 +29,6 @@ type fileReader struct { Path string log log.Logger dashboardRepo dashboards.Repository - cache *dashboardCache createWalk func(filesOnDisk map[string]os.FileInfo) filepath.WalkFunc } @@ -54,7 +53,6 @@ func NewDashboardFileReader(cfg *DashboardsAsConfig, log log.Logger) (*fileReade Path: path, log: log, dashboardRepo: dashboards.GetRepository(), - cache: NewDashboardCache(), createWalk: createWalkFn, }, nil } @@ -104,8 +102,10 @@ func (fr *fileReader) startWalkingDisk() error { } filesFoundOnDisk := map[string]os.FileInfo{} - err = filepath.Walk(fr.Path, fr.createWalk(filesFoundOnDisk)) + if err != nil { + return err + } // find dashboards to delete since json file is missing var dashboardToDelete []int64 @@ -113,7 +113,6 @@ func (fr *fileReader) startWalkingDisk() error { _, existsInDatabase := filesFoundOnDisk[path] if !existsInDatabase { dashboardToDelete = append(dashboardToDelete, provisioningData.DashboardId) - fr.cache.deleteDashboard(path) } } @@ -129,8 +128,9 @@ func (fr *fileReader) startWalkingDisk() error { // insert/update dashboards based on json files for path, fileInfo := range filesFoundOnDisk { - err = fr.upsertDashboard(path, folderId, fileInfo, provisionedDashboardRefs) + err = fr.saveDashboard(path, folderId, fileInfo, provisionedDashboardRefs) if err != nil { + fr.log.Error("Failed to save dashboard", "error", err) return err } } @@ -138,62 +138,31 @@ func (fr *fileReader) startWalkingDisk() error { return nil } -func (fr *fileReader) upsertDashboard(path string, folderId int64, fileInfo os.FileInfo, provisionedDashboardRefs map[string]*models.DashboardProvisioning) error { +func (fr *fileReader) saveDashboard(path string, folderId int64, fileInfo os.FileInfo, provisionedDashboardRefs map[string]*models.DashboardProvisioning) error { resolvedFileInfo, err := resolveSymlink(fileInfo, path) if err != nil { return err } - cachedDashboard, exist := fr.cache.getDashboard(path) - if exist && cachedDashboard.UpdatedAt == resolvedFileInfo.ModTime() { - return nil + provisionedData, allReadyProvisioned := provisionedDashboardRefs[path] + if allReadyProvisioned && provisionedData.Updated.Unix() == resolvedFileInfo.ModTime().Unix() { + return nil // dashboard is already in sync with the database } - dash, err := fr.readDashboardFromFile(path, folderId) + dash, err := fr.readDashboardFromFile(path, resolvedFileInfo.ModTime(), folderId) if err != nil { fr.log.Error("failed to load dashboard from ", "file", path, "error", err) return nil } - var dbDashboard *models.Dashboard - query := &models.GetDashboardQuery{} - provisionedData, allReadyProvisioned := provisionedDashboardRefs[path] - if allReadyProvisioned { dash.Dashboard.SetId(provisionedData.DashboardId) - - query.Id = provisionedData.DashboardId - } else { - if dash.Dashboard.Id != 0 { - fr.log.Error("Cannot provision dashboard. Please remove the id property from the json file") - return nil - } - - query.Slug = dash.Dashboard.Slug } - err = bus.Dispatch(query) - dbDashboard = query.Result - - // if we don't have the dashboard in the db, save it! - if err == models.ErrDashboardNotFound { - fr.log.Debug("saving new dashboard", "file", path) - err = saveDashboard(fr, path, dash) - return err - } - - if err != nil { - fr.log.Error("failed to query for dashboard", "slug", dash.Dashboard.Slug, "error", err) - return nil - } - - // break if db version is newer then fil version - if dbDashboard.Updated.Unix() >= resolvedFileInfo.ModTime().Unix() { - return nil - } - - fr.log.Debug("loading dashboard from disk into database.", "file", path) - return saveDashboard(fr, path, dash) + fr.log.Debug("saving new dashboard", "file", path) + dp := &models.DashboardProvisioning{ExternalId: path, Name: fr.Cfg.Name, Updated: resolvedFileInfo.ModTime()} + _, err = fr.dashboardRepo.SaveProvisionedDashboard(dash, dp) + return err } func getProvisionedDashboardByPath(repo dashboards.Repository, name string) (map[string]*models.DashboardProvisioning, error) { @@ -275,20 +244,6 @@ func createWalkFn(filesOnDisk map[string]os.FileInfo) filepath.WalkFunc { } } -func saveDashboard(fr *fileReader, path string, dash *dashboards.SaveDashboardDTO) error { - d := &models.DashboardProvisioning{ - ExternalId: path, - Name: fr.Cfg.Name, - } - - _, err := fr.dashboardRepo.SaveProvisionedDashboard(dash, d) - if err != nil { - return err - } - - return nil -} - func validateWalkablePath(fileInfo os.FileInfo) (bool, error) { if fileInfo.IsDir() { if strings.HasPrefix(fileInfo.Name(), ".") { @@ -304,7 +259,7 @@ func validateWalkablePath(fileInfo os.FileInfo) (bool, error) { return true, nil } -func (fr *fileReader) readDashboardFromFile(path string, folderId int64) (*dashboards.SaveDashboardDTO, error) { +func (fr *fileReader) readDashboardFromFile(path string, lastModified time.Time, folderId int64) (*dashboards.SaveDashboardDTO, error) { reader, err := os.Open(path) if err != nil { return nil, err @@ -316,17 +271,10 @@ func (fr *fileReader) readDashboardFromFile(path string, folderId int64) (*dashb return nil, err } - stat, err := os.Stat(path) + dash, err := createDashboardJson(data, lastModified, fr.Cfg, folderId) if err != nil { return nil, err } - dash, err := createDashboardJson(data, stat.ModTime(), fr.Cfg, folderId) - if err != nil { - return nil, err - } - - fr.cache.addDashboardCache(path, dash) - return dash, nil } diff --git a/pkg/services/provisioning/dashboards/file_reader_test.go b/pkg/services/provisioning/dashboards/file_reader_test.go index 1e1f2df1e78..863a63ba156 100644 --- a/pkg/services/provisioning/dashboards/file_reader_test.go +++ b/pkg/services/provisioning/dashboards/file_reader_test.go @@ -66,23 +66,6 @@ func TestDashboardFileReader(t *testing.T) { So(dashboards, ShouldEqual, 2) }) - Convey("Should not update dashboards when db is newer", func() { - cfg.Options["path"] = oneDashboard - - fakeRepo.getDashboard = append(fakeRepo.getDashboard, &models.Dashboard{ - Updated: time.Now().Add(time.Hour), - Slug: "grafana", - }) - - reader, err := NewDashboardFileReader(cfg, logger) - So(err, ShouldBeNil) - - err = reader.startWalkingDisk() - So(err, ShouldBeNil) - - So(len(fakeRepo.inserted), ShouldEqual, 0) - }) - Convey("Can read default dashboard and replace old version in database", func() { cfg.Options["path"] = oneDashboard diff --git a/pkg/services/sqlstore/dashboard_provisioning.go b/pkg/services/sqlstore/dashboard_provisioning.go index c064f78ce67..54068334b4b 100644 --- a/pkg/services/sqlstore/dashboard_provisioning.go +++ b/pkg/services/sqlstore/dashboard_provisioning.go @@ -1,8 +1,6 @@ package sqlstore import ( - "time" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/models" ) @@ -28,6 +26,9 @@ func SaveProvisionedDashboard(cmd *models.SaveProvisionedDashboardCommand) error } cmd.Result = cmd.DashboardCmd.Result + if cmd.DashboardProvisioning.Updated.IsZero() { + cmd.DashboardProvisioning.Updated = cmd.Result.Updated + } return saveProvionedData(sess, cmd.DashboardProvisioning, cmd.Result) }) @@ -42,7 +43,6 @@ func saveProvionedData(sess *DBSession, cmd *models.DashboardProvisioning, dashb } cmd.Id = result.Id - cmd.Updated = time.Now() cmd.DashboardId = dashboard.Id if exist { diff --git a/pkg/services/sqlstore/migrations/dashboard_mig.go b/pkg/services/sqlstore/migrations/dashboard_mig.go index c281c6a8121..1c40e241e15 100644 --- a/pkg/services/sqlstore/migrations/dashboard_mig.go +++ b/pkg/services/sqlstore/migrations/dashboard_mig.go @@ -183,7 +183,7 @@ func addDashboardMigration(mg *Migrator) { {Name: "dashboard_id", Type: DB_BigInt, Nullable: true}, {Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false}, {Name: "external_id", Type: DB_Text, Nullable: false}, - {Name: "updated", Type: DB_Int, Nullable: false}, + {Name: "updated", Type: DB_DateTime, Nullable: false}, }, Indices: []*Index{ {Cols: []string{"dashboard_id"}}, From e93fe9db25cd4b710e7490ffa8e0aa3d777907ff Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 9 Feb 2018 15:25:31 +0100 Subject: [PATCH 09/12] provisioning: update sample config to use path --- conf/provisioning/dashboards/sample.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/provisioning/dashboards/sample.yaml b/conf/provisioning/dashboards/sample.yaml index e40612af508..f0dcca9b47a 100644 --- a/conf/provisioning/dashboards/sample.yaml +++ b/conf/provisioning/dashboards/sample.yaml @@ -3,4 +3,4 @@ # folder: '' # type: file # options: -# folder: /var/lib/grafana/dashboards +# path: /var/lib/grafana/dashboards From 5a7662400321227e88d88ae1895e011419c2d922 Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 9 Feb 2018 15:33:54 +0100 Subject: [PATCH 10/12] provisioning: createWalkFn doesnt have to be attached to the filereader anymore --- .../provisioning/dashboards/file_reader.go | 4 +--- .../provisioning/dashboards/file_reader_test.go | 17 ++--------------- 2 files changed, 3 insertions(+), 18 deletions(-) diff --git a/pkg/services/provisioning/dashboards/file_reader.go b/pkg/services/provisioning/dashboards/file_reader.go index d753255744e..082992c4b75 100644 --- a/pkg/services/provisioning/dashboards/file_reader.go +++ b/pkg/services/provisioning/dashboards/file_reader.go @@ -29,7 +29,6 @@ type fileReader struct { Path string log log.Logger dashboardRepo dashboards.Repository - createWalk func(filesOnDisk map[string]os.FileInfo) filepath.WalkFunc } func NewDashboardFileReader(cfg *DashboardsAsConfig, log log.Logger) (*fileReader, error) { @@ -53,7 +52,6 @@ func NewDashboardFileReader(cfg *DashboardsAsConfig, log log.Logger) (*fileReade Path: path, log: log, dashboardRepo: dashboards.GetRepository(), - createWalk: createWalkFn, }, nil } @@ -102,7 +100,7 @@ func (fr *fileReader) startWalkingDisk() error { } filesFoundOnDisk := map[string]os.FileInfo{} - err = filepath.Walk(fr.Path, fr.createWalk(filesFoundOnDisk)) + err = filepath.Walk(fr.Path, createWalkFn(filesFoundOnDisk)) if err != nil { return err } diff --git a/pkg/services/provisioning/dashboards/file_reader_test.go b/pkg/services/provisioning/dashboards/file_reader_test.go index 863a63ba156..a81b502c50a 100644 --- a/pkg/services/provisioning/dashboards/file_reader_test.go +++ b/pkg/services/provisioning/dashboards/file_reader_test.go @@ -144,28 +144,15 @@ func TestDashboardFileReader(t *testing.T) { }) Convey("Walking the folder with dashboards", func() { - cfg := &DashboardsAsConfig{ - Name: "Default", - Type: "file", - OrgId: 1, - Folder: "", - Options: map[string]interface{}{ - "path": defaultDashboards, - }, - } - - reader, err := NewDashboardFileReader(cfg, log.New("test-logger")) - So(err, ShouldBeNil) - noFiles := map[string]os.FileInfo{} Convey("should skip dirs that starts with .", func() { - shouldSkip := reader.createWalk(noFiles)("path", &FakeFileInfo{isDirectory: true, name: ".folder"}, nil) + shouldSkip := createWalkFn(noFiles)("path", &FakeFileInfo{isDirectory: true, name: ".folder"}, nil) So(shouldSkip, ShouldEqual, filepath.SkipDir) }) Convey("should keep walking if file is not .json", func() { - shouldSkip := reader.createWalk(noFiles)("path", &FakeFileInfo{isDirectory: true, name: "folder"}, nil) + shouldSkip := createWalkFn(noFiles)("path", &FakeFileInfo{isDirectory: true, name: "folder"}, nil) So(shouldSkip, ShouldBeNil) }) }) From 84fea7c148cac1217ff4d32d01963040009320da Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 9 Feb 2018 15:43:58 +0100 Subject: [PATCH 11/12] provisioning: dont return error unless you want to cancel all operations --- .../provisioning/dashboards/file_reader.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/pkg/services/provisioning/dashboards/file_reader.go b/pkg/services/provisioning/dashboards/file_reader.go index 082992c4b75..4193fc5fdc2 100644 --- a/pkg/services/provisioning/dashboards/file_reader.go +++ b/pkg/services/provisioning/dashboards/file_reader.go @@ -120,16 +120,15 @@ func (fr *fileReader) startWalkingDisk() error { cmd := &models.DeleteDashboardCommand{OrgId: fr.Cfg.OrgId, Id: dashboardId} err := bus.Dispatch(cmd) if err != nil { - return err + fr.log.Error("failed to delete dashboard", "id", cmd.Id) } } - // insert/update dashboards based on json files + // save dashboards based on json files for path, fileInfo := range filesFoundOnDisk { err = fr.saveDashboard(path, folderId, fileInfo, provisionedDashboardRefs) if err != nil { - fr.log.Error("Failed to save dashboard", "error", err) - return err + fr.log.Error("failed to save dashboard", "error", err) } } @@ -142,8 +141,8 @@ func (fr *fileReader) saveDashboard(path string, folderId int64, fileInfo os.Fil return err } - provisionedData, allReadyProvisioned := provisionedDashboardRefs[path] - if allReadyProvisioned && provisionedData.Updated.Unix() == resolvedFileInfo.ModTime().Unix() { + provisionedData, alreadyProvisioned := provisionedDashboardRefs[path] + if alreadyProvisioned && provisionedData.Updated.Unix() == resolvedFileInfo.ModTime().Unix() { return nil // dashboard is already in sync with the database } @@ -153,7 +152,12 @@ func (fr *fileReader) saveDashboard(path string, folderId int64, fileInfo os.Fil return nil } - if allReadyProvisioned { + if dash.Dashboard.Id != 0 { + fr.log.Error("provisioned dashboard json files cannot contain id") + return nil + } + + if alreadyProvisioned { dash.Dashboard.SetId(provisionedData.DashboardId) } From f01890c3ff622f2d8d1b7fb4f30075fa74935d93 Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 12 Feb 2018 08:22:34 +0100 Subject: [PATCH 12/12] provisioning: better variable naming --- pkg/services/provisioning/dashboards/file_reader.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/services/provisioning/dashboards/file_reader.go b/pkg/services/provisioning/dashboards/file_reader.go index 4193fc5fdc2..c67f355a36e 100644 --- a/pkg/services/provisioning/dashboards/file_reader.go +++ b/pkg/services/provisioning/dashboards/file_reader.go @@ -108,8 +108,8 @@ func (fr *fileReader) startWalkingDisk() error { // find dashboards to delete since json file is missing var dashboardToDelete []int64 for path, provisioningData := range provisionedDashboardRefs { - _, existsInDatabase := filesFoundOnDisk[path] - if !existsInDatabase { + _, existsOnDisk := filesFoundOnDisk[path] + if !existsOnDisk { dashboardToDelete = append(dashboardToDelete, provisioningData.DashboardId) } }