public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

provisioning: delete dashboards before insert/update

Browse Source
This commit is contained in:
bergquist
2018-02-09 12:17:58 +01:00
parent e0abd862e0
commit 31f3fdcba3
5 changed files with 89 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/provisioning/dashboards/dashboard_cache.go
+5 -1
View File
@@ -18,7 +18,11 @@ func (fr *dashboardCache) addDashboardCache(key string, json *dashboards.SaveDas
fr.internalCache.Add(key, json, time.Minute*10)
}
func (fr *dashboardCache) getCache(key string) (*dashboards.SaveDashboardDTO, bool) {
func (fr *dashboardCache) deleteDashboard(key string) {
fr.internalCache.Delete(key)
}
func (fr *dashboardCache) getDashboard(key string) (*dashboards.SaveDashboardDTO, bool) {
obj, exist := fr.internalCache.Get(key)
if !exist {
return nil, exist
pkg/services/provisioning/dashboards/file_reader.go
+77 -68
View File
@@ -30,7 +30,7 @@ type fileReader struct {
log log.Logger
dashboardRepo dashboards.Repository
cache *dashboardCache
createWalk func(fr *fileReader, folderId int64, provisionedDashboards map[string]*models.DashboardProvisioning, filesOnDisk map[string]bool) filepath.WalkFunc
createWalk func(filesOnDisk map[string]os.FileInfo) filepath.WalkFunc
}
func NewDashboardFileReader(cfg *DashboardsAsConfig, log log.Logger) (*fileReader, error) {
@@ -98,24 +98,26 @@ func (fr *fileReader) startWalkingDisk() error {
return err
}
byPath, err := getProvisionedDashboardByPath(fr.dashboardRepo, fr.Cfg.Name)
provisionedDashboardRefs, err := getProvisionedDashboardByPath(fr.dashboardRepo, fr.Cfg.Name)
if err != nil {
return err
}
filesFoundOnDisk := map[string]bool{}
filesFoundOnDisk := map[string]os.FileInfo{}
err = filepath.Walk(fr.Path, fr.createWalk(fr, folderId, byPath, filesFoundOnDisk))
err = filepath.Walk(fr.Path, fr.createWalk(filesFoundOnDisk))
//delete dashboards without files
// find dashboards to delete since json file is missing
var dashboardToDelete []int64
for path, provisioningData := range byPath {
for path, provisioningData := range provisionedDashboardRefs {
_, existsInDatabase := filesFoundOnDisk[path]
if !existsInDatabase {
dashboardToDelete = append(dashboardToDelete, provisioningData.DashboardId)
fr.cache.deleteDashboard(path)
}
}
// delete dashboard that are missing json file
for _, dashboardId := range dashboardToDelete {
fr.log.Debug("deleting provisioned dashboard. missing on disk", "id", dashboardId)
cmd := &models.DeleteDashboardCommand{OrgId: fr.Cfg.OrgId, Id: dashboardId}
@@ -125,9 +127,75 @@ func (fr *fileReader) startWalkingDisk() error {
}
}
// insert/update dashboards based on json files
for path, fileInfo := range filesFoundOnDisk {
err = fr.upsertDashboard(path, folderId, fileInfo, provisionedDashboardRefs)
if err != nil {
return err
}
}
return nil
}
func (fr *fileReader) upsertDashboard(path string, folderId int64, fileInfo os.FileInfo, provisionedDashboardRefs map[string]*models.DashboardProvisioning) error {
resolvedFileInfo, err := resolveSymlink(fileInfo, path)
if err != nil {
return err
}
cachedDashboard, exist := fr.cache.getDashboard(path)
if exist && cachedDashboard.UpdatedAt == resolvedFileInfo.ModTime() {
return nil
}
dash, err := fr.readDashboardFromFile(path, folderId)
if err != nil {
fr.log.Error("failed to load dashboard from ", "file", path, "error", err)
return nil
}
var dbDashboard *models.Dashboard
query := &models.GetDashboardQuery{}
provisionedData, allReadyProvisioned := provisionedDashboardRefs[path]
if allReadyProvisioned {
dash.Dashboard.SetId(provisionedData.DashboardId)
query.Id = provisionedData.DashboardId
} else {
if dash.Dashboard.Id != 0 {
fr.log.Error("Cannot provision dashboard. Please remove the id property from the json file")
return nil
}
query.Slug = dash.Dashboard.Slug
}
err = bus.Dispatch(query)
dbDashboard = query.Result
// if we don't have the dashboard in the db, save it!
if err == models.ErrDashboardNotFound {
fr.log.Debug("saving new dashboard", "file", path)
err = saveDashboard(fr, path, dash)
return err
}
if err != nil {
fr.log.Error("failed to query for dashboard", "slug", dash.Dashboard.Slug, "error", err)
return nil
}
// break if db version is newer then fil version
if dbDashboard.Updated.Unix() >= resolvedFileInfo.ModTime().Unix() {
return nil
}
fr.log.Debug("loading dashboard from disk into database.", "file", path)
return saveDashboard(fr, path, dash)
}
func getProvisionedDashboardByPath(repo dashboards.Repository, name string) (map[string]*models.DashboardProvisioning, error) {
arr, err := repo.GetProvisionedDashboardData(name)
if err != nil {
@@ -191,7 +259,7 @@ func resolveSymlink(fileinfo os.FileInfo, path string) (os.FileInfo, error) {
return fileinfo, err
}
func createWalkFn(fr *fileReader, folderId int64, provisionedDashboards map[string]*models.DashboardProvisioning, filesOnDisk map[string]bool) filepath.WalkFunc {
func createWalkFn(filesOnDisk map[string]os.FileInfo) filepath.WalkFunc {
return func(path string, fileInfo os.FileInfo, err error) error {
if err != nil {
return err
@@ -202,66 +270,8 @@ func createWalkFn(fr *fileReader, folderId int64, provisionedDashboards map[stri
return err
}
resolvedFileInfo, err := resolveSymlink(fileInfo, path)
if err != nil {
return err
}
// mark file as provisioned
filesOnDisk[path] = true
cachedDashboard, exist := fr.cache.getCache(path)
if exist && cachedDashboard.UpdatedAt == resolvedFileInfo.ModTime() {
return nil
}
dash, err := fr.readDashboardFromFile(path, folderId)
if err != nil {
fr.log.Error("failed to load dashboard from ", "file", path, "error", err)
return nil
}
var dbDashboard *models.Dashboard
query := &models.GetDashboardQuery{}
provisionedData, allReadyProvisioned := provisionedDashboards[path]
if allReadyProvisioned {
dash.Dashboard.SetId(provisionedData.DashboardId)
query.Id = provisionedData.DashboardId
} else {
if dash.Dashboard.Id != 0 {
fr.log.Error("Cannot provision dashboard. Please remove the id property from the json file")
return nil
}
query.Slug = dash.Dashboard.Slug
}
err = bus.Dispatch(query)
dbDashboard = query.Result
// if we don't have the dashboard in the db, save it!
if err == models.ErrDashboardNotFound {
fr.log.Debug("saving new dashboard", "file", path)
err = saveDashboard(fr, path, dash)
return err
}
if err != nil {
fr.log.Error("failed to query for dashboard", "slug", dash.Dashboard.Slug, "error", err)
return nil
}
// break if db version is newer then fil version
if dbDashboard.Updated.Unix() >= resolvedFileInfo.ModTime().Unix() {
return nil
}
fr.log.Debug("loading dashboard from disk into database.", "file", path)
err = saveDashboard(fr, path, dash)
return err
filesOnDisk[path] = fileInfo
return nil
}
}
@@ -276,7 +286,6 @@ func saveDashboard(fr *fileReader, path string, dash *dashboards.SaveDashboardDT
return err
}
fr.cache.addDashboardCache(path, dash)
return nil
}
pkg/services/provisioning/dashboards/file_reader_test.go
+5 -5
View File
@@ -62,8 +62,8 @@ func TestDashboardFileReader(t *testing.T) {
}
}
So(dashboards, ShouldEqual, 2)
So(folders, ShouldEqual, 1)
So(dashboards, ShouldEqual, 2)
})
Convey("Should not update dashboards when db is newer", func() {
@@ -174,16 +174,15 @@ func TestDashboardFileReader(t *testing.T) {
reader, err := NewDashboardFileReader(cfg, log.New("test-logger"))
So(err, ShouldBeNil)
emptyProvisioned := map[string]*models.DashboardProvisioning{}
noFiles := map[string]bool{}
noFiles := map[string]os.FileInfo{}
Convey("should skip dirs that starts with .", func() {
shouldSkip := reader.createWalk(reader, 0, emptyProvisioned, noFiles)("path", &FakeFileInfo{isDirectory: true, name: ".folder"}, nil)
shouldSkip := reader.createWalk(noFiles)("path", &FakeFileInfo{isDirectory: true, name: ".folder"}, nil)
So(shouldSkip, ShouldEqual, filepath.SkipDir)
})
Convey("should keep walking if file is not .json", func() {
shouldSkip := reader.createWalk(reader, 0, emptyProvisioned, noFiles)("path", &FakeFileInfo{isDirectory: true, name: "folder"}, nil)
shouldSkip := reader.createWalk(noFiles)("path", &FakeFileInfo{isDirectory: true, name: "folder"}, nil)
So(shouldSkip, ShouldBeNil)
})
})
@@ -260,6 +259,7 @@ func (repo *fakeDashboardRepo) GetProvisionedDashboardData(name string) ([]*mode
func (repo *fakeDashboardRepo) SaveProvisionedDashboard(dto *dashboards.SaveDashboardDTO, provisioning *models.DashboardProvisioning) (*models.Dashboard, error) {
repo.inserted = append(repo.inserted, dto)
repo.provisioned = append(repo.provisioned, provisioning)
return dto.Dashboard, nil
}
pkg/services/provisioning/dashboards/test-dashboards/folder-one/dashboard1.json
+1 -2
View File
@@ -1,5 +1,5 @@
{
"title": "Grafana",
"title": "Grafana1",
"tags": [],
"style": "dark",
"timezone": "browser",
@@ -170,4 +170,3 @@
},
"version": 5
}
pkg/services/provisioning/dashboards/test-dashboards/folder-one/dashboard2.json
+1 -2
View File
@@ -1,5 +1,5 @@
{
"title": "Grafana",
"title": "Grafana2",
"tags": [],
"style": "dark",
"timezone": "browser",
@@ -170,4 +170,3 @@
},
"version": 5
}