diff --git a/pkg/models/dashboards.go b/pkg/models/dashboards.go index 51df2e55a57..091f27ec413 100644 --- a/pkg/models/dashboards.go +++ b/pkg/models/dashboards.go @@ -139,8 +139,12 @@ func (dash *Dashboard) GetString(prop string, defaultValue string) string { // UpdateSlug updates the slug func (dash *Dashboard) UpdateSlug() { - title := strings.ToLower(dash.Data.Get("title").MustString()) - dash.Slug = slug.Make(title) + title := dash.Data.Get("title").MustString() + dash.Slug = SlugifyTitle(title) +} + +func SlugifyTitle(title string) string { + return slug.Make(strings.ToLower(title)) } // diff --git a/pkg/models/dashboards_test.go b/pkg/models/dashboards_test.go index 0ec773dfb97..ad865b575bb 100644 --- a/pkg/models/dashboards_test.go +++ b/pkg/models/dashboards_test.go @@ -16,6 +16,12 @@ func TestDashboardModel(t *testing.T) { So(dashboard.Slug, ShouldEqual, "grafana-play-home") }) + Convey("Can slugify title", t, func() { + slug := SlugifyTitle("Grafana Play Home") + + So(slug, ShouldEqual, "grafana-play-home") + }) + Convey("Given a dashboard json", t, func() { json := simplejson.New() json.Set("title", "test dash") diff --git a/pkg/services/provisioning/dashboards/dashboard_cache.go b/pkg/services/provisioning/dashboards/dashboard_cache.go new file mode 100644 index 00000000000..da6b7e8a5e8 --- /dev/null +++ b/pkg/services/provisioning/dashboards/dashboard_cache.go @@ -0,0 +1,33 @@ +package dashboards + +import ( + "github.com/grafana/grafana/pkg/services/dashboards" + gocache "github.com/patrickmn/go-cache" + "time" +) + +type dashboardCache struct { + internalCache *gocache.Cache +} + +func NewDashboardCache() *dashboardCache { + return &dashboardCache{internalCache: gocache.New(5*time.Minute, 30*time.Minute)} +} + +func (fr *dashboardCache) addDashboardCache(key string, json *dashboards.SaveDashboardItem) { + fr.internalCache.Add(key, json, time.Minute*10) +} + +func (fr *dashboardCache) getCache(key string) (*dashboards.SaveDashboardItem, bool) { + obj, exist := fr.internalCache.Get(key) + if !exist { + return nil, exist + } + + dash, ok := obj.(*dashboards.SaveDashboardItem) + if !ok { + return nil, ok + } + + return dash, ok +} diff --git a/pkg/services/provisioning/dashboards/file_reader.go b/pkg/services/provisioning/dashboards/file_reader.go index 22fe11b45c1..eb3085296fd 100644 --- a/pkg/services/provisioning/dashboards/file_reader.go +++ b/pkg/services/provisioning/dashboards/file_reader.go @@ -2,6 +2,7 @@ package dashboards import ( "context" + "errors" "fmt" "os" "path/filepath" @@ -15,7 +16,12 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/models" - gocache "github.com/patrickmn/go-cache" +) + +var ( + checkDiskForChangesInterval time.Duration = time.Second * 3 + + ErrFolderNameMissing error = errors.New("Folder name missing") ) type fileReader struct { @@ -23,7 +29,8 @@ type fileReader struct { Path string log log.Logger dashboardRepo dashboards.Repository - cache *gocache.Cache + cache *dashboardCache + createWalk func(fr *fileReader, folderId int64) filepath.WalkFunc } func NewDashboardFileReader(cfg *DashboardsAsConfig, log log.Logger) (*fileReader, error) { @@ -41,32 +48,15 @@ func NewDashboardFileReader(cfg *DashboardsAsConfig, log log.Logger) (*fileReade Path: path, log: log, dashboardRepo: dashboards.GetRepository(), - cache: gocache.New(5*time.Minute, 30*time.Minute), + cache: NewDashboardCache(), + createWalk: createWalkFn, }, nil } -func (fr *fileReader) addCache(key string, json *dashboards.SaveDashboardItem) { - fr.cache.Add(key, json, time.Minute*10) -} - -func (fr *fileReader) getCache(key string) (*dashboards.SaveDashboardItem, bool) { - obj, exist := fr.cache.Get(key) - if !exist { - return nil, exist - } - - dash, ok := obj.(*dashboards.SaveDashboardItem) - if !ok { - return nil, ok - } - - return dash, ok -} - func (fr *fileReader) ReadAndListen(ctx context.Context) error { - ticker := time.NewTicker(time.Second * 3) + ticker := time.NewTicker(checkDiskForChangesInterval) - if err := fr.walkFolder(); err != nil { + if err := fr.startWalkingDisk(); err != nil { fr.log.Error("failed to search for dashboards", "error", err) } @@ -78,7 +68,9 @@ func (fr *fileReader) ReadAndListen(ctx context.Context) error { if !running { // avoid walking the filesystem in parallel. incase fs is very slow. running = true go func() { - fr.walkFolder() + if err := fr.startWalkingDisk(); err != nil { + fr.log.Error("failed to search for dashboards", "error", err) + } running = false }() } @@ -88,14 +80,57 @@ func (fr *fileReader) ReadAndListen(ctx context.Context) error { } } -func (fr *fileReader) walkFolder() error { +func (fr *fileReader) startWalkingDisk() error { if _, err := os.Stat(fr.Path); err != nil { if os.IsNotExist(err) { return err } } - return filepath.Walk(fr.Path, func(path string, fileInfo os.FileInfo, err error) error { + folderId, err := getOrCreateFolderId(fr.Cfg, fr.dashboardRepo) + if err != nil && err != ErrFolderNameMissing { + return err + } + + return filepath.Walk(fr.Path, fr.createWalk(fr, folderId)) +} + +func getOrCreateFolderId(cfg *DashboardsAsConfig, repo dashboards.Repository) (int64, error) { + if cfg.Folder == "" { + return 0, ErrFolderNameMissing + } + + cmd := &models.GetDashboardQuery{Slug: models.SlugifyTitle(cfg.Folder), OrgId: cfg.OrgId} + err := bus.Dispatch(cmd) + + if err != nil && err != models.ErrDashboardNotFound { + return 0, err + } + + // dashboard folder not found. create one. + if err == models.ErrDashboardNotFound { + dash := &dashboards.SaveDashboardItem{} + dash.Dashboard = models.NewDashboard(cfg.Folder) + dash.Dashboard.IsFolder = true + dash.Overwrite = true + dash.OrgId = cfg.OrgId + dbDash, err := repo.SaveDashboard(dash) + if err != nil { + return 0, err + } + + return dbDash.Id, nil + } + + if !cmd.Result.IsFolder { + return 0, fmt.Errorf("Got invalid response. Expected folder, found dashboard") + } + + return cmd.Result.Id, nil +} + +func createWalkFn(fr *fileReader, folderId int64) filepath.WalkFunc { + return func(path string, fileInfo os.FileInfo, err error) error { if err != nil { return err } @@ -110,12 +145,12 @@ func (fr *fileReader) walkFolder() error { return nil } - cachedDashboard, exist := fr.getCache(path) + cachedDashboard, exist := fr.cache.getCache(path) if exist && cachedDashboard.UpdatedAt == fileInfo.ModTime() { return nil } - dash, err := fr.readDashboardFromFile(path) + dash, err := fr.readDashboardFromFile(path, folderId) if err != nil { fr.log.Error("failed to load dashboard from ", "file", path, "error", err) return nil @@ -147,10 +182,10 @@ func (fr *fileReader) walkFolder() error { fr.log.Debug("loading dashboard from disk into database.", "file", path) _, err = fr.dashboardRepo.SaveDashboard(dash) return err - }) + } } -func (fr *fileReader) readDashboardFromFile(path string) (*dashboards.SaveDashboardItem, error) { +func (fr *fileReader) readDashboardFromFile(path string, folderId int64) (*dashboards.SaveDashboardItem, error) { reader, err := os.Open(path) if err != nil { return nil, err @@ -167,12 +202,12 @@ func (fr *fileReader) readDashboardFromFile(path string) (*dashboards.SaveDashbo return nil, err } - dash, err := createDashboardJson(data, stat.ModTime(), fr.Cfg) + dash, err := createDashboardJson(data, stat.ModTime(), fr.Cfg, folderId) if err != nil { return nil, err } - fr.addCache(path, dash) + fr.cache.addDashboardCache(path, dash) return dash, nil } diff --git a/pkg/services/provisioning/dashboards/file_reader_test.go b/pkg/services/provisioning/dashboards/file_reader_test.go index e91f278322a..16e3e1184b8 100644 --- a/pkg/services/provisioning/dashboards/file_reader_test.go +++ b/pkg/services/provisioning/dashboards/file_reader_test.go @@ -2,6 +2,7 @@ package dashboards import ( "os" + "path/filepath" "testing" "time" @@ -22,7 +23,7 @@ var ( ) func TestDashboardFileReader(t *testing.T) { - Convey("Reading dashboards from disk", t, func() { + Convey("Dashboard file reader", t, func() { bus.ClearBusHandlers() fakeRepo = &fakeDashboardRepo{} @@ -30,91 +31,191 @@ func TestDashboardFileReader(t *testing.T) { dashboards.SetRepository(fakeRepo) logger := log.New("test.logger") - cfg := &DashboardsAsConfig{ - Name: "Default", - Type: "file", - OrgId: 1, - Folder: "", - Options: map[string]interface{}{}, - } + Convey("Reading dashboards from disk", func() { - Convey("Can read default dashboard", func() { - cfg.Options["folder"] = defaultDashboards - - reader, err := NewDashboardFileReader(cfg, logger) - So(err, ShouldBeNil) - - err = reader.walkFolder() - So(err, ShouldBeNil) - - So(len(fakeRepo.inserted), ShouldEqual, 2) - }) - - Convey("Should not update dashboards when db is newer", func() { - cfg.Options["folder"] = oneDashboard - - fakeRepo.getDashboard = append(fakeRepo.getDashboard, &models.Dashboard{ - Updated: time.Now().Add(time.Hour), - Slug: "grafana", - }) - - reader, err := NewDashboardFileReader(cfg, logger) - So(err, ShouldBeNil) - - err = reader.walkFolder() - So(err, ShouldBeNil) - - So(len(fakeRepo.inserted), ShouldEqual, 0) - }) - - Convey("Can read default dashboard and replace old version in database", func() { - cfg.Options["folder"] = oneDashboard - - stat, _ := os.Stat(oneDashboard + "/dashboard1.json") - - fakeRepo.getDashboard = append(fakeRepo.getDashboard, &models.Dashboard{ - Updated: stat.ModTime().AddDate(0, 0, -1), - Slug: "grafana", - }) - - reader, err := NewDashboardFileReader(cfg, logger) - So(err, ShouldBeNil) - - err = reader.walkFolder() - So(err, ShouldBeNil) - - So(len(fakeRepo.inserted), ShouldEqual, 1) - }) - - Convey("Invalid configuration should return error", func() { cfg := &DashboardsAsConfig{ - Name: "Default", - Type: "file", - OrgId: 1, - Folder: "", + Name: "Default", + Type: "file", + OrgId: 1, + Folder: "", + Options: map[string]interface{}{}, } - _, err := NewDashboardFileReader(cfg, logger) - So(err, ShouldNotBeNil) + Convey("Can read default dashboard", func() { + cfg.Options["folder"] = defaultDashboards + cfg.Folder = "Team A" + + reader, err := NewDashboardFileReader(cfg, logger) + So(err, ShouldBeNil) + + err = reader.startWalkingDisk() + So(err, ShouldBeNil) + + folders := 0 + dashboards := 0 + + for _, i := range fakeRepo.inserted { + if i.Dashboard.IsFolder { + folders++ + } else { + dashboards++ + } + } + + So(dashboards, ShouldEqual, 2) + So(folders, ShouldEqual, 1) + }) + + Convey("Should not update dashboards when db is newer", func() { + cfg.Options["folder"] = oneDashboard + + fakeRepo.getDashboard = append(fakeRepo.getDashboard, &models.Dashboard{ + Updated: time.Now().Add(time.Hour), + Slug: "grafana", + }) + + reader, err := NewDashboardFileReader(cfg, logger) + So(err, ShouldBeNil) + + err = reader.startWalkingDisk() + So(err, ShouldBeNil) + + So(len(fakeRepo.inserted), ShouldEqual, 0) + }) + + Convey("Can read default dashboard and replace old version in database", func() { + cfg.Options["folder"] = oneDashboard + + stat, _ := os.Stat(oneDashboard + "/dashboard1.json") + + fakeRepo.getDashboard = append(fakeRepo.getDashboard, &models.Dashboard{ + Updated: stat.ModTime().AddDate(0, 0, -1), + Slug: "grafana", + }) + + reader, err := NewDashboardFileReader(cfg, logger) + So(err, ShouldBeNil) + + err = reader.startWalkingDisk() + So(err, ShouldBeNil) + + So(len(fakeRepo.inserted), ShouldEqual, 1) + }) + + Convey("Invalid configuration should return error", func() { + cfg := &DashboardsAsConfig{ + Name: "Default", + Type: "file", + OrgId: 1, + Folder: "", + } + + _, err := NewDashboardFileReader(cfg, logger) + So(err, ShouldNotBeNil) + }) + + Convey("Broken dashboards should not cause error", func() { + cfg.Options["folder"] = brokenDashboards + + _, err := NewDashboardFileReader(cfg, logger) + So(err, ShouldBeNil) + }) }) - Convey("Broken dashboards should not cause error", func() { + Convey("Should not create new folder if folder name is missing", func() { cfg := &DashboardsAsConfig{ Name: "Default", Type: "file", OrgId: 1, Folder: "", Options: map[string]interface{}{ - "folder": brokenDashboards, + "folder": defaultDashboards, }, } - _, err := NewDashboardFileReader(cfg, logger) + _, err := getOrCreateFolderId(cfg, fakeRepo) + So(err, ShouldEqual, ErrFolderNameMissing) + }) + + Convey("can get or Create dashboard folder", func() { + cfg := &DashboardsAsConfig{ + Name: "Default", + Type: "file", + OrgId: 1, + Folder: "TEAM A", + Options: map[string]interface{}{ + "folder": defaultDashboards, + }, + } + + folderId, err := getOrCreateFolderId(cfg, fakeRepo) So(err, ShouldBeNil) + inserted := false + for _, d := range fakeRepo.inserted { + if d.Dashboard.IsFolder && d.Dashboard.Id == folderId { + inserted = true + } + } + So(len(fakeRepo.inserted), ShouldEqual, 1) + So(inserted, ShouldBeTrue) + }) + + Convey("Walking the folder with dashboards", func() { + cfg := &DashboardsAsConfig{ + Name: "Default", + Type: "file", + OrgId: 1, + Folder: "", + Options: map[string]interface{}{ + "folder": defaultDashboards, + }, + } + + reader, err := NewDashboardFileReader(cfg, log.New("test-logger")) + So(err, ShouldBeNil) + + Convey("should skip dirs that starts with .", func() { + shouldSkip := reader.createWalk(reader, 0)("path", &FakeFileInfo{isDirectory: true, name: ".folder"}, nil) + So(shouldSkip, ShouldEqual, filepath.SkipDir) + }) + + Convey("should keep walking if file is not .json", func() { + shouldSkip := reader.createWalk(reader, 0)("path", &FakeFileInfo{isDirectory: true, name: "folder"}, nil) + So(shouldSkip, ShouldBeNil) + }) }) }) } +type FakeFileInfo struct { + isDirectory bool + name string +} + +func (ffi *FakeFileInfo) IsDir() bool { + return ffi.isDirectory +} + +func (ffi FakeFileInfo) Size() int64 { + return 1 +} + +func (ffi FakeFileInfo) Mode() os.FileMode { + return 0777 +} + +func (ffi FakeFileInfo) Name() string { + return ffi.name +} + +func (ffi FakeFileInfo) ModTime() time.Time { + return time.Time{} +} + +func (ffi FakeFileInfo) Sys() interface{} { + return nil +} + type fakeDashboardRepo struct { inserted []*dashboards.SaveDashboardItem getDashboard []*models.Dashboard diff --git a/pkg/services/provisioning/dashboards/types.go b/pkg/services/provisioning/dashboards/types.go index c4c4a67a755..46ca3c9246e 100644 --- a/pkg/services/provisioning/dashboards/types.go +++ b/pkg/services/provisioning/dashboards/types.go @@ -18,14 +18,16 @@ type DashboardsAsConfig struct { Options map[string]interface{} `json:"options" yaml:"options"` } -func createDashboardJson(data *simplejson.Json, lastModified time.Time, cfg *DashboardsAsConfig) (*dashboards.SaveDashboardItem, error) { - +func createDashboardJson(data *simplejson.Json, lastModified time.Time, cfg *DashboardsAsConfig, folderId int64) (*dashboards.SaveDashboardItem, error) { dash := &dashboards.SaveDashboardItem{} dash.Dashboard = models.NewDashboardFromJson(data) dash.UpdatedAt = lastModified dash.Overwrite = true dash.OrgId = cfg.OrgId - dash.Dashboard.Data.Set("editable", cfg.Editable) + dash.Dashboard.FolderId = folderId + if !cfg.Editable { + dash.Dashboard.Data.Set("editable", cfg.Editable) + } if dash.Dashboard.Title == "" { return nil, models.ErrDashboardTitleEmpty