From 30662802161ecece7c373298583a464783474c85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Fri, 23 Jun 2017 17:13:58 -0400 Subject: [PATCH] added code from #8504, and #8021 --- pkg/api/api.go | 1 - pkg/api/dashboard.go | 17 --- pkg/api/search.go | 6 -- pkg/services/search/handlers.go | 76 +------------- pkg/services/search/handlers_test.go | 14 --- pkg/services/search/json_index.go | 137 ------------------------- pkg/services/search/json_index_test.go | 42 -------- pkg/services/search/models.go | 12 +-- pkg/services/sqlstore/dashboard.go | 62 +++++++---- 9 files changed, 50 insertions(+), 317 deletions(-) delete mode 100644 pkg/services/search/json_index.go delete mode 100644 pkg/services/search/json_index_test.go diff --git a/pkg/api/api.go b/pkg/api/api.go index 1131088f103..af080c73929 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -238,7 +238,6 @@ func (hs *HttpServer) registerRoutes() { r.Post("/db", bind(m.SaveDashboardCommand{}), wrap(PostDashboard)) r.Post("/calculate-diff", bind(dtos.CalculateDiffOptions{}), wrap(CalculateDashboardDiff)) - r.Get("/file/:file", GetDashboardFromJsonFile) r.Get("/home", wrap(GetHomeDashboard)) r.Get("/tags", GetDashboardTags) r.Post("/import", bind(dtos.ImportDashboardCommand{}), wrap(ImportDashboard)) diff --git a/pkg/api/dashboard.go b/pkg/api/dashboard.go index 7c8e1cdc9bb..60b3aba56fb 100644 --- a/pkg/api/dashboard.go +++ b/pkg/api/dashboard.go @@ -17,7 +17,6 @@ import ( "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/services/alerting" "github.com/grafana/grafana/pkg/services/guardian" - "github.com/grafana/grafana/pkg/services/search" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util" ) @@ -290,22 +289,6 @@ func addGettingStartedPanelToHomeDashboard(dash *simplejson.Json) { row.Set("panels", panels) } -func GetDashboardFromJsonFile(c *middleware.Context) { - file := c.Params(":file") - - dashboard := search.GetDashboardFromJsonIndex(file) - if dashboard == nil { - c.JsonApiErr(404, "Dashboard not found", nil) - return - } - - dash := dtos.DashboardFullWithMeta{Dashboard: dashboard.Data} - dash.Meta.Type = m.DashTypeJson - dash.Meta.CanEdit = c.SignedInUser.HasRole(m.ROLE_READ_ONLY_EDITOR) - - c.JSON(200, &dash) -} - // GetDashboardVersions returns all dashboard versions as JSON func GetDashboardVersions(c *middleware.Context) Response { dashId := c.ParamsInt64(":dashboardId") diff --git a/pkg/api/search.go b/pkg/api/search.go index 17b1ac0da61..3998fb0298c 100644 --- a/pkg/api/search.go +++ b/pkg/api/search.go @@ -16,16 +16,11 @@ func Search(c *middleware.Context) { limit := c.QueryInt("limit") dashboardType := c.Query("type") folderId := c.QueryInt64("folderId") - mode := c.Query("mode") if limit == 0 { limit = 1000 } - if mode == "" { - mode = "list" - } - dbids := make([]int64, 0) for _, id := range c.QueryStrings("dashboardIds") { dashboardId, err := strconv.ParseInt(id, 10, 64) @@ -44,7 +39,6 @@ func Search(c *middleware.Context) { DashboardIds: dbids, Type: dashboardType, FolderId: folderId, - Mode: mode, } err := bus.Dispatch(&searchQuery) diff --git a/pkg/services/search/handlers.go b/pkg/services/search/handlers.go index 44fd4d9da45..9197c8f77e6 100644 --- a/pkg/services/search/handlers.go +++ b/pkg/services/search/handlers.go @@ -1,43 +1,17 @@ package search import ( - "log" - "path/filepath" "sort" "github.com/grafana/grafana/pkg/bus" m "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/setting" ) -var jsonDashIndex *JsonDashIndex - func Init() { bus.AddHandler("search", searchHandler) - - jsonIndexCfg, _ := setting.Cfg.GetSection("dashboards.json") - - if jsonIndexCfg == nil { - log.Fatal("Config section missing: dashboards.json") - return - } - - jsonIndexEnabled := jsonIndexCfg.Key("enabled").MustBool(false) - - if jsonIndexEnabled { - jsonFilesPath := jsonIndexCfg.Key("path").String() - if !filepath.IsAbs(jsonFilesPath) { - jsonFilesPath = filepath.Join(setting.HomePath, jsonFilesPath) - } - - jsonDashIndex = NewJsonDashIndex(jsonFilesPath) - go jsonDashIndex.updateLoop() - } } func searchHandler(query *Query) error { - hits := make(HitList, 0) - dashQuery := FindPersistedDashboardsQuery{ Title: query.Title, SignedInUser: query.SignedInUser, @@ -45,35 +19,17 @@ func searchHandler(query *Query) error { DashboardIds: query.DashboardIds, Type: query.Type, FolderId: query.FolderId, - Mode: query.Mode, + Tags: query.Tags, + Limit: query.Limit, } if err := bus.Dispatch(&dashQuery); err != nil { return err } + hits := make(HitList, 0) hits = append(hits, dashQuery.Result...) - if jsonDashIndex != nil { - jsonHits, err := jsonDashIndex.Search(query) - if err != nil { - return err - } - - hits = append(hits, jsonHits...) - } - - // filter out results with tag filter - if len(query.Tags) > 0 { - filtered := HitList{} - for _, hit := range hits { - if hasRequiredTags(query.Tags, hit.Tags) { - filtered = append(filtered, hit) - } - } - hits = filtered - } - // sort main result array sort.Sort(hits) @@ -95,25 +51,6 @@ func searchHandler(query *Query) error { return nil } -func stringInSlice(a string, list []string) bool { - for _, b := range list { - if b == a { - return true - } - } - return false -} - -func hasRequiredTags(queryTags, hitTags []string) bool { - for _, queryTag := range queryTags { - if !stringInSlice(queryTag, hitTags) { - return false - } - } - - return true -} - func setIsStarredFlagOnSearchResults(userId int64, hits []*Hit) error { query := m.GetUserStarsQuery{UserId: userId} if err := bus.Dispatch(&query); err != nil { @@ -128,10 +65,3 @@ func setIsStarredFlagOnSearchResults(userId int64, hits []*Hit) error { return nil } - -func GetDashboardFromJsonIndex(filename string) *m.Dashboard { - if jsonDashIndex == nil { - return nil - } - return jsonDashIndex.GetDashboard(filename) -} diff --git a/pkg/services/search/handlers_test.go b/pkg/services/search/handlers_test.go index 3f6ede3e572..92a96dfd102 100644 --- a/pkg/services/search/handlers_test.go +++ b/pkg/services/search/handlers_test.go @@ -11,9 +11,7 @@ import ( func TestSearch(t *testing.T) { Convey("Given search query", t, func() { - jsonDashIndex = NewJsonDashIndex("../../../public/dashboards/") query := Query{Limit: 2000, SignedInUser: &m.SignedInUser{IsGrafanaAdmin: true}} - bus.AddHandler("test", func(query *FindPersistedDashboardsQuery) error { query.Result = HitList{ &Hit{Id: 16, Title: "CCAA", Type: "dash-db", Tags: []string{"BB", "AA"}}, @@ -54,17 +52,5 @@ func TestSearch(t *testing.T) { }) }) - Convey("That filters by tag", func() { - query.Tags = []string{"BB", "AA"} - err := searchHandler(&query) - So(err, ShouldBeNil) - - Convey("should return correct results", func() { - So(len(query.Result), ShouldEqual, 3) - So(query.Result[0].Title, ShouldEqual, "BBAA") - So(query.Result[2].Title, ShouldEqual, "CCAA") - }) - - }) }) } diff --git a/pkg/services/search/json_index.go b/pkg/services/search/json_index.go deleted file mode 100644 index 79c238b27f9..00000000000 --- a/pkg/services/search/json_index.go +++ /dev/null @@ -1,137 +0,0 @@ -package search - -import ( - "os" - "path/filepath" - "strings" - "time" - - "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/log" - m "github.com/grafana/grafana/pkg/models" -) - -type JsonDashIndex struct { - path string - items []*JsonDashIndexItem -} - -type JsonDashIndexItem struct { - TitleLower string - TagsCsv string - Path string - Dashboard *m.Dashboard -} - -func NewJsonDashIndex(path string) *JsonDashIndex { - log.Info("Creating json dashboard index for path: %v", path) - - index := JsonDashIndex{} - index.path = path - index.updateIndex() - return &index -} - -func (index *JsonDashIndex) updateLoop() { - ticker := time.NewTicker(time.Minute) - for { - select { - case <-ticker.C: - if err := index.updateIndex(); err != nil { - log.Error(3, "Failed to update dashboard json index %v", err) - } - } - } -} - -func (index *JsonDashIndex) Search(query *Query) ([]*Hit, error) { - results := make([]*Hit, 0) - - if query.IsStarred { - return results, nil - } - - queryStr := strings.ToLower(query.Title) - - for _, item := range index.items { - if len(results) > query.Limit { - break - } - - // add results with matchig title filter - if strings.Contains(item.TitleLower, queryStr) { - results = append(results, &Hit{ - Type: DashHitJson, - Title: item.Dashboard.Title, - Tags: item.Dashboard.GetTags(), - Uri: "file/" + item.Path, - }) - } - } - - return results, nil -} - -func (index *JsonDashIndex) GetDashboard(path string) *m.Dashboard { - for _, item := range index.items { - if item.Path == path { - return item.Dashboard - } - } - - return nil -} - -func (index *JsonDashIndex) updateIndex() error { - var items = make([]*JsonDashIndexItem, 0) - - visitor := func(path string, f os.FileInfo, err error) error { - if err != nil { - return err - } - if f.IsDir() { - return nil - } - - if strings.HasSuffix(f.Name(), ".json") { - dash, err := loadDashboardFromFile(path) - if err != nil { - return err - } - - items = append(items, dash) - } - - return nil - } - - if err := filepath.Walk(index.path, visitor); err != nil { - return err - } - - index.items = items - return nil -} - -func loadDashboardFromFile(filename string) (*JsonDashIndexItem, error) { - reader, err := os.Open(filename) - if err != nil { - return nil, err - } - defer reader.Close() - - data, err := simplejson.NewFromReader(reader) - if err != nil { - return nil, err - } - - stat, _ := os.Stat(filename) - - item := &JsonDashIndexItem{} - item.Dashboard = m.NewDashboardFromJson(data) - item.TitleLower = strings.ToLower(item.Dashboard.Title) - item.TagsCsv = strings.Join(item.Dashboard.GetTags(), ",") - item.Path = stat.Name() - - return item, nil -} diff --git a/pkg/services/search/json_index_test.go b/pkg/services/search/json_index_test.go deleted file mode 100644 index 145e1ac1e99..00000000000 --- a/pkg/services/search/json_index_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package search - -import ( - "testing" - - . "github.com/smartystreets/goconvey/convey" -) - -func TestJsonDashIndex(t *testing.T) { - - Convey("Given the json dash index", t, func() { - index := NewJsonDashIndex("../../../public/dashboards/") - - Convey("Should be able to update index", func() { - err := index.updateIndex() - So(err, ShouldBeNil) - }) - - Convey("Should be able to search index", func() { - res, err := index.Search(&Query{Title: "", Limit: 20}) - So(err, ShouldBeNil) - - So(len(res), ShouldEqual, 3) - }) - - Convey("Should be able to search index by title", func() { - res, err := index.Search(&Query{Title: "home", Limit: 20}) - So(err, ShouldBeNil) - - So(len(res), ShouldEqual, 1) - So(res[0].Title, ShouldEqual, "Home") - }) - - Convey("Should not return when starred is filtered", func() { - res, err := index.Search(&Query{Title: "", IsStarred: true}) - So(err, ShouldBeNil) - - So(len(res), ShouldEqual, 0) - }) - - }) -} diff --git a/pkg/services/search/models.go b/pkg/services/search/models.go index 5940a824b6a..a95a5dc3d4a 100644 --- a/pkg/services/search/models.go +++ b/pkg/services/search/models.go @@ -6,11 +6,9 @@ import "github.com/grafana/grafana/pkg/models" type HitType string const ( - DashHitDB HitType = "dash-db" - DashHitHome HitType = "dash-home" - DashHitJson HitType = "dash-json" - DashHitScripted HitType = "dash-scripted" - DashHitFolder HitType = "dash-folder" + DashHitDB HitType = "dash-db" + DashHitHome HitType = "dash-home" + DashHitFolder HitType = "dash-folder" ) type Hit struct { @@ -51,7 +49,6 @@ type Query struct { Type string DashboardIds []int64 FolderId int64 - Mode string Result HitList } @@ -64,7 +61,8 @@ type FindPersistedDashboardsQuery struct { DashboardIds []int64 Type string FolderId int64 - Mode string + Tags []string + Limit int Result HitList } diff --git a/pkg/services/sqlstore/dashboard.go b/pkg/services/sqlstore/dashboard.go index 4e93325efc0..16d08894367 100644 --- a/pkg/services/sqlstore/dashboard.go +++ b/pkg/services/sqlstore/dashboard.go @@ -3,6 +3,7 @@ package sqlstore import ( "bytes" "fmt" + "strings" "time" "github.com/grafana/grafana/pkg/bus" @@ -159,22 +160,49 @@ type DashboardSearchProjection struct { } func findDashboards(query *search.FindPersistedDashboardsQuery) ([]DashboardSearchProjection, error) { + limit := query.Limit + if limit == 0 { + limit = 1000 + } + var sql bytes.Buffer params := make([]interface{}, 0) - sql.WriteString(`SELECT - dashboard.id, - dashboard.title, - dashboard.slug, - dashboard_tag.term, - dashboard.is_folder, - dashboard.folder_id, - f.slug as folder_slug, - f.title as folder_title - FROM dashboard - LEFT OUTER JOIN dashboard f on f.id = dashboard.folder_id - LEFT OUTER JOIN dashboard_tag on dashboard_tag.dashboard_id = dashboard.id`) + sql.WriteString(` + SELECT + dashboard.id, + dashboard.title, + dashboard.slug, + dashboard_tag.term, + dashboard.is_folder, + dashboard.folder_id, + folder.slug as folder_slug, + folder.title as folder_title + FROM ( + SELECT + dashboard.id FROM dashboard + LEFT OUTER JOIN dashboard_tag ON dashboard_tag.dashboard_id = dashboard.id + `) + // add tags filter + if len(query.Tags) > 0 { + sql.WriteString(` WHERE dashboard_tag.term IN (?` + strings.Repeat(",?", len(query.Tags)-1) + `)`) + for _, tag := range query.Tags { + params = append(params, tag) + } + } + + // this ends the inner select (tag filtered part) + sql.WriteString(` + GROUP BY dashboard.id HAVING COUNT(dashboard.id) >= ? + ORDER BY dashboard.title ASC LIMIT ?) as ids`) + params = append(params, len(query.Tags)) + params = append(params, limit) + + sql.WriteString(` + INNER JOIN dashboard on ids.id = dashboard.id + LEFT OUTER JOIN dashboard folder on folder.id = dashboard.folder_id + LEFT OUTER JOIN dashboard_tag on dashboard.id = dashboard_tag.dashboard_id`) if query.IsStarred { sql.WriteString(" INNER JOIN star on star.dashboard_id = dashboard.id") } @@ -188,16 +216,10 @@ func findDashboards(query *search.FindPersistedDashboardsQuery) ([]DashboardSear } if len(query.DashboardIds) > 0 { - sql.WriteString(" AND (") - for i, dashboardId := range query.DashboardIds { - if i != 0 { - sql.WriteString(" OR") - } - - sql.WriteString(" dashboard.id = ?") + sql.WriteString(` AND dashboard.id IN (?` + strings.Repeat(",?", len(query.DashboardIds)-1) + `)`) + for _, dashboardId := range query.DashboardIds { params = append(params, dashboardId) } - sql.WriteString(")") } if query.SignedInUser.OrgRole != m.ROLE_ADMIN {