Refactor search (#23550)
Co-Authored-By: Arve Knudsen <arve.knudsen@grafana.com> Co-Authored-By: Leonard Gram <leonard.gram@grafana.com>
This commit is contained in:
Emil Tullstedt
co-authored by
Arve Knudsen
Leonard Gram
Arve Knudsen
Leonard Gram
parent
e5dd7efdee
commit
55c306eb6d
@@ -0,0 +1,112 @@
|
||||
package searchstore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Builder defaults to returning a SQL query to get a list of all dashboards
|
||||
// in default order, but can be modified by applying filters.
|
||||
type Builder struct {
|
||||
// List of FilterWhere/FilterGroupBy/FilterOrderBy/FilterLeftJoin
|
||||
// to modify the query.
|
||||
Filters []interface{}
|
||||
Dialect migrator.Dialect
|
||||
|
||||
params []interface{}
|
||||
sql bytes.Buffer
|
||||
}
|
||||
|
||||
// ToSql builds the SQL query and returns it as a string, together with the SQL parameters.
|
||||
func (b *Builder) ToSql(limit, page int64) (string, []interface{}) {
|
||||
b.params = make([]interface{}, 0)
|
||||
b.sql = bytes.Buffer{}
|
||||
|
||||
b.buildSelect()
|
||||
|
||||
b.sql.WriteString("( ")
|
||||
b.applyFilters()
|
||||
|
||||
b.sql.WriteString(b.Dialect.LimitOffset(limit, (page-1)*limit) + `) AS ids
|
||||
INNER JOIN dashboard ON ids.id = dashboard.id
|
||||
`)
|
||||
|
||||
b.sql.WriteString(`
|
||||
LEFT OUTER JOIN dashboard AS folder ON folder.id = dashboard.folder_id
|
||||
LEFT OUTER JOIN dashboard_tag ON dashboard.id = dashboard_tag.dashboard_id`)
|
||||
|
||||
return b.sql.String(), b.params
|
||||
}
|
||||
|
||||
func (b *Builder) buildSelect() {
|
||||
b.sql.WriteString(
|
||||
`SELECT
|
||||
dashboard.id,
|
||||
dashboard.uid,
|
||||
dashboard.title,
|
||||
dashboard.slug,
|
||||
dashboard_tag.term,
|
||||
dashboard.is_folder,
|
||||
dashboard.folder_id,
|
||||
folder.uid AS folder_uid,
|
||||
folder.slug AS folder_slug,
|
||||
folder.title AS folder_title
|
||||
FROM `)
|
||||
}
|
||||
|
||||
func (b *Builder) applyFilters() {
|
||||
joins := []string{}
|
||||
|
||||
wheres := []string{}
|
||||
whereParams := []interface{}{}
|
||||
|
||||
groups := []string{}
|
||||
groupParams := []interface{}{}
|
||||
|
||||
orders := []string{}
|
||||
|
||||
for _, f := range b.Filters {
|
||||
if f, ok := f.(FilterLeftJoin); ok {
|
||||
joins = append(joins, fmt.Sprintf(" LEFT OUTER JOIN %s ", f.LeftJoin()))
|
||||
}
|
||||
|
||||
if f, ok := f.(FilterWhere); ok {
|
||||
sql, params := f.Where()
|
||||
if sql != "" {
|
||||
wheres = append(wheres, sql)
|
||||
whereParams = append(whereParams, params...)
|
||||
}
|
||||
}
|
||||
|
||||
if f, ok := f.(FilterGroupBy); ok {
|
||||
sql, params := f.GroupBy()
|
||||
if sql != "" {
|
||||
groups = append(groups, sql)
|
||||
groupParams = append(groupParams, params...)
|
||||
}
|
||||
}
|
||||
|
||||
if f, ok := f.(FilterOrderBy); ok {
|
||||
orders = append(orders, f.OrderBy())
|
||||
}
|
||||
}
|
||||
|
||||
b.sql.WriteString("SELECT dashboard.id FROM dashboard")
|
||||
b.sql.WriteString(strings.Join(joins, ""))
|
||||
|
||||
if len(wheres) > 0 {
|
||||
b.sql.WriteString(fmt.Sprintf(" WHERE %s", strings.Join(wheres, " AND ")))
|
||||
b.params = append(b.params, whereParams...)
|
||||
}
|
||||
|
||||
if len(groups) > 0 {
|
||||
b.sql.WriteString(fmt.Sprintf(" GROUP BY %s", strings.Join(groups, ", ")))
|
||||
b.params = append(b.params, groupParams...)
|
||||
}
|
||||
|
||||
if len(orders) > 0 {
|
||||
b.sql.WriteString(fmt.Sprintf(" ORDER BY %s", strings.Join(orders, ", ")))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
// Package searchstore converts search queries to SQL.
|
||||
//
|
||||
// Because of the wide array of deployments supported by Grafana,
|
||||
// search strives to be both performant enough to handle heavy users
|
||||
// and lightweight enough to not increase complexity/resource
|
||||
// utilization for light users. To allow this we're currently searching
|
||||
// without fuzziness and in a single SQL query.
|
||||
//
|
||||
// Search queries are a combination of an outer query which Builder
|
||||
// creates automatically when calling the Builder.ToSql method and an
|
||||
// inner query feeding that which lists the IDs of the dashboards that
|
||||
// should be part of the result set. By default search will return all
|
||||
// dashboards (behind pagination) but it is possible to dynamically add
|
||||
// filters capable of adding more specific inclusion or ordering
|
||||
// requirements.
|
||||
//
|
||||
// A filter is any data type which implements one or more of the
|
||||
// FilterWhere, FilterGroupBy, FilterOrderBy, or FilterLeftJoin
|
||||
// interfaces. The filters will be applied (in order) to limit or
|
||||
// reorder the results.
|
||||
//
|
||||
// Filters will be applied in order with the final result like such:
|
||||
//
|
||||
// SELECT id FROM dashboard LEFT OUTER JOIN <FilterLeftJoin...>
|
||||
// WHERE <FilterWhere[0]> AND ... AND <FilterWhere[n]>
|
||||
// GROUP BY <FilterGroupBy...>
|
||||
// ORDER BY <FilterOrderBy...>
|
||||
// LIMIT <limit> OFFSET <(page-1)*limit>;
|
||||
//
|
||||
// This structure is intended to isolate the filters from each other
|
||||
// and implementors are expected to add all the required joins, where
|
||||
// clauses, groupings, and/or orderings necessary for applying a
|
||||
// filter in the filter. Using side-effects of other filters is
|
||||
// bad manners and increases the complexity and volatility of the code.
|
||||
package searchstore
|
||||
@@ -0,0 +1,145 @@
|
||||
package searchstore
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// FilterWhere limits the set of dashboard IDs to the dashboards for
|
||||
// which the filter is applicable. Results where the first value is
|
||||
// an empty string are discarded.
|
||||
type FilterWhere interface {
|
||||
Where() (string, []interface{})
|
||||
}
|
||||
|
||||
// FilterGroupBy should be used after performing an outer join on the
|
||||
// search result to ensure there is only one of each ID in the results.
|
||||
// The id column must be present in the result.
|
||||
type FilterGroupBy interface {
|
||||
GroupBy() (string, []interface{})
|
||||
}
|
||||
|
||||
// FilterOrderBy provides an ordering for the search result.
|
||||
type FilterOrderBy interface {
|
||||
OrderBy() string
|
||||
}
|
||||
|
||||
// FilterLeftJoin adds the returned string as a "LEFT OUTER JOIN" to
|
||||
// allow for fetching extra columns from a table outside of the
|
||||
// dashboard column.
|
||||
type FilterLeftJoin interface {
|
||||
LeftJoin() string
|
||||
}
|
||||
|
||||
const (
|
||||
TypeFolder = "dash-folder"
|
||||
TypeDashboard = "dash-db"
|
||||
)
|
||||
|
||||
type TypeFilter struct {
|
||||
Dialect migrator.Dialect
|
||||
Type string
|
||||
}
|
||||
|
||||
func (f TypeFilter) Where() (string, []interface{}) {
|
||||
if f.Type == TypeFolder {
|
||||
return "dashboard.is_folder = " + f.Dialect.BooleanStr(true), nil
|
||||
}
|
||||
|
||||
if f.Type == TypeDashboard {
|
||||
return "dashboard.is_folder = " + f.Dialect.BooleanStr(false), nil
|
||||
}
|
||||
|
||||
return "", nil
|
||||
}
|
||||
|
||||
type OrgFilter struct {
|
||||
OrgId int64
|
||||
}
|
||||
|
||||
func (f OrgFilter) Where() (string, []interface{}) {
|
||||
return "dashboard.org_id=?", []interface{}{f.OrgId}
|
||||
}
|
||||
|
||||
type StarredFilter struct {
|
||||
UserId int64
|
||||
}
|
||||
|
||||
func (f StarredFilter) Where() (string, []interface{}) {
|
||||
return `(SELECT count(*)
|
||||
FROM star
|
||||
WHERE star.dashboard_id = dashboard.id AND star.user_id = ?) > 0`, []interface{}{f.UserId}
|
||||
}
|
||||
|
||||
type TitleFilter struct {
|
||||
Dialect migrator.Dialect
|
||||
Title string
|
||||
}
|
||||
|
||||
func (f TitleFilter) Where() (string, []interface{}) {
|
||||
return fmt.Sprintf("dashboard.title %s ?", f.Dialect.LikeStr()), []interface{}{"%" + f.Title + "%"}
|
||||
}
|
||||
|
||||
type FolderFilter struct {
|
||||
IDs []int64
|
||||
}
|
||||
|
||||
func (f FolderFilter) Where() (string, []interface{}) {
|
||||
return sqlIDin("dashboard.folder_id", f.IDs)
|
||||
}
|
||||
|
||||
type DashboardFilter struct {
|
||||
IDs []int64
|
||||
}
|
||||
|
||||
func (f DashboardFilter) Where() (string, []interface{}) {
|
||||
return sqlIDin("dashboard.id", f.IDs)
|
||||
}
|
||||
|
||||
type TagsFilter struct {
|
||||
Tags []string
|
||||
}
|
||||
|
||||
func (f TagsFilter) LeftJoin() string {
|
||||
return `dashboard_tag ON dashboard_tag.dashboard_id = dashboard.id`
|
||||
}
|
||||
|
||||
func (f TagsFilter) GroupBy() (string, []interface{}) {
|
||||
return `dashboard.id HAVING COUNT(dashboard.id) >= ?`, []interface{}{len(f.Tags)}
|
||||
}
|
||||
|
||||
func (f TagsFilter) Where() (string, []interface{}) {
|
||||
params := make([]interface{}, len(f.Tags))
|
||||
for i, tag := range f.Tags {
|
||||
params[i] = tag
|
||||
}
|
||||
return `dashboard_tag.term IN (?` + strings.Repeat(",?", len(f.Tags)-1) + `)`, params
|
||||
}
|
||||
|
||||
type TitleSorter struct {
|
||||
Descending bool
|
||||
}
|
||||
|
||||
func (s TitleSorter) OrderBy() string {
|
||||
if s.Descending {
|
||||
return "dashboard.title DESC"
|
||||
}
|
||||
|
||||
return "dashboard.title ASC"
|
||||
}
|
||||
|
||||
func sqlIDin(column string, ids []int64) (string, []interface{}) {
|
||||
length := len(ids)
|
||||
if length < 1 {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
sqlArray := "(?" + strings.Repeat(",?", length-1) + ")"
|
||||
|
||||
params := []interface{}{}
|
||||
for _, id := range ids {
|
||||
params = append(params, id)
|
||||
}
|
||||
return fmt.Sprintf("%s IN %s", column, sqlArray), params
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
// package search_test contains integration tests for search
|
||||
package searchstore_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/grafana/grafana/pkg/components/simplejson"
|
||||
"github.com/grafana/grafana/pkg/models"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore/permissions"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore/searchstore"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
var dialect migrator.Dialect
|
||||
|
||||
const (
|
||||
limit int64 = 15
|
||||
page int64 = 1
|
||||
)
|
||||
|
||||
func TestBuilder_EqualResults_Basic(t *testing.T) {
|
||||
user := &models.SignedInUser{
|
||||
UserId: 1,
|
||||
OrgId: 1,
|
||||
OrgRole: models.ROLE_EDITOR,
|
||||
}
|
||||
|
||||
db := setupTestEnvironment(t)
|
||||
err := createDashboards(0, 1, user.OrgId)
|
||||
require.NoError(t, err)
|
||||
|
||||
// create one dashboard in another organization that shouldn't
|
||||
// be listed in the results.
|
||||
err = createDashboards(1, 2, 2)
|
||||
require.NoError(t, err)
|
||||
|
||||
builder := &searchstore.Builder{
|
||||
Filters: []interface{}{
|
||||
searchstore.OrgFilter{OrgId: user.OrgId},
|
||||
searchstore.TitleSorter{},
|
||||
},
|
||||
Dialect: dialect,
|
||||
}
|
||||
|
||||
prevBuilder := sqlstore.NewSearchBuilder(user, limit, page, models.PERMISSION_EDIT)
|
||||
prevBuilder.WithDialect(dialect)
|
||||
|
||||
newRes := []sqlstore.DashboardSearchProjection{}
|
||||
err = db.WithDbSession(context.Background(), func(sess *sqlstore.DBSession) error {
|
||||
sql, params := builder.ToSql(limit, page)
|
||||
return sess.SQL(sql, params...).Find(&newRes)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
oldRes := []sqlstore.DashboardSearchProjection{}
|
||||
err = db.WithDbSession(context.Background(), func(sess *sqlstore.DBSession) error {
|
||||
sql, params := prevBuilder.ToSql()
|
||||
return sess.SQL(sql, params...).Find(&oldRes)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Len(t, newRes, 1)
|
||||
assert.EqualValues(t, oldRes, newRes)
|
||||
}
|
||||
|
||||
func TestBuilder_Pagination(t *testing.T) {
|
||||
user := &models.SignedInUser{
|
||||
UserId: 1,
|
||||
OrgId: 1,
|
||||
OrgRole: models.ROLE_VIEWER,
|
||||
}
|
||||
|
||||
db := setupTestEnvironment(t)
|
||||
err := createDashboards(0, 25, user.OrgId)
|
||||
require.NoError(t, err)
|
||||
|
||||
builder := &searchstore.Builder{
|
||||
Filters: []interface{}{
|
||||
searchstore.OrgFilter{OrgId: user.OrgId},
|
||||
searchstore.TitleSorter{},
|
||||
},
|
||||
Dialect: dialect,
|
||||
}
|
||||
|
||||
resPg1 := []sqlstore.DashboardSearchProjection{}
|
||||
resPg2 := []sqlstore.DashboardSearchProjection{}
|
||||
resPg3 := []sqlstore.DashboardSearchProjection{}
|
||||
err = db.WithDbSession(context.Background(), func(sess *sqlstore.DBSession) error {
|
||||
sql, params := builder.ToSql(15, 1)
|
||||
err := sess.SQL(sql, params...).Find(&resPg1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sql, params = builder.ToSql(15, 2)
|
||||
err = sess.SQL(sql, params...).Find(&resPg2)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sql, params = builder.ToSql(15, 3)
|
||||
return sess.SQL(sql, params...).Find(&resPg3)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Len(t, resPg1, 15)
|
||||
assert.Len(t, resPg2, 10)
|
||||
assert.Len(t, resPg3, 0, "sanity check: pages after last should be empty")
|
||||
|
||||
assert.Equal(t, "A", resPg1[0].Title, "page 1 should start with the first dashboard")
|
||||
assert.Equal(t, "P", resPg2[0].Title, "page 2 should start with the 16th dashboard")
|
||||
}
|
||||
|
||||
func TestBuilder_Permissions(t *testing.T) {
|
||||
user := &models.SignedInUser{
|
||||
UserId: 1,
|
||||
OrgId: 1,
|
||||
OrgRole: models.ROLE_VIEWER,
|
||||
}
|
||||
|
||||
db := setupTestEnvironment(t)
|
||||
err := createDashboards(0, 1, user.OrgId)
|
||||
require.NoError(t, err)
|
||||
|
||||
level := models.PERMISSION_EDIT
|
||||
|
||||
builder := &searchstore.Builder{
|
||||
Filters: []interface{}{
|
||||
searchstore.OrgFilter{OrgId: user.OrgId},
|
||||
searchstore.TitleSorter{},
|
||||
permissions.DashboardPermissionFilter{
|
||||
Dialect: dialect,
|
||||
OrgRole: user.OrgRole,
|
||||
OrgId: user.OrgId,
|
||||
UserId: user.UserId,
|
||||
PermissionLevel: level,
|
||||
},
|
||||
},
|
||||
Dialect: dialect,
|
||||
}
|
||||
|
||||
prevBuilder := sqlstore.NewSearchBuilder(user, limit, page, level)
|
||||
prevBuilder.WithDialect(dialect)
|
||||
|
||||
newRes := []sqlstore.DashboardSearchProjection{}
|
||||
err = db.WithDbSession(context.Background(), func(sess *sqlstore.DBSession) error {
|
||||
sql, params := builder.ToSql(limit, page)
|
||||
return sess.SQL(sql, params...).Find(&newRes)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
oldRes := []sqlstore.DashboardSearchProjection{}
|
||||
err = db.WithDbSession(context.Background(), func(sess *sqlstore.DBSession) error {
|
||||
sql, params := prevBuilder.ToSql()
|
||||
return sess.SQL(sql, params...).Find(&oldRes)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Len(t, newRes, 0)
|
||||
assert.EqualValues(t, oldRes, newRes)
|
||||
}
|
||||
|
||||
func setupTestEnvironment(t *testing.T) *sqlstore.SqlStore {
|
||||
t.Helper()
|
||||
store := sqlstore.InitTestDB(t)
|
||||
dialect = store.Dialect
|
||||
return store
|
||||
}
|
||||
|
||||
func createDashboards(startID, endID int, orgID int64) error {
|
||||
if endID < startID {
|
||||
return fmt.Errorf("startID must be smaller than endID")
|
||||
}
|
||||
|
||||
for i := startID; i < endID; i++ {
|
||||
dashboard, err := simplejson.NewJson([]byte(`{
|
||||
"id": null,
|
||||
"uid": null,
|
||||
"title": "` + lexiCounter(i) + `",
|
||||
"tags": [ "templated" ],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 16,
|
||||
"version": 0
|
||||
}`))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = sqlstore.SaveDashboard(&models.SaveDashboardCommand{
|
||||
Dashboard: dashboard,
|
||||
UserId: 1,
|
||||
OrgId: orgID,
|
||||
UpdatedAt: time.Now(),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// lexiCounter counts in a lexicographically sortable order.
|
||||
func lexiCounter(n int) string {
|
||||
alphabet := "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
||||
value := string(alphabet[n%26])
|
||||
|
||||
if n >= 26 {
|
||||
value = lexiCounter(n/26-1) + value
|
||||
}
|
||||
|
||||
return value
|
||||
}
|
||||
Reference in New Issue
Block a user