Dash previews: populate crawler queue from SQL query (#44083)

* add SQL migrations

* dashboard previews from sql: poc

* added todos

* refactor: use the same enums where possible

* use useEffect, always return json

* added todo

* refactor + delete files after use

* refactor + fix manual thumbnail upload

* refactor: move all interactions with sqlStore to thumbnail repo

* refactor: remove file operations in thumb crawler/service

* refactor: fix dashboard_thumbs sql store

* refactor: extracted thumbnail fetching/updating to a hook

* refactor: store thumbnails in redux store

* refactor: store thumbnails in redux store

* refactor: private'd repo methods

* removed redux storage, saving images as blobs

* allow for configurable rendering timeouts

* added 1) query for dashboards with stale thumbnails, 2) command for marking thumbnails as stale

* use sql-based queue in crawler

* ui for marking thumbnails as stale

* replaced `stale` boolean prop with `state` enum

* introduce rendering session

* compilation errors

* fix crawler stop button

* rename thumbnail state frozen to locked

* #44449: fix merge conflicts

* #44449: remove thumb methods from `Store` interface

* #44449: clean filepath, defer file closing

* #44449: fix rendering.Theme cyclic import

* #44449: linting

* #44449: linting

* #44449: mutex'd crawlerStatus access

* #44449: added integration tests for `sqlstore.dashboard_thumbs`

* #44449: added comments to explain the `ThumbnailState` enum

* #44449: use os.ReadFile rather then os.Open

* #44449: always enable dashboardPreviews feature during integration tests

* #44449: remove sleep time, adjust number of threads

* #44449: review fix: add `orgId` to `DashboardThumbnailMeta`

* #44449: review fix: automatic parsing of thumbnailState

* #44449: lint fixes

* #44449: review fix: prefer `WithDbSession` over `WithTransactionalDbSession`

* #44449: review fix: add a comment explaining source of the filepath

* #44449: review fix: added filepath validation

* #44449: review fixes https://github.com/grafana/grafana/pull/45063/files @fzambia

Co-authored-by: Ryan McKinley <ryantxu@gmail.com>
Co-authored-by: Alexander Emelin <frvzmb@gmail.com>
This commit is contained in:
Artur Wierzbicki
2022-02-09 13:23:32 +04:00
committed by GitHub
co-authored by Ryan McKinley Alexander Emelin
parent 4e3a72fc2a
commit a025109647
22 changed files with 1003 additions and 306 deletions
+167
View File
@@ -0,0 +1,167 @@
package sqlstore
import (
"context"
"errors"
"time"
"github.com/grafana/grafana/pkg/models"
)
func (ss *SQLStore) GetThumbnail(ctx context.Context, query *models.GetDashboardThumbnailCommand) (*models.DashboardThumbnail, error) {
err := ss.WithDbSession(ctx, func(sess *DBSession) error {
result, err := findThumbnailByMeta(sess, query.DashboardThumbnailMeta)
if err != nil {
return err
}
query.Result = result
return nil
})
return query.Result, err
}
func (ss *SQLStore) SaveThumbnail(ctx context.Context, cmd *models.SaveDashboardThumbnailCommand) (*models.DashboardThumbnail, error) {
err := ss.WithTransactionalDbSession(ctx, func(sess *DBSession) error {
existing, err := findThumbnailByMeta(sess, cmd.DashboardThumbnailMeta)
if err != nil && !errors.Is(err, models.ErrDashboardThumbnailNotFound) {
return err
}
if existing != nil {
existing.Image = cmd.Image
existing.MimeType = cmd.MimeType
existing.Updated = time.Now()
existing.DashboardVersion = cmd.DashboardVersion
existing.State = models.ThumbnailStateDefault
_, err = sess.ID(existing.Id).Update(existing)
cmd.Result = existing
return err
}
thumb := &models.DashboardThumbnail{}
dash, err := findDashboardIdByThumbMeta(sess, cmd.DashboardThumbnailMeta)
if err != nil {
return err
}
thumb.Updated = time.Now()
thumb.Theme = cmd.Theme
thumb.Kind = cmd.Kind
thumb.Image = cmd.Image
thumb.MimeType = cmd.MimeType
thumb.DashboardId = dash.Id
thumb.DashboardVersion = cmd.DashboardVersion
thumb.State = models.ThumbnailStateDefault
thumb.PanelId = cmd.PanelID
_, err = sess.Insert(thumb)
cmd.Result = thumb
return err
})
return cmd.Result, err
}
func (ss *SQLStore) UpdateThumbnailState(ctx context.Context, cmd *models.UpdateThumbnailStateCommand) error {
err := ss.WithTransactionalDbSession(ctx, func(sess *DBSession) error {
existing, err := findThumbnailByMeta(sess, cmd.DashboardThumbnailMeta)
if err != nil {
return err
}
existing.State = cmd.State
_, err = sess.ID(existing.Id).Update(existing)
return err
})
return err
}
func (ss *SQLStore) FindDashboardsWithStaleThumbnails(ctx context.Context, cmd *models.FindDashboardsWithStaleThumbnailsCommand) ([]*models.DashboardWithStaleThumbnail, error) {
err := ss.WithDbSession(ctx, func(sess *DBSession) error {
sess.Table("dashboard")
sess.Join("LEFT", "dashboard_thumbnail", "dashboard.id = dashboard_thumbnail.dashboard_id")
sess.Where("dashboard.is_folder = ?", dialect.BooleanStr(false))
sess.Where("(dashboard.version != dashboard_thumbnail.dashboard_version "+
"OR dashboard_thumbnail.state = ? "+
"OR dashboard_thumbnail.id IS NULL)", models.ThumbnailStateStale)
if !cmd.IncludeManuallyUploadedThumbnails {
sess.Where("(dashboard_thumbnail.id is not null AND dashboard_thumbnail.dashboard_version != ?) "+
"OR dashboard_thumbnail.id is null "+
"OR dashboard_thumbnail.state = ?", models.DashboardVersionForManualThumbnailUpload, models.ThumbnailStateStale)
}
sess.Where("(dashboard_thumbnail.id IS NULL OR dashboard_thumbnail.state != ?)", models.ThumbnailStateLocked)
sess.Cols("dashboard.id",
"dashboard.uid",
"dashboard.org_id",
"dashboard.version",
"dashboard.slug")
var dashboards = make([]*models.DashboardWithStaleThumbnail, 0)
err := sess.Find(&dashboards)
if err != nil {
return err
}
cmd.Result = dashboards
return err
})
return cmd.Result, err
}
func findThumbnailByMeta(sess *DBSession, meta models.DashboardThumbnailMeta) (*models.DashboardThumbnail, error) {
result := &models.DashboardThumbnail{}
sess.Table("dashboard_thumbnail")
sess.Join("INNER", "dashboard", "dashboard.id = dashboard_thumbnail.dashboard_id")
sess.Where("dashboard.uid = ? AND dashboard.org_id = ? AND panel_id = ? AND kind = ? AND theme = ?", meta.DashboardUID, meta.OrgId, meta.PanelID, meta.Kind, meta.Theme)
sess.Cols("dashboard_thumbnail.id",
"dashboard_thumbnail.dashboard_id",
"dashboard_thumbnail.panel_id",
"dashboard_thumbnail.image",
"dashboard_thumbnail.dashboard_version",
"dashboard_thumbnail.state",
"dashboard_thumbnail.kind",
"dashboard_thumbnail.mime_type",
"dashboard_thumbnail.theme",
"dashboard_thumbnail.updated")
exists, err := sess.Get(result)
if !exists {
return nil, models.ErrDashboardThumbnailNotFound
}
if err != nil {
return nil, err
}
return result, nil
}
type dash struct {
Id int64
}
func findDashboardIdByThumbMeta(sess *DBSession, meta models.DashboardThumbnailMeta) (*dash, error) {
result := &dash{}
sess.Table("dashboard").Where("dashboard.uid = ? AND dashboard.org_id = ?", meta.DashboardUID, meta.OrgId).Cols("id")
exists, err := sess.Get(result)
if err != nil {
return nil, err
}
if !exists {
return nil, models.ErrDashboardNotFound
}
return result, err
}
@@ -0,0 +1,216 @@
//go:build integration
// +build integration
package sqlstore
import (
"context"
"testing"
"github.com/grafana/grafana/pkg/models"
"github.com/stretchr/testify/require"
)
func TestSqlStorage(t *testing.T) {
var sqlStore *SQLStore
var savedFolder *models.Dashboard
setup := func() {
sqlStore = InitTestDB(t)
savedFolder = insertTestDashboard(t, sqlStore, "1 test dash folder", 1, 0, true, "prod", "webapp")
}
t.Run("Should insert dashboard in default state", func(t *testing.T) {
setup()
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version)
thumb := getThumbnail(t, sqlStore, dash.Uid, dash.OrgId)
require.Positive(t, thumb.Id)
require.Equal(t, models.ThumbnailStateDefault, thumb.State)
require.Equal(t, dash.Version, thumb.DashboardVersion)
})
t.Run("Should be able to update the thumbnail", func(t *testing.T) {
setup()
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version)
thumb := getThumbnail(t, sqlStore, dash.Uid, dash.OrgId)
insertedThumbnailId := thumb.Id
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version+1)
updatedThumb := getThumbnail(t, sqlStore, dash.Uid, dash.OrgId)
require.Equal(t, insertedThumbnailId, updatedThumb.Id)
require.Equal(t, dash.Version+1, updatedThumb.DashboardVersion)
})
t.Run("Should return empty array if all dashboards have thumbnails", func(t *testing.T) {
setup()
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version)
cmd := models.FindDashboardsWithStaleThumbnailsCommand{}
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
require.NoError(t, err)
require.Len(t, res, 0)
})
t.Run("Should return dashboards with thumbnails marked as stale", func(t *testing.T) {
setup()
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version)
updateThumbnailState(t, sqlStore, dash.Uid, dash.OrgId, models.ThumbnailStateStale)
cmd := models.FindDashboardsWithStaleThumbnailsCommand{}
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
require.NoError(t, err)
require.Len(t, res, 1)
require.Equal(t, dash.Id, res[0].Id)
})
t.Run("Should not return dashboards with updated thumbnails that had been marked as stale", func(t *testing.T) {
setup()
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version)
updateThumbnailState(t, sqlStore, dash.Uid, dash.OrgId, models.ThumbnailStateStale)
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version)
cmd := models.FindDashboardsWithStaleThumbnailsCommand{}
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
require.NoError(t, err)
require.Len(t, res, 0)
})
t.Run("Should find dashboards without thumbnails", func(t *testing.T) {
setup()
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
cmd := models.FindDashboardsWithStaleThumbnailsCommand{}
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
require.NoError(t, err)
require.Len(t, res, 1)
require.Equal(t, dash.Id, res[0].Id)
})
t.Run("Should find dashboards with outdated thumbnails", func(t *testing.T) {
setup()
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version)
updateTestDashboard(t, sqlStore, dash, map[string]interface{}{
"tags": "different-tag",
})
cmd := models.FindDashboardsWithStaleThumbnailsCommand{}
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
require.NoError(t, err)
require.Len(t, res, 1)
require.Equal(t, dash.Id, res[0].Id)
})
t.Run("Should not return dashboards with locked thumbnails even if they are outdated", func(t *testing.T) {
setup()
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version)
updateThumbnailState(t, sqlStore, dash.Uid, dash.OrgId, models.ThumbnailStateLocked)
updateTestDashboard(t, sqlStore, dash, map[string]interface{}{
"tags": "different-tag",
})
cmd := models.FindDashboardsWithStaleThumbnailsCommand{}
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
require.NoError(t, err)
require.Len(t, res, 0)
})
t.Run("Should not return dashboards with manually uploaded thumbnails by default", func(t *testing.T) {
setup()
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, models.DashboardVersionForManualThumbnailUpload)
updateTestDashboard(t, sqlStore, dash, map[string]interface{}{
"tags": "different-tag",
})
cmd := models.FindDashboardsWithStaleThumbnailsCommand{}
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
require.NoError(t, err)
require.Len(t, res, 0)
})
t.Run("Should return dashboards with manually uploaded thumbnails if requested", func(t *testing.T) {
setup()
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, models.DashboardVersionForManualThumbnailUpload)
updateTestDashboard(t, sqlStore, dash, map[string]interface{}{
"tags": "different-tag",
})
cmd := models.FindDashboardsWithStaleThumbnailsCommand{
IncludeManuallyUploadedThumbnails: true,
}
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
require.NoError(t, err)
require.Len(t, res, 1)
require.Equal(t, dash.Id, res[0].Id)
})
}
func getThumbnail(t *testing.T, sqlStore *SQLStore, dashboardUID string, orgId int64) *models.DashboardThumbnail {
t.Helper()
cmd := models.GetDashboardThumbnailCommand{
DashboardThumbnailMeta: models.DashboardThumbnailMeta{
DashboardUID: dashboardUID,
OrgId: orgId,
PanelID: 0,
Kind: models.ThumbnailKindDefault,
Theme: models.ThemeDark,
},
}
thumb, err := sqlStore.GetThumbnail(context.Background(), &cmd)
require.NoError(t, err)
return thumb
}
func upsertTestDashboardThumbnail(t *testing.T, sqlStore *SQLStore, dashboardUID string, orgId int64, dashboardVersion int) *models.DashboardThumbnail {
t.Helper()
cmd := models.SaveDashboardThumbnailCommand{
DashboardThumbnailMeta: models.DashboardThumbnailMeta{
DashboardUID: dashboardUID,
OrgId: orgId,
PanelID: 0,
Kind: models.ThumbnailKindDefault,
Theme: models.ThemeDark,
},
DashboardVersion: dashboardVersion,
Image: make([]byte, 0),
MimeType: "image/png",
}
dash, err := sqlStore.SaveThumbnail(context.Background(), &cmd)
require.NoError(t, err)
require.NotNil(t, dash)
return dash
}
func updateThumbnailState(t *testing.T, sqlStore *SQLStore, dashboardUID string, orgId int64, state models.ThumbnailState) {
t.Helper()
cmd := models.UpdateThumbnailStateCommand{
DashboardThumbnailMeta: models.DashboardThumbnailMeta{
DashboardUID: dashboardUID,
OrgId: orgId,
PanelID: 0,
Kind: models.ThumbnailKindDefault,
Theme: models.ThemeDark,
},
State: state,
}
err := sqlStore.UpdateThumbnailState(context.Background(), &cmd)
require.NoError(t, err)
}
@@ -0,0 +1,27 @@
package migrations
import "github.com/grafana/grafana/pkg/services/sqlstore/migrator"
func addDashboardThumbsMigrations(mg *migrator.Migrator) {
dashThumbs := migrator.Table{
Name: "dashboard_thumbnail",
Columns: []*migrator.Column{
{Name: "id", Type: migrator.DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
{Name: "dashboard_id", Type: migrator.DB_BigInt, Nullable: false}, // can join with dashboard table
{Name: "dashboard_version", Type: migrator.DB_Int, Nullable: false}, // screenshoted version of the dashboard
{Name: "state", Type: migrator.DB_NVarchar, Length: 10, Nullable: false}, // stale | locked
{Name: "panel_id", Type: migrator.DB_SmallInt, Nullable: false, Default: "0"}, // for panel thumbnails
{Name: "image", Type: migrator.DB_MediumBlob, Nullable: false}, // image stored as blob. MediumBlob has a max limit of 16mb in MySQL
{Name: "mime_type", Type: migrator.DB_NVarchar, Length: 255, Nullable: false}, // e.g. image/png, image/webp
{Name: "kind", Type: migrator.DB_NVarchar, Length: 8, Nullable: false}, // thumb | tall
{Name: "theme", Type: migrator.DB_NVarchar, Length: 8, Nullable: false}, // light|dark
{Name: "updated", Type: migrator.DB_DateTime, Nullable: false},
},
Indices: []*migrator.Index{
{Cols: []string{"dashboard_id", "panel_id", "kind", "theme"}, Type: migrator.UniqueIndex},
},
}
mg.AddMigration("create dashboard_thumbnail table", migrator.NewAddTableMigration(dashThumbs))
mg.AddMigration("add unique indexes for dashboard_thumbnail", migrator.NewAddIndexMigration(dashThumbs, dashThumbs.Indices[0]))
}
@@ -61,6 +61,9 @@ func (*OSSMigrations) AddMigration(mg *Migrator) {
if mg.Cfg.IsFeatureToggleEnabled(featuremgmt.FlagLiveConfig) {
addLiveChannelMigrations(mg)
}
if mg.Cfg.IsFeatureToggleEnabled(featuremgmt.FlagDashboardPreviews) {
addDashboardThumbsMigrations(mg)
}
}
ualert.RerunDashAlertMigration(mg)
+13 -1
View File
@@ -463,6 +463,10 @@ type InitTestDBOpt struct {
EnsureDefaultOrgAndUser bool
}
var featuresEnabledDuringTests = []string{
featuremgmt.FlagDashboardPreviews,
}
// InitTestDBWithMigration initializes the test DB given custom migrations.
func InitTestDBWithMigration(t ITestDB, migration registry.DatabaseMigrator, opts ...InitTestDBOpt) *SQLStore {
t.Helper()
@@ -500,7 +504,15 @@ func initTestDB(migration registry.DatabaseMigrator, opts ...InitTestDBOpt) (*SQ
// set test db config
cfg := setting.NewCfg()
cfg.IsFeatureToggleEnabled = func(key string) bool { return false }
cfg.IsFeatureToggleEnabled = func(requestedFeature string) bool {
for _, enabledFeature := range featuresEnabledDuringTests {
if enabledFeature == requestedFeature {
return true
}
}
return false
}
sec, err := cfg.Raw.NewSection("database")
if err != nil {
return nil, err