Chore: Split dashboard thumbnail service (#55344)

* Chore: split dashboard thumbnail service

* fix typo

* move tests

* make linter happy
This commit is contained in:
Serge Zaitsev
2022-09-19 11:29:22 +02:00
committed by GitHub
parent 1ee6a1f7c2
commit 96b032e103
7 changed files with 639 additions and 569 deletions
-211
View File
@@ -1,211 +0,0 @@
package sqlstore
import (
"context"
"encoding/json"
"errors"
"time"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/dashboards"
)
func (ss *SQLStore) GetThumbnail(ctx context.Context, query *models.GetDashboardThumbnailCommand) (*models.DashboardThumbnail, error) {
err := ss.WithDbSession(ctx, func(sess *DBSession) error {
result, err := findThumbnailByMeta(sess, query.DashboardThumbnailMeta)
if err != nil {
return err
}
query.Result = result
return nil
})
return query.Result, err
}
func marshalDatasourceUids(dsUids []string) (string, error) {
if dsUids == nil {
return "", nil
}
b, err := json.Marshal(dsUids)
if err != nil {
return "", err
}
return string(b), nil
}
func (ss *SQLStore) SaveThumbnail(ctx context.Context, cmd *models.SaveDashboardThumbnailCommand) (*models.DashboardThumbnail, error) {
err := ss.WithTransactionalDbSession(ctx, func(sess *DBSession) error {
existing, err := findThumbnailByMeta(sess, cmd.DashboardThumbnailMeta)
if err != nil && !errors.Is(err, dashboards.ErrDashboardThumbnailNotFound) {
return err
}
dsUids, err := marshalDatasourceUids(cmd.DatasourceUIDs)
if err != nil {
return err
}
if existing != nil {
existing.Image = cmd.Image
existing.MimeType = cmd.MimeType
existing.Updated = time.Now()
existing.DashboardVersion = cmd.DashboardVersion
existing.DsUIDs = dsUids
existing.State = models.ThumbnailStateDefault
_, err = sess.ID(existing.Id).Update(existing)
cmd.Result = existing
return err
}
thumb := &models.DashboardThumbnail{}
dash, err := findDashboardIdByThumbMeta(sess, cmd.DashboardThumbnailMeta)
if err != nil {
return err
}
thumb.Updated = time.Now()
thumb.Theme = cmd.Theme
thumb.Kind = cmd.Kind
thumb.Image = cmd.Image
thumb.DsUIDs = dsUids
thumb.MimeType = cmd.MimeType
thumb.DashboardId = dash.Id
thumb.DashboardVersion = cmd.DashboardVersion
thumb.State = models.ThumbnailStateDefault
thumb.PanelId = cmd.PanelID
_, err = sess.Insert(thumb)
cmd.Result = thumb
return err
})
return cmd.Result, err
}
func (ss *SQLStore) UpdateThumbnailState(ctx context.Context, cmd *models.UpdateThumbnailStateCommand) error {
err := ss.WithTransactionalDbSession(ctx, func(sess *DBSession) error {
existing, err := findThumbnailByMeta(sess, cmd.DashboardThumbnailMeta)
if err != nil {
return err
}
existing.State = cmd.State
_, err = sess.ID(existing.Id).Update(existing)
return err
})
return err
}
func (ss *SQLStore) FindThumbnailCount(ctx context.Context, cmd *models.FindDashboardThumbnailCountCommand) (int64, error) {
err := ss.WithDbSession(ctx, func(sess *DBSession) error {
count, err := sess.Count(&models.DashboardThumbnail{})
if err != nil {
return err
}
cmd.Result = count
return nil
})
return cmd.Result, err
}
func (ss *SQLStore) FindDashboardsWithStaleThumbnails(ctx context.Context, cmd *models.FindDashboardsWithStaleThumbnailsCommand) ([]*models.DashboardWithStaleThumbnail, error) {
err := ss.WithDbSession(ctx, func(sess *DBSession) error {
sess.Table("dashboard")
sess.Join("LEFT", "dashboard_thumbnail", "dashboard.id = dashboard_thumbnail.dashboard_id AND dashboard_thumbnail.theme = ? AND dashboard_thumbnail.kind = ?", cmd.Theme, cmd.Kind)
sess.Where("dashboard.is_folder = ?", dialect.BooleanStr(false))
query := "(dashboard.version != dashboard_thumbnail.dashboard_version " +
"OR dashboard_thumbnail.state = ? " +
"OR dashboard_thumbnail.id IS NULL"
args := []interface{}{models.ThumbnailStateStale}
if cmd.IncludeThumbnailsWithEmptyDsUIDs {
query += " OR dashboard_thumbnail.ds_uids = ? OR dashboard_thumbnail.ds_uids IS NULL"
args = append(args, "")
}
sess.Where(query+")", args...)
if !cmd.IncludeManuallyUploadedThumbnails {
sess.Where("(dashboard_thumbnail.id is not null AND dashboard_thumbnail.dashboard_version != ?) "+
"OR dashboard_thumbnail.id is null "+
"OR dashboard_thumbnail.state = ?", models.DashboardVersionForManualThumbnailUpload, models.ThumbnailStateStale)
}
sess.Where("(dashboard_thumbnail.id IS NULL OR dashboard_thumbnail.state != ?)", models.ThumbnailStateLocked)
sess.Cols("dashboard.id",
"dashboard.uid",
"dashboard.org_id",
"dashboard.version",
"dashboard.slug")
var result = make([]*models.DashboardWithStaleThumbnail, 0)
err := sess.Find(&result)
if err != nil {
return err
}
cmd.Result = result
return err
})
return cmd.Result, err
}
func findThumbnailByMeta(sess *DBSession, meta models.DashboardThumbnailMeta) (*models.DashboardThumbnail, error) {
result := &models.DashboardThumbnail{}
sess.Table("dashboard_thumbnail")
sess.Join("INNER", "dashboard", "dashboard.id = dashboard_thumbnail.dashboard_id")
sess.Where("dashboard.uid = ? AND dashboard.org_id = ? AND panel_id = ? AND kind = ? AND theme = ?", meta.DashboardUID, meta.OrgId, meta.PanelID, meta.Kind, meta.Theme)
sess.Cols("dashboard_thumbnail.id",
"dashboard_thumbnail.dashboard_id",
"dashboard_thumbnail.panel_id",
"dashboard_thumbnail.image",
"dashboard_thumbnail.dashboard_version",
"dashboard_thumbnail.state",
"dashboard_thumbnail.kind",
"dashboard_thumbnail.ds_uids",
"dashboard_thumbnail.mime_type",
"dashboard_thumbnail.theme",
"dashboard_thumbnail.updated")
exists, err := sess.Get(result)
if !exists {
return nil, dashboards.ErrDashboardThumbnailNotFound
}
if err != nil {
return nil, err
}
return result, nil
}
type dash struct {
Id int64
}
func findDashboardIdByThumbMeta(sess *DBSession, meta models.DashboardThumbnailMeta) (*dash, error) {
result := &dash{}
sess.Table("dashboard").Where("dashboard.uid = ? AND dashboard.org_id = ?", meta.DashboardUID, meta.OrgId).Cols("id")
exists, err := sess.Get(result)
if err != nil {
return nil, err
}
if !exists {
return nil, dashboards.ErrDashboardNotFound
}
return result, err
}
@@ -1,342 +0,0 @@
package sqlstore
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/dashboards"
dashver "github.com/grafana/grafana/pkg/services/dashboardversion"
"github.com/grafana/grafana/pkg/util"
)
var theme = models.ThemeDark
var kind = models.ThumbnailKindDefault
func TestIntegrationSqlStorage(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test")
}
var sqlStore *SQLStore
var savedFolder *models.Dashboard
setup := func() {
sqlStore = InitTestDB(t)
savedFolder = insertTestDashboard(t, sqlStore, "1 test dash folder", 1, 0, true, "prod", "webapp")
}
t.Run("Should insert dashboard in default state", func(t *testing.T) {
setup()
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version)
thumb := getThumbnail(t, sqlStore, dash.Uid, dash.OrgId)
require.Positive(t, thumb.Id)
require.Equal(t, models.ThumbnailStateDefault, thumb.State)
require.Equal(t, dash.Version, thumb.DashboardVersion)
})
t.Run("Should be able to update the thumbnail", func(t *testing.T) {
setup()
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version)
thumb := getThumbnail(t, sqlStore, dash.Uid, dash.OrgId)
insertedThumbnailId := thumb.Id
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version+1)
updatedThumb := getThumbnail(t, sqlStore, dash.Uid, dash.OrgId)
require.Equal(t, insertedThumbnailId, updatedThumb.Id)
require.Equal(t, dash.Version+1, updatedThumb.DashboardVersion)
})
t.Run("Should return empty array if all dashboards have thumbnails", func(t *testing.T) {
setup()
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version)
cmd := models.FindDashboardsWithStaleThumbnailsCommand{
Kind: kind,
Theme: theme,
}
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
require.NoError(t, err)
require.Len(t, res, 0)
})
t.Run("Should return dashboards with thumbnails with empty ds_uids array", func(t *testing.T) {
setup()
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version)
cmd := models.FindDashboardsWithStaleThumbnailsCommand{
Kind: kind,
IncludeThumbnailsWithEmptyDsUIDs: true,
Theme: theme,
}
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
require.NoError(t, err)
require.Len(t, res, 1)
require.Equal(t, dash.Id, res[0].Id)
})
t.Run("Should return dashboards with thumbnails marked as stale", func(t *testing.T) {
setup()
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version)
updateThumbnailState(t, sqlStore, dash.Uid, dash.OrgId, models.ThumbnailStateStale)
cmd := models.FindDashboardsWithStaleThumbnailsCommand{
Kind: kind,
Theme: theme,
}
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
require.NoError(t, err)
require.Len(t, res, 1)
require.Equal(t, dash.Id, res[0].Id)
})
t.Run("Should not return dashboards with updated thumbnails that had been marked as stale", func(t *testing.T) {
setup()
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version)
updateThumbnailState(t, sqlStore, dash.Uid, dash.OrgId, models.ThumbnailStateStale)
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version)
cmd := models.FindDashboardsWithStaleThumbnailsCommand{
Kind: kind,
Theme: theme,
}
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
require.NoError(t, err)
require.Len(t, res, 0)
})
t.Run("Should find dashboards without thumbnails", func(t *testing.T) {
setup()
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
cmd := models.FindDashboardsWithStaleThumbnailsCommand{
Kind: kind,
Theme: theme,
}
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
require.NoError(t, err)
require.Len(t, res, 1)
require.Equal(t, dash.Id, res[0].Id)
})
t.Run("Should find dashboards with outdated thumbnails", func(t *testing.T) {
setup()
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version)
updateTestDashboard(t, sqlStore, dash, map[string]interface{}{
"tags": "different-tag",
})
cmd := models.FindDashboardsWithStaleThumbnailsCommand{
Kind: kind,
Theme: theme,
}
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
require.NoError(t, err)
require.Len(t, res, 1)
require.Equal(t, dash.Id, res[0].Id)
})
t.Run("Should not return dashboards with locked thumbnails even if they are outdated", func(t *testing.T) {
setup()
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version)
updateThumbnailState(t, sqlStore, dash.Uid, dash.OrgId, models.ThumbnailStateLocked)
updateTestDashboard(t, sqlStore, dash, map[string]interface{}{
"tags": "different-tag",
})
cmd := models.FindDashboardsWithStaleThumbnailsCommand{
Kind: kind,
Theme: theme,
}
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
require.NoError(t, err)
require.Len(t, res, 0)
})
t.Run("Should not return dashboards with manually uploaded thumbnails by default", func(t *testing.T) {
setup()
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, models.DashboardVersionForManualThumbnailUpload)
updateTestDashboard(t, sqlStore, dash, map[string]interface{}{
"tags": "different-tag",
})
cmd := models.FindDashboardsWithStaleThumbnailsCommand{
Kind: kind,
Theme: theme,
}
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
require.NoError(t, err)
require.Len(t, res, 0)
})
t.Run("Should return dashboards with manually uploaded thumbnails if requested", func(t *testing.T) {
setup()
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, models.DashboardVersionForManualThumbnailUpload)
updateTestDashboard(t, sqlStore, dash, map[string]interface{}{
"tags": "different-tag",
})
cmd := models.FindDashboardsWithStaleThumbnailsCommand{
Kind: kind,
Theme: theme,
IncludeManuallyUploadedThumbnails: true,
}
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
require.NoError(t, err)
require.Len(t, res, 1)
require.Equal(t, dash.Id, res[0].Id)
})
t.Run("Should count all dashboard thumbnails", func(t *testing.T) {
setup()
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, 1)
dash2 := insertTestDashboard(t, sqlStore, "test dash 23", 2, savedFolder.Id, false, "prod", "webapp")
upsertTestDashboardThumbnail(t, sqlStore, dash2.Uid, dash2.OrgId, 1)
updateTestDashboard(t, sqlStore, dash, map[string]interface{}{
"tags": "different-tag",
})
cmd := models.FindDashboardThumbnailCountCommand{}
res, err := sqlStore.FindThumbnailCount(context.Background(), &cmd)
require.NoError(t, err)
require.Equal(t, res, int64(2))
})
}
func getThumbnail(t *testing.T, sqlStore *SQLStore, dashboardUID string, orgId int64) *models.DashboardThumbnail {
t.Helper()
cmd := models.GetDashboardThumbnailCommand{
DashboardThumbnailMeta: models.DashboardThumbnailMeta{
DashboardUID: dashboardUID,
OrgId: orgId,
PanelID: 0,
Kind: kind,
Theme: theme,
},
}
thumb, err := sqlStore.GetThumbnail(context.Background(), &cmd)
require.NoError(t, err)
return thumb
}
func upsertTestDashboardThumbnail(t *testing.T, sqlStore *SQLStore, dashboardUID string, orgId int64, dashboardVersion int) *models.DashboardThumbnail {
t.Helper()
cmd := models.SaveDashboardThumbnailCommand{
DashboardThumbnailMeta: models.DashboardThumbnailMeta{
DashboardUID: dashboardUID,
OrgId: orgId,
PanelID: 0,
Kind: kind,
Theme: theme,
},
DashboardVersion: dashboardVersion,
Image: make([]byte, 0),
MimeType: "image/png",
}
dash, err := sqlStore.SaveThumbnail(context.Background(), &cmd)
require.NoError(t, err)
require.NotNil(t, dash)
return dash
}
func updateThumbnailState(t *testing.T, sqlStore *SQLStore, dashboardUID string, orgId int64, state models.ThumbnailState) {
t.Helper()
cmd := models.UpdateThumbnailStateCommand{
DashboardThumbnailMeta: models.DashboardThumbnailMeta{
DashboardUID: dashboardUID,
OrgId: orgId,
PanelID: 0,
Kind: kind,
Theme: theme,
},
State: state,
}
err := sqlStore.UpdateThumbnailState(context.Background(), &cmd)
require.NoError(t, err)
}
func updateTestDashboard(t *testing.T, sqlStore *SQLStore, dashboard *models.Dashboard, data map[string]interface{}) {
t.Helper()
data["id"] = dashboard.Id
parentVersion := dashboard.Version
cmd := models.SaveDashboardCommand{
OrgId: dashboard.OrgId,
Overwrite: true,
Dashboard: simplejson.NewFromAny(data),
}
var dash *models.Dashboard
err := sqlStore.WithDbSession(context.Background(), func(sess *DBSession) error {
var existing models.Dashboard
dash = cmd.GetDashboardModel()
dashWithIdExists, err := sess.Where("id=? AND org_id=?", dash.Id, dash.OrgId).Get(&existing)
require.NoError(t, err)
require.True(t, dashWithIdExists)
if dash.Version != existing.Version {
dash.SetVersion(existing.Version)
dash.Version = existing.Version
}
dash.SetVersion(dash.Version + 1)
dash.Created = time.Now()
dash.Updated = time.Now()
dash.Id = dashboard.Id
dash.Uid = util.GenerateShortUID()
_, err = sess.MustCols("folder_id").ID(dash.Id).Update(dash)
return err
})
require.Nil(t, err)
err = sqlStore.WithDbSession(context.Background(), func(sess *DBSession) error {
dashVersion := &dashver.DashboardVersion{
DashboardID: dash.Id,
ParentVersion: parentVersion,
RestoredFrom: cmd.RestoredFrom,
Version: dash.Version,
Created: time.Now(),
CreatedBy: dash.UpdatedBy,
Message: cmd.Message,
Data: dash.Data,
}
if affectedRows, err := sess.Insert(dashVersion); err != nil {
return err
} else if affectedRows == 0 {
return dashboards.ErrDashboardNotFound
}
return nil
})
require.NoError(t, err)
}