diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index 47c68323e99..b3fd3adee7f 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -22,6 +22,7 @@ export interface FeatureToggles { ['service-accounts']?: boolean; database_metrics?: boolean; dashboardPreviews?: boolean; + dashboardPreviewsScheduler?: boolean; ['live-config']?: boolean; ['live-pipeline']?: boolean; ['live-service-web-worker']?: boolean; diff --git a/pkg/models/dashboard_thumbs.go b/pkg/models/dashboard_thumbs.go index da8ee5212e5..954a07b1a42 100644 --- a/pkg/models/dashboard_thumbs.go +++ b/pkg/models/dashboard_thumbs.go @@ -119,6 +119,8 @@ type DashboardWithStaleThumbnail struct { type FindDashboardsWithStaleThumbnailsCommand struct { IncludeManuallyUploadedThumbnails bool + Theme Theme + Kind ThumbnailKind Result []*DashboardWithStaleThumbnail } diff --git a/pkg/models/dashboards.go b/pkg/models/dashboards.go index 5c4b37fb970..ff11aa463b3 100644 --- a/pkg/models/dashboards.go +++ b/pkg/models/dashboards.go @@ -323,8 +323,8 @@ func GetDashboardUrl(uid string, slug string) string { } // GetKioskModeDashboardUrl returns the HTML url for a dashboard in kiosk mode. -func GetKioskModeDashboardUrl(uid string, slug string) string { - return fmt.Sprintf("%s?kiosk", GetDashboardUrl(uid, slug)) +func GetKioskModeDashboardUrl(uid string, slug string, theme Theme) string { + return fmt.Sprintf("%s?kiosk&theme=%s", GetDashboardUrl(uid, slug), string(theme)) } // GetFullDashboardUrl returns the full URL for a dashboard. diff --git a/pkg/server/backgroundsvcs/background_services.go b/pkg/server/backgroundsvcs/background_services.go index 5f413a34fd0..1cc420ff98f 100644 --- a/pkg/server/backgroundsvcs/background_services.go +++ b/pkg/server/backgroundsvcs/background_services.go @@ -22,6 +22,7 @@ import ( "github.com/grafana/grafana/pkg/services/rendering" secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" "github.com/grafana/grafana/pkg/services/serviceaccounts" + "github.com/grafana/grafana/pkg/services/thumbs" "github.com/grafana/grafana/pkg/services/updatechecker" ) @@ -32,7 +33,7 @@ func ProvideBackgroundServiceRegistry( provisioning *provisioning.ProvisioningServiceImpl, alerting *alerting.AlertEngine, usageStats *uss.UsageStats, grafanaUpdateChecker *updatechecker.GrafanaService, pluginsUpdateChecker *updatechecker.PluginsService, metrics *metrics.InternalMetricsService, secretsService *secretsManager.SecretsService, - remoteCache *remotecache.RemoteCache, + remoteCache *remotecache.RemoteCache, thumbnailsService thumbs.Service, // Need to make sure these are initialized, is there a better place to put them? _ *plugindashboards.Service, _ *dashboardsnapshots.Service, _ *pluginsettings.Service, _ *alerting.AlertNotificationService, _ serviceaccounts.Service, @@ -55,7 +56,8 @@ func ProvideBackgroundServiceRegistry( usageStats, tracing, remoteCache, - secretsService) + secretsService, + thumbnailsService) } // BackgroundServiceRegistry provides background services. diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index ec3fcb34f93..454abca8278 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -33,6 +33,11 @@ var ( Description: "Create and show thumbnails for dashboard search results", State: FeatureStateAlpha, }, + { + Name: "dashboardPreviewsScheduler", + Description: "Schedule automatic updates to dashboard previews", + State: FeatureStateAlpha, + }, { Name: "live-config", Description: "Save grafana live configuration in SQL tables", diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index ec589a510c3..452fb73d6d2 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -27,6 +27,10 @@ const ( // Create and show thumbnails for dashboard search results FlagDashboardPreviews = "dashboardPreviews" + // FlagDashboardPreviewsScheduler + // Schedule automatic updates to dashboard previews + FlagDashboardPreviewsScheduler = "dashboardPreviewsScheduler" + // FlagLiveConfig // Save grafana live configuration in SQL tables FlagLiveConfig = "live-config" diff --git a/pkg/services/sqlstore/dashboard_thumbs.go b/pkg/services/sqlstore/dashboard_thumbs.go index f2e1814c2bd..6b819ca46a2 100644 --- a/pkg/services/sqlstore/dashboard_thumbs.go +++ b/pkg/services/sqlstore/dashboard_thumbs.go @@ -84,7 +84,7 @@ func (ss *SQLStore) UpdateThumbnailState(ctx context.Context, cmd *models.Update func (ss *SQLStore) FindDashboardsWithStaleThumbnails(ctx context.Context, cmd *models.FindDashboardsWithStaleThumbnailsCommand) ([]*models.DashboardWithStaleThumbnail, error) { err := ss.WithDbSession(ctx, func(sess *DBSession) error { sess.Table("dashboard") - sess.Join("LEFT", "dashboard_thumbnail", "dashboard.id = dashboard_thumbnail.dashboard_id") + sess.Join("LEFT", "dashboard_thumbnail", "dashboard.id = dashboard_thumbnail.dashboard_id AND dashboard_thumbnail.theme = ? AND dashboard_thumbnail.kind = ?", cmd.Theme, cmd.Kind) sess.Where("dashboard.is_folder = ?", dialect.BooleanStr(false)) sess.Where("(dashboard.version != dashboard_thumbnail.dashboard_version "+ "OR dashboard_thumbnail.state = ? "+ diff --git a/pkg/services/sqlstore/dashboard_thumbs_test.go b/pkg/services/sqlstore/dashboard_thumbs_test.go index d2f363f5ac4..5cba1949f64 100644 --- a/pkg/services/sqlstore/dashboard_thumbs_test.go +++ b/pkg/services/sqlstore/dashboard_thumbs_test.go @@ -11,6 +11,9 @@ import ( "github.com/stretchr/testify/require" ) +var theme = models.ThemeDark +var kind = models.ThumbnailKindDefault + func TestSqlStorage(t *testing.T) { var sqlStore *SQLStore @@ -52,7 +55,10 @@ func TestSqlStorage(t *testing.T) { upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version) - cmd := models.FindDashboardsWithStaleThumbnailsCommand{} + cmd := models.FindDashboardsWithStaleThumbnailsCommand{ + Kind: kind, + Theme: theme, + } res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd) require.NoError(t, err) require.Len(t, res, 0) @@ -64,7 +70,10 @@ func TestSqlStorage(t *testing.T) { upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version) updateThumbnailState(t, sqlStore, dash.Uid, dash.OrgId, models.ThumbnailStateStale) - cmd := models.FindDashboardsWithStaleThumbnailsCommand{} + cmd := models.FindDashboardsWithStaleThumbnailsCommand{ + Kind: kind, + Theme: theme, + } res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd) require.NoError(t, err) require.Len(t, res, 1) @@ -78,7 +87,10 @@ func TestSqlStorage(t *testing.T) { updateThumbnailState(t, sqlStore, dash.Uid, dash.OrgId, models.ThumbnailStateStale) upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version) - cmd := models.FindDashboardsWithStaleThumbnailsCommand{} + cmd := models.FindDashboardsWithStaleThumbnailsCommand{ + Kind: kind, + Theme: theme, + } res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd) require.NoError(t, err) require.Len(t, res, 0) @@ -88,7 +100,10 @@ func TestSqlStorage(t *testing.T) { setup() dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp") - cmd := models.FindDashboardsWithStaleThumbnailsCommand{} + cmd := models.FindDashboardsWithStaleThumbnailsCommand{ + Kind: kind, + Theme: theme, + } res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd) require.NoError(t, err) require.Len(t, res, 1) @@ -104,7 +119,10 @@ func TestSqlStorage(t *testing.T) { "tags": "different-tag", }) - cmd := models.FindDashboardsWithStaleThumbnailsCommand{} + cmd := models.FindDashboardsWithStaleThumbnailsCommand{ + Kind: kind, + Theme: theme, + } res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd) require.NoError(t, err) require.Len(t, res, 1) @@ -121,7 +139,10 @@ func TestSqlStorage(t *testing.T) { "tags": "different-tag", }) - cmd := models.FindDashboardsWithStaleThumbnailsCommand{} + cmd := models.FindDashboardsWithStaleThumbnailsCommand{ + Kind: kind, + Theme: theme, + } res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd) require.NoError(t, err) require.Len(t, res, 0) @@ -136,7 +157,10 @@ func TestSqlStorage(t *testing.T) { "tags": "different-tag", }) - cmd := models.FindDashboardsWithStaleThumbnailsCommand{} + cmd := models.FindDashboardsWithStaleThumbnailsCommand{ + Kind: kind, + Theme: theme, + } res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd) require.NoError(t, err) require.Len(t, res, 0) @@ -152,6 +176,8 @@ func TestSqlStorage(t *testing.T) { }) cmd := models.FindDashboardsWithStaleThumbnailsCommand{ + Kind: kind, + Theme: theme, IncludeManuallyUploadedThumbnails: true, } res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd) @@ -168,8 +194,8 @@ func getThumbnail(t *testing.T, sqlStore *SQLStore, dashboardUID string, orgId i DashboardUID: dashboardUID, OrgId: orgId, PanelID: 0, - Kind: models.ThumbnailKindDefault, - Theme: models.ThemeDark, + Kind: kind, + Theme: theme, }, } @@ -185,8 +211,8 @@ func upsertTestDashboardThumbnail(t *testing.T, sqlStore *SQLStore, dashboardUID DashboardUID: dashboardUID, OrgId: orgId, PanelID: 0, - Kind: models.ThumbnailKindDefault, - Theme: models.ThemeDark, + Kind: kind, + Theme: theme, }, DashboardVersion: dashboardVersion, Image: make([]byte, 0), @@ -206,8 +232,8 @@ func updateThumbnailState(t *testing.T, sqlStore *SQLStore, dashboardUID string, DashboardUID: dashboardUID, OrgId: orgId, PanelID: 0, - Kind: models.ThumbnailKindDefault, - Theme: models.ThemeDark, + Kind: kind, + Theme: theme, }, State: state, } diff --git a/pkg/services/thumbs/crawler.go b/pkg/services/thumbs/crawler.go index 73474708a88..d9ef0118eba 100644 --- a/pkg/services/thumbs/crawler.go +++ b/pkg/services/thumbs/crawler.go @@ -8,6 +8,9 @@ import ( "sync" "time" + "golang.org/x/sync/errgroup" + + "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/live" "github.com/grafana/grafana/pkg/services/rendering" @@ -26,6 +29,7 @@ type simpleCrawler struct { statusMutex sync.RWMutex queue []*models.DashboardWithStaleThumbnail queueMutex sync.Mutex + log log.Logger renderingSession rendering.Session } @@ -35,6 +39,7 @@ func newSimpleCrawler(renderService rendering.Service, gl *live.GrafanaLive, rep threadCount: 6, glive: gl, thumbnailRepo: repo, + log: log.New("thumbnails_crawler"), status: crawlStatus{ State: initializing, Complete: 0, @@ -62,63 +67,47 @@ func (r *simpleCrawler) next() *models.DashboardWithStaleThumbnail { func (r *simpleCrawler) broadcastStatus() { s, err := r.Status() if err != nil { - tlog.Warn("error reading status") + r.log.Warn("Error reading status", "err", err) return } msg, err := json.Marshal(s) if err != nil { - tlog.Warn("error making message") + r.log.Warn("Error making message", "err", err) return } err = r.glive.Publish(r.opts.OrgID, "grafana/broadcast/crawler", msg) if err != nil { - tlog.Warn("error Publish message") + r.log.Warn("Error Publish message", "err", err) return } } -func (r *simpleCrawler) Start(c *models.ReqContext, mode CrawlerMode, theme models.Theme, thumbnailKind models.ThumbnailKind) (crawlStatus, error) { - if r.status.State == running { - tlog.Info("already running") - return r.Status() - } - +func (r *simpleCrawler) Run(ctx context.Context, authOpts rendering.AuthOpts, mode CrawlerMode, theme models.Theme, thumbnailKind models.ThumbnailKind) error { r.queueMutex.Lock() - defer r.queueMutex.Unlock() + if r.IsRunning() { + r.queueMutex.Unlock() + r.log.Info("Already running") + return nil + } now := time.Now() - ctx := c.Req.Context() - items, err := r.thumbnailRepo.findDashboardsWithStaleThumbnails(ctx) + items, err := r.thumbnailRepo.findDashboardsWithStaleThumbnails(ctx, theme, thumbnailKind) if err != nil { - tlog.Error("error when fetching dashboards with stale thumbnails", "err", err.Error()) - return crawlStatus{ - Started: now, - Finished: now, - Last: now, - State: stopped, - Complete: 0, - }, err + r.log.Error("Error when fetching dashboards with stale thumbnails", "err", err.Error()) + r.queueMutex.Unlock() + return err } if len(items) == 0 { - return crawlStatus{ - Started: now, - Finished: now, - Last: now, - State: stopped, - Complete: 0, - }, err + r.queueMutex.Unlock() + return nil } r.mode = mode r.thumbnailKind = thumbnailKind r.opts = rendering.Opts{ - AuthOpts: rendering.AuthOpts{ - OrgID: c.OrgId, - UserID: c.UserId, - OrgRole: c.OrgRole, - }, + AuthOpts: authOpts, TimeoutOpts: rendering.TimeoutOpts{ Timeout: 10 * time.Second, RequestTimeoutMultiplier: 3, @@ -126,19 +115,14 @@ func (r *simpleCrawler) Start(c *models.ReqContext, mode CrawlerMode, theme mode Theme: theme, ConcurrentLimit: 10, } - renderingSession, err := r.renderService.CreateRenderingSession(context.Background(), r.opts.AuthOpts, rendering.SessionOpts{ + renderingSession, err := r.renderService.CreateRenderingSession(ctx, r.opts.AuthOpts, rendering.SessionOpts{ Expiry: 5 * time.Minute, RefreshExpiryOnEachRequest: true, }) if err != nil { - tlog.Error("error when creating rendering session", "err", err.Error()) - return crawlStatus{ - Started: now, - Finished: now, - Last: now, - State: stopped, - Complete: 0, - }, err + r.log.Error("Error when creating rendering session", "err", err.Error()) + r.queueMutex.Unlock() + return err } r.renderingSession = renderingSession @@ -149,14 +133,34 @@ func (r *simpleCrawler) Start(c *models.ReqContext, mode CrawlerMode, theme mode Complete: 0, } r.broadcastStatus() + r.queueMutex.Unlock() - tlog.Info("Starting dashboard crawler", "dashboardsToCrawl", len(items)) + r.log.Info("Starting dashboard crawler", "dashboardsToCrawl", len(items), "mode", string(mode), "theme", string(theme), "kind", string(thumbnailKind)) + group, gCtx := errgroup.WithContext(ctx) // create a pool of workers for i := 0; i < r.threadCount; i++ { - go r.walk(ctx) + walkerId := i + group.Go(func() error { + r.walk(gCtx, walkerId) + return nil + }) } - return r.Status() + + err = group.Wait() + if err != nil { + r.log.Error("Crawl ended with an error", "err", err) + } + + r.crawlFinished() + r.broadcastStatus() + return err +} + +func (r *simpleCrawler) IsRunning() bool { + r.statusMutex.Lock() + defer r.statusMutex.Unlock() + return r.status.State == running } func (r *simpleCrawler) Stop() (crawlStatus, error) { @@ -200,13 +204,13 @@ func (r *simpleCrawler) newSuccessResult() { r.status.Last = time.Now() } -func (r *simpleCrawler) walkFinished() { +func (r *simpleCrawler) crawlFinished() { r.statusMutex.Lock() defer r.statusMutex.Unlock() r.status.State = stopped r.status.Finished = time.Now() - tlog.Info("Crawler finished", "startTime", r.status.Started, "endTime", r.status.Finished, "durationInSeconds", int64(time.Since(r.status.Started)/time.Second)) + r.log.Info("Crawler finished", "startTime", r.status.Started, "endTime", r.status.Finished, "durationInSeconds", int64(time.Since(r.status.Started)/time.Second)) } func (r *simpleCrawler) shouldWalk() bool { @@ -216,7 +220,7 @@ func (r *simpleCrawler) shouldWalk() bool { return r.status.State == running } -func (r *simpleCrawler) walk(ctx context.Context) { +func (r *simpleCrawler) walk(ctx context.Context, id int) { for { if !r.shouldWalk() { break @@ -227,10 +231,10 @@ func (r *simpleCrawler) walk(ctx context.Context) { break } - url := models.GetKioskModeDashboardUrl(item.Uid, item.Slug) - tlog.Info("Getting dashboard thumbnail", "dashboardUID", item.Uid, "url", url) + url := models.GetKioskModeDashboardUrl(item.Uid, item.Slug, r.opts.Theme) + r.log.Info("Getting dashboard thumbnail", "walkerId", id, "dashboardUID", item.Uid, "url", url) - res, err := r.renderService.Render(context.Background(), rendering.Opts{ + res, err := r.renderService.Render(ctx, rendering.Opts{ Width: 320, Height: 240, Path: strings.TrimPrefix(url, "/"), @@ -241,13 +245,13 @@ func (r *simpleCrawler) walk(ctx context.Context) { DeviceScaleFactor: -5, // negative numbers will render larger and then scale down. }, r.renderingSession) if err != nil { - tlog.Warn("error getting image", "dashboardUID", item.Uid, "url", url, "err", err) + r.log.Warn("Error getting image", "walkerId", id, "dashboardUID", item.Uid, "url", url, "err", err) r.newErrorResult() } else if res.FilePath == "" { - tlog.Warn("error getting image... no response", "dashboardUID", item.Uid, "url", url) + r.log.Warn("Error getting image... no response", "walkerId", id, "dashboardUID", item.Uid, "url", url) r.newErrorResult() } else if strings.Contains(res.FilePath, "public/img") { - tlog.Warn("error getting image... internal result", "dashboardUID", item.Uid, "url", url, "img", res.FilePath) + r.log.Warn("Error getting image... internal result", "walkerId", id, "dashboardUID", item.Uid, "url", url, "img", res.FilePath) // rendering service returned a static error image - we should not remove that file r.newErrorResult() } else { @@ -255,7 +259,7 @@ func (r *simpleCrawler) walk(ctx context.Context) { defer func() { err := os.Remove(res.FilePath) if err != nil { - tlog.Error("failed to remove thumbnail temp file", "dashboardUID", item.Uid, "url", url, "err", err) + r.log.Error("Failed to remove thumbnail temp file", "walkerId", id, "dashboardUID", item.Uid, "url", url, "err", err) } }() @@ -267,10 +271,10 @@ func (r *simpleCrawler) walk(ctx context.Context) { }, item.Version) if err != nil { - tlog.Warn("error saving image image", "dashboardUID", item.Uid, "url", url, "err", err) + r.log.Warn("Error saving image image", "walkerId", id, "dashboardUID", item.Uid, "url", url, "err", err) r.newErrorResult() } else { - tlog.Info("saved thumbnail", "dashboardUID", item.Uid, "url", url, "thumbnailId", thumbnailId) + r.log.Info("Saved thumbnail", "walkerId", id, "dashboardUID", item.Uid, "url", url, "thumbnailId", thumbnailId) r.newSuccessResult() } }() @@ -278,6 +282,5 @@ func (r *simpleCrawler) walk(ctx context.Context) { r.broadcastStatus() } - r.walkFinished() - r.broadcastStatus() + r.log.Info("Walker finished", "walkerId", id) } diff --git a/pkg/services/thumbs/dummy.go b/pkg/services/thumbs/dummy.go index 800cd25bd05..45fd30f3b2a 100644 --- a/pkg/services/thumbs/dummy.go +++ b/pkg/services/thumbs/dummy.go @@ -1,6 +1,8 @@ package thumbs import ( + "context" + "github.com/grafana/grafana/pkg/api/response" "github.com/grafana/grafana/pkg/models" ) @@ -41,3 +43,7 @@ func (ds *dummyService) CrawlerStatus(c *models.ReqContext) response.Response { result["error"] = "Not enabled" return response.JSON(200, result) } + +func (ds *dummyService) Run(ctx context.Context) error { + return nil +} diff --git a/pkg/services/thumbs/models.go b/pkg/services/thumbs/models.go index 0accd58cb95..38b2448b302 100644 --- a/pkg/services/thumbs/models.go +++ b/pkg/services/thumbs/models.go @@ -5,6 +5,7 @@ import ( "time" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/rendering" ) type CrawlerMode string @@ -55,13 +56,15 @@ type crawlStatus struct { type dashRenderer interface { // Run Assumes you have already authenticated as admin. - Start(c *models.ReqContext, mode CrawlerMode, theme models.Theme, kind models.ThumbnailKind) (crawlStatus, error) + Run(ctx context.Context, authOpts rendering.AuthOpts, mode CrawlerMode, theme models.Theme, kind models.ThumbnailKind) error // Assumes you have already authenticated as admin. Stop() (crawlStatus, error) // Assumes you have already authenticated as admin. Status() (crawlStatus, error) + + IsRunning() bool } type thumbnailRepo interface { @@ -69,5 +72,5 @@ type thumbnailRepo interface { saveFromFile(ctx context.Context, filePath string, meta models.DashboardThumbnailMeta, dashboardVersion int) (int64, error) saveFromBytes(ctx context.Context, bytes []byte, mimeType string, meta models.DashboardThumbnailMeta, dashboardVersion int) (int64, error) getThumbnail(ctx context.Context, meta models.DashboardThumbnailMeta) (*models.DashboardThumbnail, error) - findDashboardsWithStaleThumbnails(ctx context.Context) ([]*models.DashboardWithStaleThumbnail, error) + findDashboardsWithStaleThumbnails(ctx context.Context, theme models.Theme, thumbnailKind models.ThumbnailKind) ([]*models.DashboardWithStaleThumbnail, error) } diff --git a/pkg/services/thumbs/repo.go b/pkg/services/thumbs/repo.go index d37727ed13a..7ce5d0aa006 100644 --- a/pkg/services/thumbs/repo.go +++ b/pkg/services/thumbs/repo.go @@ -7,6 +7,7 @@ import ( "path/filepath" "strings" + "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/sqlstore" ) @@ -14,12 +15,14 @@ import ( func newThumbnailRepo(store *sqlstore.SQLStore) thumbnailRepo { repo := &sqlThumbnailRepository{ store: store, + log: log.New("thumbnails_repo"), } return repo } type sqlThumbnailRepository struct { store *sqlstore.SQLStore + log log.Logger } func (r *sqlThumbnailRepository) saveFromFile(ctx context.Context, filePath string, meta models.DashboardThumbnailMeta, dashboardVersion int) (int64, error) { @@ -28,14 +31,14 @@ func (r *sqlThumbnailRepository) saveFromFile(ctx context.Context, filePath stri // 2. the rendering service, when image-renderer returns a screenshot if !filepath.IsAbs(filePath) { - tlog.Error("Received relative path", "dashboardUID", meta.DashboardUID, "err", filePath) + r.log.Error("Received relative path", "dashboardUID", meta.DashboardUID, "err", filePath) return 0, errors.New("relative paths are not supported") } content, err := os.ReadFile(filepath.Clean(filePath)) if err != nil { - tlog.Error("error reading file", "dashboardUID", meta.DashboardUID, "err", err) + r.log.Error("error reading file", "dashboardUID", meta.DashboardUID, "err", err) return 0, err } @@ -60,7 +63,7 @@ func (r *sqlThumbnailRepository) saveFromBytes(ctx context.Context, content []by _, err := r.store.SaveThumbnail(ctx, cmd) if err != nil { - tlog.Error("error saving to the db", "dashboardUID", meta.DashboardUID, "err", err) + r.log.Error("error saving to the db", "dashboardUID", meta.DashboardUID, "err", err) return 0, err } @@ -81,8 +84,10 @@ func (r *sqlThumbnailRepository) getThumbnail(ctx context.Context, meta models.D return r.store.GetThumbnail(ctx, query) } -func (r *sqlThumbnailRepository) findDashboardsWithStaleThumbnails(ctx context.Context) ([]*models.DashboardWithStaleThumbnail, error) { +func (r *sqlThumbnailRepository) findDashboardsWithStaleThumbnails(ctx context.Context, theme models.Theme, kind models.ThumbnailKind) ([]*models.DashboardWithStaleThumbnail, error) { return r.store.FindDashboardsWithStaleThumbnails(ctx, &models.FindDashboardsWithStaleThumbnailsCommand{ IncludeManuallyUploadedThumbnails: false, + Theme: theme, + Kind: kind, }) } diff --git a/pkg/services/thumbs/service.go b/pkg/services/thumbs/service.go index 990fb6df7d8..2de7249389c 100644 --- a/pkg/services/thumbs/service.go +++ b/pkg/services/thumbs/service.go @@ -1,14 +1,17 @@ package thumbs import ( + "context" "errors" "fmt" "io" "io/ioutil" + "time" "github.com/grafana/grafana/pkg/api/response" "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/infra/serverlock" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/guardian" @@ -20,11 +23,8 @@ import ( "github.com/segmentio/encoding/json" ) -var ( - tlog log.Logger = log.New("thumbnails") -) - type Service interface { + Run(ctx context.Context) error Enabled() bool GetImage(c *models.ReqContext) @@ -38,25 +38,60 @@ type Service interface { CrawlerStatus(c *models.ReqContext) response.Response } -func ProvideService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, renderService rendering.Service, gl *live.GrafanaLive, store *sqlstore.SQLStore) Service { +type thumbService struct { + scheduleOptions crawlerScheduleOptions + renderer dashRenderer + thumbnailRepo thumbnailRepo + lockService *serverlock.ServerLockService + features featuremgmt.FeatureToggles + crawlLockServiceActionName string + log log.Logger +} + +type crawlerScheduleOptions struct { + crawlInterval time.Duration + tickerInterval time.Duration + maxCrawlDuration time.Duration + crawlerMode CrawlerMode + thumbnailKind models.ThumbnailKind + auth rendering.AuthOpts + themes []models.Theme +} + +func ProvideService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, lockService *serverlock.ServerLockService, renderService rendering.Service, gl *live.GrafanaLive, store *sqlstore.SQLStore) Service { if !features.IsEnabled(featuremgmt.FlagDashboardPreviews) { return &dummyService{} } thumbnailRepo := newThumbnailRepo(store) + + authOpts := rendering.AuthOpts{ + OrgID: 0, + UserID: 0, + OrgRole: models.ROLE_ADMIN, + } return &thumbService{ - renderer: newSimpleCrawler(renderService, gl, thumbnailRepo), - thumbnailRepo: thumbnailRepo, + renderer: newSimpleCrawler(renderService, gl, thumbnailRepo), + thumbnailRepo: thumbnailRepo, + features: features, + lockService: lockService, + crawlLockServiceActionName: "dashboard-crawler", + log: log.New("thumbnails_service"), + + scheduleOptions: crawlerScheduleOptions{ + tickerInterval: time.Hour, + crawlInterval: time.Hour * 12, + maxCrawlDuration: time.Hour, + crawlerMode: CrawlerModeThumbs, + thumbnailKind: models.ThumbnailKindDefault, + themes: []models.Theme{models.ThemeDark, models.ThemeLight}, + auth: authOpts, + }, } } -type thumbService struct { - renderer dashRenderer - thumbnailRepo thumbnailRepo -} - func (hs *thumbService) Enabled() bool { - return true + return hs.features.IsEnabled(featuremgmt.FlagDashboardPreviews) } func (hs *thumbService) parseImageReq(c *models.ReqContext, checkSave bool) *previewRequest { @@ -109,7 +144,7 @@ func (hs *thumbService) UpdateThumbnailState(c *models.ReqContext) { err := web.Bind(c.Req, body) if err != nil { - tlog.Error("Error parsing update thumbnail state request", "dashboardUid", req.UID, "err", err.Error()) + hs.log.Error("Error parsing update thumbnail state request", "dashboardUid", req.UID, "err", err.Error()) c.JSON(500, map[string]string{"dashboardUID": req.UID, "error": "unknown"}) return } @@ -122,12 +157,12 @@ func (hs *thumbService) UpdateThumbnailState(c *models.ReqContext) { }) if err != nil { - tlog.Error("Error when trying to update thumbnail state", "dashboardUid", req.UID, "err", err.Error(), "newState", body.State) + hs.log.Error("Error when trying to update thumbnail state", "dashboardUid", req.UID, "err", err.Error(), "newState", body.State) c.JSON(500, map[string]string{"dashboardUID": req.UID, "error": "unknown"}) return } - tlog.Info("Updated dashboard thumbnail state", "dashboardUid", req.UID, "theme", req.Theme, "newState", body.State) + hs.log.Info("Updated dashboard thumbnail state", "dashboardUid", req.UID, "theme", req.Theme, "newState", body.State) c.JSON(200, map[string]string{"success": "true"}) } @@ -150,14 +185,14 @@ func (hs *thumbService) GetImage(c *models.ReqContext) { } if err != nil { - tlog.Error("Error when retrieving thumbnail", "dashboardUid", req.UID, "err", err.Error()) + hs.log.Error("Error when retrieving thumbnail", "dashboardUid", req.UID, "err", err.Error()) c.JSON(500, map[string]string{"dashboardUID": req.UID, "error": "unknown"}) return } c.Resp.Header().Set("Content-Type", res.MimeType) if _, err := c.Resp.Write(res.Image); err != nil { - tlog.Error("Error writing to response", "dashboardUid", req.UID, "err", err) + hs.log.Error("Error writing to response", "dashboardUid", req.UID, "err", err) } } @@ -190,9 +225,9 @@ func (hs *thumbService) SetImage(c *models.ReqContext) { defer func() { _ = file.Close() }() - tlog.Info("Uploaded File: %+v\n", handler.Filename) - tlog.Info("File Size: %+v\n", handler.Size) - tlog.Info("MIME Header: %+v\n", handler.Header) + hs.log.Info("Uploaded File: %+v\n", handler.Filename) + hs.log.Info("File Size: %+v\n", handler.Size) + hs.log.Info("MIME Header: %+v\n", handler.Header) fileBytes, err := ioutil.ReadAll(file) if err != nil { @@ -230,11 +265,19 @@ func (hs *thumbService) StartCrawler(c *models.ReqContext) response.Response { if cmd.Mode == "" { cmd.Mode = CrawlerModeThumbs } - msg, err := hs.renderer.Start(c, cmd.Mode, cmd.Theme, models.ThumbnailKindDefault) + + go hs.runOnDemandCrawl(context.Background(), cmd.Theme, cmd.Mode, models.ThumbnailKindDefault, rendering.AuthOpts{ + OrgID: c.OrgId, + UserID: c.UserId, + OrgRole: c.OrgRole, + }) + + status, err := hs.renderer.Status() if err != nil { return response.Error(500, "error starting", err) } - return response.JSON(200, msg) + + return response.JSON(200, status) } func (hs *thumbService) StopCrawler(c *models.ReqContext) response.Response { @@ -284,3 +327,57 @@ func (hs *thumbService) getDashboardId(c *models.ReqContext, uid string) (int64, return query.Result.Id, nil } + +func (hs *thumbService) runOnDemandCrawl(parentCtx context.Context, theme models.Theme, mode CrawlerMode, kind models.ThumbnailKind, authOpts rendering.AuthOpts) { + crawlerCtx, cancel := context.WithTimeout(parentCtx, hs.scheduleOptions.maxCrawlDuration) + defer cancel() + + // wait for at least a minute after the last completed run + interval := time.Minute + err := hs.lockService.LockAndExecute(crawlerCtx, hs.crawlLockServiceActionName, interval, func(ctx context.Context) { + if err := hs.renderer.Run(crawlerCtx, authOpts, mode, theme, kind); err != nil { + hs.log.Error("On demand crawl error", "mode", mode, "theme", theme, "kind", kind, "userId", authOpts.UserID, "orgId", authOpts.OrgID, "orgRole", authOpts.OrgRole) + } + }) + + if err != nil { + hs.log.Error("On demand crawl lock error", "err", err) + } +} + +func (hs *thumbService) runScheduledCrawl(parentCtx context.Context) { + crawlerCtx, cancel := context.WithTimeout(parentCtx, hs.scheduleOptions.maxCrawlDuration) + defer cancel() + + err := hs.lockService.LockAndExecute(crawlerCtx, hs.crawlLockServiceActionName, hs.scheduleOptions.crawlInterval, func(ctx context.Context) { + for _, theme := range hs.scheduleOptions.themes { + if err := hs.renderer.Run(crawlerCtx, hs.scheduleOptions.auth, hs.scheduleOptions.crawlerMode, theme, hs.scheduleOptions.thumbnailKind); err != nil { + hs.log.Error("Scheduled crawl error", "theme", theme, "kind", hs.scheduleOptions.thumbnailKind, "err", err) + } + } + }) + + if err != nil { + hs.log.Error("Scheduled crawl lock error", "err", err) + } +} + +func (hs *thumbService) Run(ctx context.Context) error { + if !hs.features.IsEnabled(featuremgmt.FlagDashboardPreviewsScheduler) { + return nil + } + + gc := time.NewTicker(hs.scheduleOptions.tickerInterval) + + for { + select { + case <-gc.C: + go hs.runScheduledCrawl(ctx) + case <-ctx.Done(): + hs.log.Debug("Grafana is shutting down - stopping dashboard crawler") + gc.Stop() + + return nil + } + } +}