public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Unified: Run resource data migrations at startup (#114857)

Browse Source 
* chore: uncomment unified migration

* chore: adapt and fix tests

* chore: dynamically bump max conns if needed during migration

* chore: copilot suggestions

* chore: pass ctx in RegisterMigration

* chore: make playlists opt-out and dashboards opt-in

* chore: adjust dashboard test

* chore: disable enable log in test

* chore: address review comments

- do not use pointer config
- add migration registry

* chore: more consistent naming

* chore: fix playlist discovery test
This commit is contained in:
Rafael Bortolon Paulovic
2025-12-17 10:09:57 +01:00
committed by GitHub
parent 00ea4024a8
commit aa3b9dc4da
24 changed files with 705 additions and 274 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/infra/metrics/metrics.go
+1 -1
View File
@@ -224,7 +224,7 @@ var (
MStatTotalRepositories prometheus.Gauge
// MUnifiedStorageMigrationStatus indicates the migration status for unified storage in this instance.
// Possible values: 0 (default/undefined), 1 (migration disabled), 2 (migration would run).
// Possible values: 0 (default/undefined), 1 (migration disabled), 2 (migration would run), 3 (migration will run).
MUnifiedStorageMigrationStatus prometheus.Gauge
)
pkg/registry/apis/dashboard/legacy/sql_dashboards.go
-6
View File
@@ -154,12 +154,6 @@ func (a *dashboardSqlAccess) executeQuery(ctx context.Context, helper *legacysql
return nil
})
// Use transaction from unified storage if available in the context.
// This allows us to run migrations in a transaction which is specifically required for SQLite.
if tx == nil {
tx = resource.TransactionFromContext(ctx)
}
if tx != nil {
return tx.QueryContext(ctx, query, args...)
}
pkg/setting/setting.go
+3
View File
@@ -632,6 +632,9 @@ type UnifiedStorageConfig struct {
DataSyncerInterval time.Duration
// DataSyncerRecordsLimit defines how many records will be processed at max during a sync invocation.
DataSyncerRecordsLimit int
// EnableMigration indicates whether migration is enabled for the resource.
// If not set, will use the default from MigratedUnifiedResources.
EnableMigration bool
}
type InstallPlugin struct {
pkg/setting/setting_unified_storage.go
+32 -11
View File
@@ -8,10 +8,17 @@ import (
"github.com/grafana/grafana/pkg/util/osutil"
)
var MigratedUnifiedResources = []string{
"playlists.playlist.grafana.app",
"folders.folder.grafana.app",
"dashboards.dashboard.grafana.app",
const (
PlaylistResource = "playlists.playlist.grafana.app"
FolderResource = "folders.folder.grafana.app"
DashboardResource = "dashboards.dashboard.grafana.app"
)
// MigratedUnifiedResources maps resources to a boolean indicating if migration is enabled by default
var MigratedUnifiedResources = map[string]bool{
PlaylistResource: true, // enabled by default
FolderResource: false,
DashboardResource: false,
}
// read storage configs from ini file. They look like:
@@ -46,12 +53,19 @@ func (cfg *Cfg) setUnifiedStorageConfig() {
// parse dataSyncerInterval from resource section
dataSyncerInterval := section.Key("dataSyncerInterval").MustDuration(time.Hour)
// parse EnableMigration from resource section
enableMigration := MigratedUnifiedResources[resourceName]
if section.HasKey("enableMigration") {
enableMigration = section.Key("enableMigration").MustBool(MigratedUnifiedResources[resourceName])
}
storageConfig[resourceName] = UnifiedStorageConfig{
DualWriterMode: rest.DualWriterMode(dualWriterMode),
DualWriterPeriodicDataSyncJobEnabled: dualWriterPeriodicDataSyncJobEnabled,
DualWriterMigrationDataSyncDisabled: dualWriterMigrationDataSyncDisabled,
DataSyncerRecordsLimit: dataSyncerRecordsLimit,
DataSyncerInterval: dataSyncerInterval,
EnableMigration: enableMigration,
}
}
cfg.UnifiedStorage = storageConfig
@@ -61,8 +75,8 @@ func (cfg *Cfg) setUnifiedStorageConfig() {
cfg.DisableDataMigrations = section.Key("disable_data_migrations").MustBool(false)
if !cfg.DisableDataMigrations && cfg.getUnifiedStorageType() == "unified" {
// Helper log to find instances running migrations in the future
cfg.Logger.Info("Unified migration configs not yet enforced")
// cfg.enforceMigrationToUnifiedConfigs() // TODO: uncomment when ready for release
cfg.Logger.Info("Unified migration configs enforced")
cfg.enforceMigrationToUnifiedConfigs()
} else {
// Helper log to find instances disabling migration
cfg.Logger.Info("Unified migration configs enforcement disabled", "storage_type", cfg.getUnifiedStorageType(), "disable_data_migrations", cfg.DisableDataMigrations)
@@ -107,7 +121,6 @@ func (cfg *Cfg) setUnifiedStorageConfig() {
cfg.MinFileIndexBuildVersion = section.Key("min_file_index_build_version").MustString("")
}
// nolint:unused
// enforceMigrationToUnifiedConfigs enforces configurations required to run migrated resources in mode 5
// All migrated resources in MigratedUnifiedResources are set to mode 5 and unified search is enabled
func (cfg *Cfg) enforceMigrationToUnifiedConfigs() {
@@ -118,14 +131,22 @@ func (cfg *Cfg) enforceMigrationToUnifiedConfigs() {
section.Key("enable_search").SetValue("true")
cfg.EnableSearch = true
}
for _, resource := range MigratedUnifiedResources {
cfg.Logger.Info("Enforcing mode 5 for resource in unified storage", "resource", resource)
if oldCfg, ok := cfg.UnifiedStorage[resource]; ok {
cfg.Logger.Info("Overriding unified storage config for migrated resource", "resource", resource, "old_config", oldCfg)
for resource, enabledByDefault := range MigratedUnifiedResources {
resourceCfg, ok := cfg.UnifiedStorage[resource]
if ok {
if !resourceCfg.EnableMigration {
cfg.Logger.Info("Resource migration disabled", "resource", resource)
continue
}
cfg.Logger.Info("Overriding unified storage config for migrated resource", "resource", resource, "old_config", resourceCfg)
} else if !enabledByDefault {
continue
}
cfg.Logger.Info("Enforcing mode 5 for resource in unified storage", "resource", resource)
cfg.UnifiedStorage[resource] = UnifiedStorageConfig{
DualWriterMode: 5,
DualWriterMigrationDataSyncDisabled: true,
EnableMigration: true,
}
}
}
pkg/setting/setting_unified_storage_test.go
+50 -16
View File
@@ -4,6 +4,7 @@ import (
"testing"
"time"
"github.com/grafana/grafana/pkg/apiserver/rest"
"github.com/stretchr/testify/assert"
)
@@ -13,31 +14,56 @@ func TestCfg_setUnifiedStorageConfig(t *testing.T) {
err := cfg.Load(CommandLineArgs{HomePath: "../../", Config: "../../conf/defaults.ini"})
assert.NoError(t, err)
s, err := cfg.Raw.NewSection("unified_storage.playlists.playlist.grafana.app")
assert.NoError(t, err)
setSectionKey := func(sectionName, key, value string) {
section := cfg.Raw.Section(sectionName) // Gets existing or creates new
_, err := section.NewKey(key, value)
assert.NoError(t, err)
}
_, err = s.NewKey("dualWriterMode", "2")
assert.NoError(t, err)
setMigratedResourceKey := func(key, value string) {
for migratedResource := range MigratedUnifiedResources {
setSectionKey("unified_storage."+migratedResource, key, value)
}
}
_, err = s.NewKey("dualWriterPeriodicDataSyncJobEnabled", "true")
assert.NoError(t, err)
validateMigratedResources := func(optIn bool) {
for migratedResource, enabled := range MigratedUnifiedResources {
resourceCfg, exists := cfg.UnifiedStorage[migratedResource]
_, err = s.NewKey("dataSyncerRecordsLimit", "1001")
assert.NoError(t, err)
isEnabled := enabled
if optIn {
isEnabled = true
}
_, err = s.NewKey("dataSyncerInterval", "10m")
assert.NoError(t, err)
if !isEnabled {
if exists {
assert.Equal(t, rest.DualWriterMode(1), resourceCfg.DualWriterMode, migratedResource)
}
continue
}
assert.Equal(t, exists, true, migratedResource)
assert.Equal(t, UnifiedStorageConfig{
DualWriterMode: 5,
DualWriterMigrationDataSyncDisabled: true,
EnableMigration: isEnabled,
}, resourceCfg, migratedResource)
}
}
setMigratedResourceKey("dualWriterMode", "1") // migrated resources enabled by default will change to 5 in setUnifiedStorageConfig
setSectionKey("unified_storage.resource.not_migrated.grafana.app", "dualWriterMode", "2")
setSectionKey("unified_storage.resource.not_migrated.grafana.app", "dualWriterPeriodicDataSyncJobEnabled", "true")
setSectionKey("unified_storage.resource.not_migrated.grafana.app", "dataSyncerRecordsLimit", "1001")
setSectionKey("unified_storage.resource.not_migrated.grafana.app", "dataSyncerInterval", "10m")
// Add unified_storage section for index settings
unifiedStorageSection, err := cfg.Raw.NewSection("unified_storage")
assert.NoError(t, err)
_, err = unifiedStorageSection.NewKey("index_min_count", "5")
assert.NoError(t, err)
setSectionKey("unified_storage", "index_min_count", "5")
cfg.setUnifiedStorageConfig()
value, exists := cfg.UnifiedStorage["playlists.playlist.grafana.app"]
value, exists := cfg.UnifiedStorage["resource.not_migrated.grafana.app"]
assert.Equal(t, exists, true)
assert.Equal(t, value, UnifiedStorageConfig{
@@ -47,6 +73,14 @@ func TestCfg_setUnifiedStorageConfig(t *testing.T) {
DataSyncerInterval: time.Minute * 10,
})
validateMigratedResources(false)
setMigratedResourceKey("enableMigration", "true") // will be changed to 5 in setUnifiedStorageConfig
cfg.setUnifiedStorageConfig()
validateMigratedResources(true)
// Test that index settings are correctly parsed
assert.Equal(t, 5, cfg.IndexMinCount)
})
pkg/storage/unified/migrations/migrator_test.go
+120 -3
View File
@@ -162,14 +162,49 @@ func runMigrationTestSuite(t *testing.T, testCases []resourceMigratorTestCase) {
}
})
t.Run("Step 3: verify data is migrated to unified storage", func(t *testing.T) {
// Migrations will run automatically at startup and mode 5 is enforced by the config
t.Run("Step 3: verify that opted-out resources are not migrated", func(t *testing.T) {
// Build unified storage config for Mode5
unifiedConfig := make(map[string]setting.UnifiedStorageConfig)
for _, tc := range testCases {
for _, gvr := range tc.resources() {
resourceKey := fmt.Sprintf("%s.%s", gvr.Resource, gvr.Group)
unifiedConfig[resourceKey] = setting.UnifiedStorageConfig{
DualWriterMode: grafanarest.Mode5,
EnableMigration: false,
}
}
}
helper := apis.NewK8sTestHelperWithOpts(t, apis.K8sTestHelperOpts{
GrafanaOpts: testinfra.GrafanaOpts{
AppModeProduction: true,
DisableAnonymous: true,
DisableDBCleanup: true,
APIServerStorageType: "unified",
UnifiedStorageConfig: unifiedConfig,
},
Org1Users: org1,
OrgBUsers: orgB,
})
t.Cleanup(helper.Shutdown)
for _, state := range testStates {
t.Run(state.tc.name(), func(t *testing.T) {
// Verify resources don't exist in unified storage yet
state.tc.verify(t, helper, false)
})
}
verifyRegisteredMigrations(t, helper, false, true)
})
t.Run("Step 4: verify data is migrated to unified storage", func(t *testing.T) {
// Migrations enabled by default will run automatically at startup and mode 5 is enforced by the config
helper := apis.NewK8sTestHelperWithOpts(t, apis.K8sTestHelperOpts{
GrafanaOpts: testinfra.GrafanaOpts{
// EnableLog: true,
AppModeProduction: true,
DisableAnonymous: true,
DisableDataMigrations: false, // Run migrations at startup
DisableDBCleanup: true,
APIServerStorageType: "unified",
},
Org1Users: org1,
@@ -183,7 +218,89 @@ func runMigrationTestSuite(t *testing.T, testCases []resourceMigratorTestCase) {
state.tc.verify(t, helper, true)
})
}
t.Logf("Verifying migrations are correctly registered")
verifyRegisteredMigrations(t, helper, true, false)
})
t.Run("Step 5: verify data is migrated for all migrations", func(t *testing.T) {
// Trigger migrations that are not enabled by default
unifiedConfig := make(map[string]setting.UnifiedStorageConfig)
for _, tc := range testCases {
for _, gvr := range tc.resources() {
resourceKey := fmt.Sprintf("%s.%s", gvr.Resource, gvr.Group)
unifiedConfig[resourceKey] = setting.UnifiedStorageConfig{
EnableMigration: true,
}
}
}
helper := apis.NewK8sTestHelperWithOpts(t, apis.K8sTestHelperOpts{
GrafanaOpts: testinfra.GrafanaOpts{
// EnableLog: true,
AppModeProduction: true,
DisableAnonymous: true,
DisableDataMigrations: false,
APIServerStorageType: "unified",
UnifiedStorageConfig: unifiedConfig,
},
Org1Users: org1,
OrgBUsers: orgB,
})
t.Cleanup(helper.Shutdown)
for _, state := range testStates {
t.Run(state.tc.name(), func(t *testing.T) {
// Verify resources still exist in unified storage after restart
state.tc.verify(t, helper, true)
})
}
t.Logf("Verifying migrations are correctly registered")
verifyRegisteredMigrations(t, helper, false, false)
})
}
const (
migrationScope = "unifiedstorage"
migrationTable = migrationScope + "_migration_log"
playlistsID = "playlists migration"
foldersAndDashboardsID = "folders and dashboards migration"
)
var migrationIDsToDefault = map[string]bool{
playlistsID: true,
foldersAndDashboardsID: false,
}
func verifyRegisteredMigrations(t *testing.T, helper *apis.K8sTestHelper, onlyDefault bool, optOut bool) {
getMigrationsQuery := fmt.Sprintf("SELECT migration_id FROM %s", migrationTable)
createTableMigrationID := fmt.Sprintf("create %s table", migrationTable)
expectedMigrationIDs := []string{createTableMigrationID}
for id, enabled := range migrationIDsToDefault {
if onlyDefault && !enabled {
continue
}
if optOut {
continue
}
expectedMigrationIDs = append(expectedMigrationIDs, id)
}
rows, err := helper.GetEnv().SQLStore.GetEngine().DB().Query(getMigrationsQuery)
require.NoError(t, err)
defer func() {
require.NoError(t, rows.Close())
}()
migrationIDs := make(map[string]struct{})
for rows.Next() {
var migrationID string
require.NoError(t, rows.Scan(&migrationID))
require.Contains(t, expectedMigrationIDs, migrationID)
migrationIDs[migrationID] = struct{}{}
}
require.NoError(t, rows.Err())
require.Len(t, migrationIDs, len(expectedMigrationIDs))
}
// verifyResourceCount verifies that the expected number of resources exist in K8s storage
pkg/storage/unified/migrations/resources.go
+54 -5
View File
@@ -7,7 +7,9 @@ import (
folders "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1"
playlists "github.com/grafana/grafana/apps/playlist/pkg/apis/playlist/v0alpha1"
"github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy"
sqlstoremigrator "github.com/grafana/grafana/pkg/services/sqlstore/migrator"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/storage/unified/resource"
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
"k8s.io/apimachinery/pkg/runtime/schema"
)
@@ -17,7 +19,13 @@ type ResourceDefinition struct {
MigratorFunc string // Name of the method: "MigrateFolders", "MigrateDashboards", etc.
}
var registeredResources = []ResourceDefinition{
type migrationDefinition struct {
name string
resources []string
registerFunc func(mg *sqlstoremigrator.Migrator, migrator UnifiedMigrator, client resource.ResourceClient)
}
var resourceRegistry = []ResourceDefinition{
{
GroupResource: schema.GroupResource{Group: folders.GROUP, Resource: folders.RESOURCE},
MigratorFunc: "MigrateFolders",
@@ -36,9 +44,50 @@ var registeredResources = []ResourceDefinition{
},
}
var migrationRegistry = []migrationDefinition{
{
name: "playlists",
resources: []string{setting.PlaylistResource},
registerFunc: registerPlaylistMigration,
},
{
name: "folders and dashboards",
resources: []string{setting.FolderResource, setting.DashboardResource},
registerFunc: registerDashboardAndFolderMigration,
},
}
func registerMigrations(cfg *setting.Cfg, mg *sqlstoremigrator.Migrator, migrator UnifiedMigrator, client resource.ResourceClient) error {
for _, migration := range migrationRegistry {
var (
hasValue bool
allEnabled bool
)
for _, res := range migration.resources {
enabled := cfg.UnifiedStorage[res].EnableMigration
if !hasValue {
allEnabled = enabled
hasValue = true
continue
}
if enabled != allEnabled {
return fmt.Errorf("cannot migrate resources separately: %v migration must be either all enabled or all disabled", migration.resources)
}
}
if !allEnabled {
logger.Info("Migration is disabled in config, skipping", "migration", migration.name)
continue
}
migration.registerFunc(mg, migrator, client)
}
return nil
}
func getResourceDefinition(group, resource string) *ResourceDefinition {
for i := range registeredResources {
r := &registeredResources[i]
for i := range resourceRegistry {
r := &resourceRegistry[i]
if r.GroupResource.Group == group && r.GroupResource.Resource == resource {
return r
}
@@ -80,13 +129,13 @@ func getMigratorFunc(accessor legacy.MigrationDashboardAccessor, group, resource
func validateRegisteredResources() error {
registeredMap := make(map[string]bool)
for _, gr := range registeredResources {
for _, gr := range resourceRegistry {
key := fmt.Sprintf("%s.%s", gr.GroupResource.Resource, gr.GroupResource.Group)
registeredMap[key] = true
}
var missing []string
for _, expected := range setting.MigratedUnifiedResources {
for expected := range setting.MigratedUnifiedResources {
if !registeredMap[expected] {
missing = append(missing, expected)
}
pkg/storage/unified/migrations/resources_test.go
+92
View File
@@ -0,0 +1,92 @@
package migrations
import (
"testing"
sqlstoremigrator "github.com/grafana/grafana/pkg/services/sqlstore/migrator"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/storage/unified/resource"
"github.com/stretchr/testify/require"
)
// TestRegisterMigrations exercises registerMigrations with various EnableMigration configs using a table-driven test.
func TestRegisterMigrations(t *testing.T) {
origRegistry := migrationRegistry
t.Cleanup(func() { migrationRegistry = origRegistry })
// helper to build a fake registry with custom register funcs that bump counters
makeFakeRegistry := func(migrationCalls map[string]int) []migrationDefinition {
return []migrationDefinition{
{
name: "playlists",
resources: []string{setting.PlaylistResource},
registerFunc: func(mg *sqlstoremigrator.Migrator, migrator UnifiedMigrator, client resource.ResourceClient) {
migrationCalls["playlists"]++
},
},
{
name: "folders and dashboards",
resources: []string{setting.FolderResource, setting.DashboardResource},
registerFunc: func(mg *sqlstoremigrator.Migrator, migrator UnifiedMigrator, client resource.ResourceClient) {
migrationCalls["folders and dashboards"]++
},
},
}
}
// Build a minimal cfg with UnifiedStorage entries used by registerMigrations
makeCfg := func(vals map[string]bool) *setting.Cfg {
cfg := &setting.Cfg{UnifiedStorage: make(map[string]setting.UnifiedStorageConfig)}
for k, v := range vals {
cfg.UnifiedStorage[k] = setting.UnifiedStorageConfig{EnableMigration: v}
}
return cfg
}
// Table of scenarios
tests := []struct {
name string
enablePlaylist bool
enableFolder bool
enableDashboard bool
wantPlaylistCalls int
wantFDCalls int
wantErr bool
}{
{name: "playlists enabled", enablePlaylist: true, wantPlaylistCalls: 1},
{name: "playlists disabled", enablePlaylist: false, wantPlaylistCalls: 0},
{name: "folders+dashboards both enabled", enableFolder: true, enableDashboard: true, wantFDCalls: 1},
{name: "folders enabled, dashboards disabled (mismatch)", enableFolder: true, enableDashboard: false, wantFDCalls: 0, wantErr: true},
{name: "folders disabled, dashboards enabled (mismatch)", enableFolder: false, enableDashboard: true, wantFDCalls: 0, wantErr: true},
{name: "folders+dashboards both disabled", enableFolder: false, enableDashboard: false, wantFDCalls: 0},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
migrationCalls := map[string]int{
"playlists": 0,
"folders and dashboards": 0,
}
migrationRegistry = makeFakeRegistry(migrationCalls)
cfg := makeCfg(map[string]bool{
setting.PlaylistResource: tt.enablePlaylist,
setting.FolderResource: tt.enableFolder,
setting.DashboardResource: tt.enableDashboard,
})
// We pass nils for migrator dependencies because our fake registerFuncs don't use them
err := registerMigrations(cfg, nil, nil, nil)
if tt.wantErr {
require.Error(t, err, "expected error for mismatched enablement")
} else {
require.NoError(t, err, "unexpected error")
}
require.Equal(t, tt.wantPlaylistCalls, migrationCalls["playlists"], "playlists register call count")
require.Equal(t, tt.wantFDCalls, migrationCalls["folders and dashboards"], "folders+dashboards register call count")
})
}
}
pkg/storage/unified/migrations/service.go
+15 -18
View File
@@ -3,7 +3,6 @@ package migrations
import (
"context"
"fmt"
"os"
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/infra/kvstore"
@@ -49,34 +48,25 @@ func ProvideUnifiedStorageMigrationService(
}
func (p *UnifiedStorageMigrationServiceImpl) Run(ctx context.Context) error {
// TODO: temporary skip migrations in test environments to prevent integration test timeouts.
if os.Getenv("GRAFANA_TEST_DB") != "" {
return nil
}
// skip migrations if disabled in config
if p.cfg.DisableDataMigrations {
metrics.MUnifiedStorageMigrationStatus.Set(1)
logger.Info("Data migrations are disabled, skipping")
return nil
} else {
metrics.MUnifiedStorageMigrationStatus.Set(2)
logger.Info("Data migrations not yet enforced, skipping")
}
// TODO: Re-enable once migrations are ready
// TODO: add guarantee that this only runs once
// return RegisterMigrations(p.migrator, p.cfg, p.sqlStore, p.client)
return nil
logger.Info("Running migrations for unified storage")
metrics.MUnifiedStorageMigrationStatus.Set(3)
return RegisterMigrations(ctx, p.migrator, p.cfg, p.sqlStore, p.client)
}
func RegisterMigrations(
ctx context.Context,
migrator UnifiedMigrator,
cfg *setting.Cfg,
sqlStore db.DB,
client resource.ResourceClient,
) error {
ctx, span := tracer.Start(context.Background(), "storage.unified.RegisterMigrations")
ctx, span := tracer.Start(ctx, "storage.unified.RegisterMigrations")
defer span.End()
mg := sqlstoremigrator.NewScopedMigrator(sqlStore.GetEngine(), cfg, "unifiedstorage")
mg.AddCreateMigration()
@@ -89,12 +79,19 @@ func RegisterMigrations(
return err
}
// Register resource migrations
registerDashboardAndFolderMigration(mg, migrator, client)
registerPlaylistMigration(mg, migrator, client)
if err := registerMigrations(cfg, mg, migrator, client); err != nil {
return err
}
// Run all registered migrations (blocking)
sec := cfg.Raw.Section("database")
db := mg.DBEngine.DB().DB
maxOpenConns := db.Stats().MaxOpenConnections
if maxOpenConns <= 2 {
// migrations require at least 3 connections due to extra GRPC connections
db.SetMaxOpenConns(3)
defer db.SetMaxOpenConns(maxOpenConns)
}
if err := mg.RunMigrations(ctx,
sec.Key("migration_locking").MustBool(true),
sec.Key("locking_attempt_timeout_sec").MustInt()); err != nil {
pkg/tests/api/alerting/api_convert_prometheus_test.go
+6 -5
View File
@@ -1166,11 +1166,12 @@ func TestIntegrationConvertPrometheusEndpoints_Editor(t *testing.T) {
testinfra.SQLiteIntegrationTest(t)
dir, gpath := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
DisableLegacyAlerting: true,
EnableUnifiedAlerting: true,
DisableAnonymous: true,
AppModeProduction: true,
EnableRecordingRules: true,
DisableAuthZClientCache: true,
DisableLegacyAlerting: true,
EnableUnifiedAlerting: true,
DisableAnonymous: true,
AppModeProduction: true,
EnableRecordingRules: true,
})
grafanaListedAddr, env := testinfra.StartGrafanaEnv(t, dir, gpath)
pkg/tests/api/alerting/api_provisioning_test.go
+5 -4
View File
@@ -1192,10 +1192,11 @@ func TestIntegrationExportFileProvisionContactPoints(t *testing.T) {
func TestIntegrationFullpath(t *testing.T) {
dir, p := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
DisableLegacyAlerting: true,
EnableUnifiedAlerting: true,
DisableAnonymous: true,
AppModeProduction: true,
DisableAuthZClientCache: true,
DisableLegacyAlerting: true,
EnableUnifiedAlerting: true,
DisableAnonymous: true,
AppModeProduction: true,
})
grafanaListedAddr, env := testinfra.StartGrafanaEnv(t, dir, p)
pkg/tests/api/alerting/api_ruler_test.go
+15 -12
View File
@@ -53,10 +53,11 @@ func TestIntegrationAlertRulePermissions(t *testing.T) {
testinfra.SQLiteIntegrationTest(t)
dir, p := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
DisableLegacyAlerting: true,
EnableUnifiedAlerting: true,
DisableAnonymous: true,
AppModeProduction: true,
DisableAuthZClientCache: true,
DisableLegacyAlerting: true,
EnableUnifiedAlerting: true,
DisableAnonymous: true,
AppModeProduction: true,
})
grafanaListedAddr, env := testinfra.StartGrafanaEnv(t, dir, p)
@@ -360,10 +361,11 @@ func TestIntegrationAlertRuleNestedPermissions(t *testing.T) {
testinfra.SQLiteIntegrationTest(t)
dir, p := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
DisableLegacyAlerting: true,
EnableUnifiedAlerting: true,
DisableAnonymous: true,
AppModeProduction: true,
DisableAuthZClientCache: true,
DisableLegacyAlerting: true,
EnableUnifiedAlerting: true,
DisableAnonymous: true,
AppModeProduction: true,
})
grafanaListedAddr, env := testinfra.StartGrafanaEnv(t, dir, p)
@@ -1429,10 +1431,11 @@ func TestIntegrationRuleGroupSequence(t *testing.T) {
testinfra.SQLiteIntegrationTest(t)
dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
DisableLegacyAlerting: true,
EnableUnifiedAlerting: true,
DisableAnonymous: true,
AppModeProduction: true,
DisableAuthZClientCache: true,
DisableLegacyAlerting: true,
EnableUnifiedAlerting: true,
DisableAnonymous: true,
AppModeProduction: true,
})
grafanaListedAddr, env := testinfra.StartGrafanaEnv(t, dir, path)
pkg/tests/api/annotations/annotations_test.go
+3 -2
View File
@@ -32,8 +32,9 @@ func TestIntegrationAnnotations(t *testing.T) {
testutil.SkipIntegrationTestInShortMode(t)
dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
DisableAnonymous: true,
EnableFeatureToggles: []string{featuremgmt.FlagAnnotationPermissionUpdate},
DisableAuthZClientCache: true,
DisableAnonymous: true,
EnableFeatureToggles: []string{featuremgmt.FlagAnnotationPermissionUpdate},
})
grafanaListedAddr, env := testinfra.StartGrafanaEnv(t, dir, path)
noneUserID := tests.CreateUser(t, env.SQLStore, env.Cfg, user.CreateUserCommand{
pkg/tests/api/dashboards/api_dashboards_test.go
+64 -31
View File
@@ -12,6 +12,7 @@ import (
"testing"
"time"
"github.com/grafana/grafana/pkg/setting"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -38,8 +39,15 @@ func TestMain(m *testing.M) {
func TestIntegrationDashboardServiceValidation(t *testing.T) {
testutil.SkipIntegrationTestInShortMode(t)
unifiedConfig := make(map[string]setting.UnifiedStorageConfig)
for _, resource := range []string{"folders.folder.grafana.app", "dashboards.dashboard.grafana.app"} {
unifiedConfig[resource] = setting.UnifiedStorageConfig{
EnableMigration: true,
}
}
dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
DisableAnonymous: true,
DisableAnonymous: true,
UnifiedStorageConfig: unifiedConfig,
})
grafanaListedAddr, env := testinfra.StartGrafanaEnv(t, dir, path)
@@ -218,11 +226,12 @@ func TestIntegrationDashboardServiceValidation(t *testing.T) {
require.NoError(t, err)
})
t.Run("When updating uid with id", func(t *testing.T) {
dashboardWithDuplicatedLegacyAnnotation := "new-uid"
t.Run("When saving a dashboard with an already used legacy ID", func(t *testing.T) {
resp, err := postDashboard(t, grafanaListedAddr, "admin", "admin", map[string]interface{}{
"dashboard": map[string]interface{}{
"id": savedDashInFolder.ID, // nolint:staticcheck
"uid": "new-uid",
"uid": dashboardWithDuplicatedLegacyAnnotation,
"title": "Updated title",
},
"folderUid": savedDashInFolder.FolderUID,
@@ -234,7 +243,48 @@ func TestIntegrationDashboardServiceValidation(t *testing.T) {
require.NoError(t, err)
})
t.Run("When updating uid with a dashboard already using that uid", func(t *testing.T) {
t.Run("When updating a dashboard with legacy ID in multiple dashboards", func(t *testing.T) {
resp, err := postDashboard(t, grafanaListedAddr, "admin", "admin", map[string]interface{}{
"dashboard": map[string]interface{}{
"id": savedDashInFolder.ID, // nolint:staticcheck
"uid": savedDashInGeneralFolder.UID,
"title": "Updated title",
},
"folderUid": savedDashInFolder.FolderUID,
"overwrite": true,
})
require.NoError(t, err)
assert.Equal(t, http.StatusInternalServerError, resp.StatusCode)
err = resp.Body.Close()
require.NoError(t, err)
// Delete the dashboard with duplicated legacy ID annotation
u := fmt.Sprintf("http://admin:admin@%s/api/dashboards/uid/%s", grafanaListedAddr, dashboardWithDuplicatedLegacyAnnotation)
req, err := http.NewRequest("DELETE", u, nil)
require.NoError(t, err)
resp, err = http.DefaultClient.Do(req)
require.NoError(t, err)
err = resp.Body.Close()
require.NoError(t, err)
assert.Equal(t, http.StatusOK, resp.StatusCode)
})
t.Run("When updating a dashboard already using that uid", func(t *testing.T) {
resp, err := postDashboard(t, grafanaListedAddr, "admin", "admin", map[string]interface{}{
"dashboard": map[string]interface{}{
"id": savedDashInFolder.ID,
"uid": savedDashInFolder.UID,
"title": "Dashboard with existing UID",
},
"folderUid": savedDashInFolder.FolderUID,
"overwrite": true,
})
require.NoError(t, err)
assert.Equal(t, http.StatusOK, resp.StatusCode)
err = resp.Body.Close()
require.NoError(t, err)
})
t.Run("When updating id with a dashboard already using that uid", func(t *testing.T) {
resp, err := postDashboard(t, grafanaListedAddr, "admin", "admin", map[string]interface{}{
"dashboard": map[string]interface{}{
"id": savedDashInFolder.ID, // nolint:staticcheck
@@ -268,8 +318,9 @@ func TestIntegrationDashboardServiceValidation(t *testing.T) {
require.NoError(t, err)
})
// Obs: in legacy, the dashboard request would fail
// After the dashboard is created, the user can see that there is an error with the library panel and can remove them manually
t.Run("When creating a dashboard that references a non-existent library panel", func(t *testing.T) {
originalCount := getDashboardCount(t, grafanaListedAddr, "admin", "admin")
resp, err := postDashboard(t, grafanaListedAddr, "admin", "admin", map[string]interface{}{
"dashboard": map[string]interface{}{
"title": "Bad dashboard",
@@ -285,15 +336,11 @@ func TestIntegrationDashboardServiceValidation(t *testing.T) {
},
})
require.NoError(t, err)
assert.Equal(t, http.StatusInternalServerError, resp.StatusCode)
body, err := io.ReadAll(resp.Body)
assert.Equal(t, http.StatusOK, resp.StatusCode)
_, err = io.ReadAll(resp.Body)
require.NoError(t, err)
require.Contains(t, string(body), "library element could not be found")
err = resp.Body.Close()
require.NoError(t, err)
// A new dashboard is not created in this situation.
require.Equal(t, originalCount, getDashboardCount(t, grafanaListedAddr, "admin", "admin"))
})
}
@@ -332,7 +379,7 @@ func TestIntegrationDashboardQuota(t *testing.T) {
dashboardDTO := &plugindashboards.PluginDashboard{}
err = json.Unmarshal(b, dashboardDTO)
require.NoError(t, err)
require.EqualValues(t, 1, dashboardDTO.DashboardId)
require.EqualValues(t, "just testing", dashboardDTO.Title)
})
t.Run("when quota limit exceeds importing a dashboard should fail", func(t *testing.T) {
@@ -421,7 +468,7 @@ providers:
dashboardUID = d.UID
dashboardID = d.ID // nolint:staticcheck
}
assert.Equal(t, int64(1), dashboardID)
assert.Len(t, *dashboardList, 1)
testCases := []struct {
desc string
@@ -781,7 +828,7 @@ func TestIntegrationImportDashboardWithLibraryPanels(t *testing.T) {
},
{
"id": 2,
"title": "Library Panel 2",
"title": "Library Panel 2",
"type": "stat",
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 0},
"libraryPanel": {
@@ -805,7 +852,7 @@ func TestIntegrationImportDashboardWithLibraryPanels(t *testing.T) {
}
},
"test-lib-panel-2": {
"uid": "test-lib-panel-2",
"uid": "test-lib-panel-2",
"name": "Test Library Panel 2",
"kind": 1,
"type": "stat",
@@ -1011,26 +1058,12 @@ func postDashboard(t *testing.T, grafanaListedAddr, user, password string, paylo
return http.Post(u, "application/json", bytes.NewBuffer(payloadBytes)) // nolint:gosec
}
func getDashboardCount(t *testing.T, grafanaListenAddr, user, password string) int {
endpoint := fmt.Sprintf("http://%s:%s@%s/apis/dashboard.grafana.app/v0alpha1/namespaces/default/search", user, password, grafanaListenAddr)
resp, err := http.Get(endpoint) //nolint:gosec
require.NoError(t, err)
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
var payload map[string]any
require.NoError(t, json.Unmarshal(body, &payload))
return int(payload["totalHits"].(float64))
}
func TestIntegrationDashboardServicePermissions(t *testing.T) {
testutil.SkipIntegrationTestInShortMode(t)
dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
DisableAnonymous: true,
DisableAnonymous: true,
DisableAuthZClientCache: true,
})
grafanaListedAddr, env := testinfra.StartGrafanaEnv(t, dir, path)
tests.CreateUser(t, env.SQLStore, env.Cfg, user.CreateUserCommand{
pkg/tests/apis/collections/stars_test.go
+4 -3
View File
@@ -42,9 +42,10 @@ func TestIntegrationStars(t *testing.T) {
}
helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: false, // required for experimental APIs
DisableAnonymous: true,
EnableFeatureToggles: flags,
DisableDataMigrations: true,
AppModeProduction: false, // required for experimental APIs
DisableAnonymous: true,
EnableFeatureToggles: flags,
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
"dashboards.dashboard.grafana.app": {
DualWriterMode: mode,
pkg/tests/apis/dashboard/dashboards_test.go
+9 -5
View File
@@ -37,7 +37,8 @@ func TestMain(m *testing.M) {
func runDashboardTest(t *testing.T, mode rest.DualWriterMode, gvr schema.GroupVersionResource) {
t.Run("simple crud+list", func(t *testing.T) {
helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
DisableAnonymous: true,
DisableAnonymous: true,
DisableDataMigrations: true,
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
"dashboards.dashboard.grafana.app": {
DualWriterMode: mode,
@@ -194,7 +195,9 @@ func TestIntegrationLegacySupport(t *testing.T) {
testutil.SkipIntegrationTestInShortMode(t)
ctx := context.Background()
helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{})
helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
DisableDataMigrations: true,
})
clientV0 := helper.GetResourceClient(apis.ResourceClientArgs{
User: helper.Org1.Admin,
@@ -500,9 +503,10 @@ func runDashboardSearchTest(t *testing.T, mode rest.DualWriterMode) {
ctx := context.Background()
helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
AppModeProduction: true,
DisableDataMigrations: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
"dashboards.dashboard.grafana.app": {DualWriterMode: mode},
"folders.folder.grafana.app": {DualWriterMode: mode},
pkg/tests/apis/dashboard/integration/api_validation_test.go
+8 -4
View File
@@ -68,7 +68,8 @@ func TestIntegrationDashboardAPIValidation(t *testing.T) {
t.Run(fmt.Sprintf("DualWriterMode %d", dualWriterMode), func(t *testing.T) {
// Create a K8sTestHelper which will set up a real API server
helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
DisableAnonymous: true,
DisableDataMigrations: true,
DisableAnonymous: true,
EnableFeatureToggles: []string{
featuremgmt.FlagKubernetesDashboards, // Enable FE-only dashboard feature flag
},
@@ -105,7 +106,8 @@ func TestIntegrationDashboardAPIValidation(t *testing.T) {
t.Run(fmt.Sprintf("DualWriterMode %d - kubernetesDashboards disabled", dualWriterMode), func(t *testing.T) {
// Create a K8sTestHelper which will set up a real API server
helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
DisableAnonymous: true,
DisableDataMigrations: true,
DisableAnonymous: true,
DisableFeatureToggles: []string{
featuremgmt.FlagKubernetesDashboards,
},
@@ -138,7 +140,8 @@ func TestIntegrationDashboardAPIAuthorization(t *testing.T) {
for _, dualWriterMode := range dualWriterModes {
t.Run(fmt.Sprintf("DualWriterMode %d", dualWriterMode), func(t *testing.T) {
helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
DisableAnonymous: true,
DisableDataMigrations: true,
DisableAnonymous: true,
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
"dashboards.dashboard.grafana.app": {
DualWriterMode: dualWriterMode,
@@ -185,7 +188,8 @@ func TestIntegrationDashboardAPI(t *testing.T) {
t.Run(fmt.Sprintf("DualWriterMode %d", dualWriterMode), func(t *testing.T) {
// Create a K8sTestHelper which will set up a real API server
helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
DisableAnonymous: true,
DisableDataMigrations: true,
DisableAnonymous: true,
EnableFeatureToggles: []string{
featuremgmt.FlagKubernetesDashboards,
},
pkg/tests/apis/dashboard/integration/library_panels_api_validation_test.go
+6 -3
View File
@@ -29,7 +29,8 @@ func TestIntegrationLibraryPanelConnections(t *testing.T) {
for _, dualWriterMode := range dualWriterModes {
t.Run(fmt.Sprintf("DualWriterMode %d", dualWriterMode), func(t *testing.T) {
helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
DisableAnonymous: true,
DisableDataMigrations: true,
DisableAnonymous: true,
EnableFeatureToggles: []string{
"kubernetesLibraryPanels",
},
@@ -93,7 +94,8 @@ func TestIntegrationLibraryElementPermissions(t *testing.T) {
for _, dualWriterMode := range dualWriterModes {
t.Run(fmt.Sprintf("DualWriterMode %d", dualWriterMode), func(t *testing.T) {
helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
DisableAnonymous: true,
DisableDataMigrations: true,
DisableAnonymous: true,
EnableFeatureToggles: []string{
"kubernetesLibraryPanels",
"grafanaAPIServerWithExperimentalAPIs", // needed until we move it to v0beta1 at least (currently v0alpha1)
@@ -296,7 +298,8 @@ func TestIntegrationLibraryPanelConnectionsWithFolderAccess(t *testing.T) {
for _, dualWriterMode := range dualWriterModes {
t.Run(fmt.Sprintf("DualWriterMode %d", dualWriterMode), func(t *testing.T) {
helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
DisableAnonymous: true,
DisableDataMigrations: true,
DisableAnonymous: true,
EnableFeatureToggles: []string{
"kubernetesLibraryPanels",
},
pkg/tests/apis/dashboard/search_test.go
+4 -3
View File
@@ -37,9 +37,10 @@ func runSearchPermissionTest(t *testing.T, mode rest.DualWriterMode) {
ctx := context.Background()
helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
DisableDataMigrations: true,
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
"dashboards.dashboard.grafana.app": {DualWriterMode: mode},
"folders.folder.grafana.app": {DualWriterMode: mode},
pkg/tests/apis/folder/folder_tree_test.go
+4 -3
View File
@@ -48,9 +48,10 @@ func TestIntegrationFolderTree(t *testing.T) {
for _, mode := range modes {
t.Run(fmt.Sprintf("mode %d", mode), func(t *testing.T) {
helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
DisableDataMigrations: true,
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
"dashboards.dashboard.grafana.app": {
DualWriterMode: mode,
pkg/tests/apis/folder/folders_test.go
+70 -51
View File
@@ -139,9 +139,10 @@ func TestIntegrationFoldersApp(t *testing.T) {
t.Run(fmt.Sprintf("with dual write (unified storage, mode %v)", modeDw), func(t *testing.T) {
doFolderTests(t, apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
DisableDataMigrations: true,
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
folders.RESOURCEGROUP: {
DualWriterMode: modeDw,
@@ -153,9 +154,10 @@ func TestIntegrationFoldersApp(t *testing.T) {
t.Run(fmt.Sprintf("with dual write (unified storage, mode %v, create nested folders)", modeDw), func(t *testing.T) {
doNestedCreateTest(t, apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
DisableDataMigrations: true,
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
folders.RESOURCEGROUP: {
DualWriterMode: modeDw,
@@ -166,9 +168,10 @@ func TestIntegrationFoldersApp(t *testing.T) {
t.Run(fmt.Sprintf("with dual write (unified storage, mode %v, create existing folder)", modeDw), func(t *testing.T) {
doCreateDuplicateFolderTest(t, apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
DisableDataMigrations: true,
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
folders.RESOURCEGROUP: {
DualWriterMode: modeDw,
@@ -179,9 +182,10 @@ func TestIntegrationFoldersApp(t *testing.T) {
t.Run(fmt.Sprintf("when creating a folder, mode %v, it should trim leading and trailing spaces", modeDw), func(t *testing.T) {
doCreateEnsureTitleIsTrimmedTest(t, apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
DisableDataMigrations: true,
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
folders.RESOURCEGROUP: {
DualWriterMode: modeDw,
@@ -192,9 +196,10 @@ func TestIntegrationFoldersApp(t *testing.T) {
t.Run(fmt.Sprintf("with dual write (unified storage, mode %v, create circular reference folder)", modeDw), func(t *testing.T) {
doCreateCircularReferenceFolderTest(t, apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
DisableDataMigrations: true,
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
folders.RESOURCEGROUP: {
DualWriterMode: modeDw,
@@ -218,9 +223,10 @@ func TestIntegrationFoldersApp(t *testing.T) {
for _, mode := range modes {
t.Run(fmt.Sprintf("mode %d", mode), func(t *testing.T) {
doListFoldersTest(t, apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
DisableDataMigrations: true,
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
folders.RESOURCEGROUP: {
DualWriterMode: mode,
@@ -735,9 +741,10 @@ func TestIntegrationFolderCreatePermissions(t *testing.T) {
t.Run(fmt.Sprintf("Mode_%d", mode), func(t *testing.T) {
modeDw := grafanarest.DualWriterMode(mode)
helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
DisableDataMigrations: true,
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
folders.RESOURCEGROUP: {
DualWriterMode: modeDw,
@@ -862,9 +869,10 @@ func TestIntegrationFolderGetPermissions(t *testing.T) {
t.Run(fmt.Sprintf("Mode_%d", mode), func(t *testing.T) {
modeDw := grafanarest.DualWriterMode(mode)
helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
DisableDataMigrations: true,
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
folders.RESOURCEGROUP: {
DualWriterMode: modeDw,
@@ -1079,9 +1087,10 @@ func TestIntegrationFoldersCreateAPIEndpointK8S(t *testing.T) {
for mode := 0; mode <= 4; mode++ {
modeDw := grafanarest.DualWriterMode(mode)
helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
DisableDataMigrations: true,
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
folders.RESOURCEGROUP: {
DualWriterMode: modeDw,
@@ -1254,9 +1263,10 @@ func TestIntegrationFoldersGetAPIEndpointK8S(t *testing.T) {
modeDw := grafanarest.DualWriterMode(mode)
helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
DisableDataMigrations: true,
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
folders.RESOURCEGROUP: {
DualWriterMode: modeDw,
@@ -1366,9 +1376,10 @@ func TestIntegrationFolderDeletionBlockedByLibraryElements(t *testing.T) {
modeDw := grafanarest.DualWriterMode(mode)
helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
DisableDataMigrations: true,
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
folders.RESOURCEGROUP: {
DualWriterMode: modeDw,
@@ -1446,9 +1457,10 @@ func TestIntegrationRootFolderDeletionBlockedByLibraryElementsInSubfolder(t *tes
modeDw := grafanarest.DualWriterMode(mode)
helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
DisableDataMigrations: true,
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
folders.RESOURCEGROUP: {
DualWriterMode: modeDw,
@@ -1542,9 +1554,10 @@ func TestIntegrationFolderDeletionBlockedByConnectedLibraryPanels(t *testing.T)
modeDw := grafanarest.DualWriterMode(mode)
helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
DisableDataMigrations: true,
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
folders.RESOURCEGROUP: {
DualWriterMode: modeDw,
@@ -1620,9 +1633,10 @@ func TestIntegrationFolderDeletionWithDanglingLibraryPanels(t *testing.T) {
modeDw := grafanarest.DualWriterMode(mode)
helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
DisableDataMigrations: true,
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
folders.RESOURCEGROUP: {
DualWriterMode: modeDw,
@@ -1901,9 +1915,10 @@ func TestIntegrationDeleteNestedFoldersPostorder(t *testing.T) {
t.Run(fmt.Sprintf("Mode %d: Delete nested folder hierarchy in postorder", mode), func(t *testing.T) {
modeDw := grafanarest.DualWriterMode(mode)
helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
DisableDataMigrations: true,
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "unified",
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
folders.RESOURCEGROUP: {
DualWriterMode: modeDw,
@@ -2027,9 +2042,10 @@ func TestIntegrationDeleteFolderWithProvisionedDashboards(t *testing.T) {
t.Run(fmt.Sprintf("Mode %d: Delete provisioned folders and dashboards", mode), func(t *testing.T) {
modeDw := grafanarest.DualWriterMode(mode)
ops := testinfra.GrafanaOpts{
DisableAnonymous: true,
AppModeProduction: true,
APIServerStorageType: "unified",
DisableDataMigrations: true,
DisableAnonymous: true,
AppModeProduction: true,
APIServerStorageType: "unified",
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
folders.RESOURCEGROUP: {
DualWriterMode: modeDw,
@@ -2131,11 +2147,14 @@ func TestIntegrationDeleteFolderWithProvisionedDashboards(t *testing.T) {
// Test that folders created during provisioning using the dual writer have the
// appropriate labels and annotations in unified storage.
func TestIntegrationProvisionedFolderPropagatesLabelsAndAnnotations(t *testing.T) {
testutil.SkipIntegrationTestInShortMode(t)
mode3 := grafanarest.DualWriterMode(3)
ops := testinfra.GrafanaOpts{
DisableAnonymous: true,
AppModeProduction: true,
APIServerStorageType: "unified",
DisableDataMigrations: true,
DisableAnonymous: true,
AppModeProduction: true,
APIServerStorageType: "unified",
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
folders.RESOURCEGROUP: {
DualWriterMode: mode3,
pkg/tests/apis/openapi_snapshots/playlist.grafana.app-v0alpha1.json
Generated
+27 -1
View File
@@ -40,7 +40,7 @@
"tags": [
"Playlist"
],
"description": "list objects of kind Playlist",
"description": "list or watch objects of kind Playlist",
"operationId": "listPlaylist",
"parameters": [
{
@@ -1850,6 +1850,32 @@
"description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.",
"type": "string",
"format": "date-time"
},
"io.k8s.apimachinery.pkg.apis.meta.v1.WatchEvent": {
"description": "Event represents a single event to a watched resource.",
"type": "object",
"required": [
"type",
"object"
],
"properties": {
"object": {
"description": "Object is:\n * If Type is Added or Modified: the new state of the object.\n * If Type is Deleted: the state of the object immediately before deletion.\n * If Type is Error: *Status is recommended; other types may make sense\n depending on context.",
"allOf": [
{
"$ref": "#/components/schemas/io.k8s.apimachinery.pkg.runtime.RawExtension"
}
]
},
"type": {
"type": "string",
"default": ""
}
}
},
"io.k8s.apimachinery.pkg.runtime.RawExtension": {
"description": "RawExtension is used to hold extensions in external versions.\n\nTo use this, make a field which has RawExtension as its type in your external, versioned struct, and Object in your internal struct. You also need to register your various plugin types.\n\n// Internal package:\n\n\ttype MyAPIObject struct {\n\t\truntime.TypeMeta `json:\",inline\"`\n\t\tMyPlugin runtime.Object `json:\"myPlugin\"`\n\t}\n\n\ttype PluginA struct {\n\t\tAOption string `json:\"aOption\"`\n\t}\n\n// External package:\n\n\ttype MyAPIObject struct {\n\t\truntime.TypeMeta `json:\",inline\"`\n\t\tMyPlugin runtime.RawExtension `json:\"myPlugin\"`\n\t}\n\n\ttype PluginA struct {\n\t\tAOption string `json:\"aOption\"`\n\t}\n\n// On the wire, the JSON will look something like this:\n\n\t{\n\t\t\"kind\":\"MyAPIObject\",\n\t\t\"apiVersion\":\"v1\",\n\t\t\"myPlugin\": {\n\t\t\t\"kind\":\"PluginA\",\n\t\t\t\"aOption\":\"foo\",\n\t\t},\n\t}\n\nSo what happens? Decode first uses json or yaml to unmarshal the serialized data into your external MyAPIObject. That causes the raw JSON to be stored, but not unpacked. The next step is to copy (using pkg/conversion) into the internal struct. The runtime package's DefaultScheme has conversion functions installed which will unpack the JSON stored in RawExtension, turning it into the correct object type, and storing it in the Object. (TODO: In the case where the object is of an unknown type, a runtime.Unknown object will be created and stored.)",
"type": "object"
}
}
}
pkg/tests/apis/playlist/playlist_test.go
+103 -87
View File
@@ -54,47 +54,48 @@ func TestIntegrationPlaylist(t *testing.T) {
require.NoError(t, err)
// t.Logf("%s", disco)
require.JSONEq(t, `[
{
"version": "v0alpha1",
"freshness": "Current",
"resources": [
{
"resource": "playlists",
"responseKind": {
"group": "",
"kind": "Playlist",
"version": ""
},
"scope": "Namespaced",
"singularResource": "playlist",
"subresources": [
{
"responseKind": {
"group": "",
"kind": "Playlist",
"version": ""
},
"subresource": "status",
"verbs": [
"get",
"patch",
"update"
]
}
],
"verbs": [
"create",
"delete",
"deletecollection",
"get",
"list",
"patch",
"update"
]
}
]
}
]`, disco)
{
"freshness": "Current",
"resources": [
{
"resource": "playlists",
"responseKind": {
"group": "",
"kind": "Playlist",
"version": ""
},
"scope": "Namespaced",
"singularResource": "playlist",
"subresources": [
{
"responseKind": {
"group": "",
"kind": "Playlist",
"version": ""
},
"subresource": "status",
"verbs": [
"get",
"patch",
"update"
]
}
],
"verbs": [
"create",
"delete",
"deletecollection",
"get",
"list",
"patch",
"update",
"watch"
]
}
],
"version": "v0alpha1"
}
]`, disco)
})
t.Run("with k8s api flag", func(t *testing.T) {
@@ -106,9 +107,10 @@ func TestIntegrationPlaylist(t *testing.T) {
t.Run("with dual write (file, mode 0)", func(t *testing.T) {
doPlaylistTests(t, apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "file", // write the files to disk
DisableDataMigrations: true,
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "file", // write the files to disk
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
RESOURCEGROUP: {
DualWriterMode: grafanarest.Mode0,
@@ -119,9 +121,10 @@ func TestIntegrationPlaylist(t *testing.T) {
t.Run("with dual write (file, mode 1)", func(t *testing.T) {
doPlaylistTests(t, apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "file", // write the files to disk
DisableDataMigrations: true,
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "file", // write the files to disk
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
RESOURCEGROUP: {
DualWriterMode: grafanarest.Mode1,
@@ -132,9 +135,10 @@ func TestIntegrationPlaylist(t *testing.T) {
t.Run("with dual write (file, mode 2)", func(t *testing.T) {
doPlaylistTests(t, apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "file", // write the files to disk
DisableDataMigrations: true,
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "file", // write the files to disk
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
RESOURCEGROUP: {
DualWriterMode: grafanarest.Mode2,
@@ -145,9 +149,10 @@ func TestIntegrationPlaylist(t *testing.T) {
t.Run("with dual write (file, mode 3)", func(t *testing.T) {
doPlaylistTests(t, apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "file", // write the files to disk
DisableDataMigrations: true,
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "file", // write the files to disk
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
RESOURCEGROUP: {
DualWriterMode: grafanarest.Mode3,
@@ -158,9 +163,10 @@ func TestIntegrationPlaylist(t *testing.T) {
t.Run("with dual write (file, mode 5)", func(t *testing.T) {
helper := doPlaylistTests(t, apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "file", // write the files to disk
DisableDataMigrations: true,
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: "file", // write the files to disk
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
RESOURCEGROUP: {
DualWriterMode: grafanarest.Mode5,
@@ -198,9 +204,10 @@ func TestIntegrationPlaylist(t *testing.T) {
t.Run("with dual write (unified storage, mode 0)", func(t *testing.T) {
doPlaylistTests(t, apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: false, // required for unified storage
DisableAnonymous: true,
APIServerStorageType: options.StorageTypeUnified, // use the entity api tables
DisableDataMigrations: true,
AppModeProduction: false, // required for unified storage
DisableAnonymous: true,
APIServerStorageType: options.StorageTypeUnified, // use the entity api tables
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
RESOURCEGROUP: {
DualWriterMode: grafanarest.Mode0,
@@ -211,18 +218,20 @@ func TestIntegrationPlaylist(t *testing.T) {
t.Run("with dual write (unified storage, mode 1)", func(t *testing.T) {
doPlaylistTests(t, apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: false,
DisableAnonymous: true,
APIServerStorageType: options.StorageTypeUnified,
EnableFeatureToggles: []string{},
DisableDataMigrations: true,
AppModeProduction: false,
DisableAnonymous: true,
APIServerStorageType: options.StorageTypeUnified,
EnableFeatureToggles: []string{},
}))
})
t.Run("with dual write (unified storage, mode 2)", func(t *testing.T) {
doPlaylistTests(t, apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: false, // required for unified storage
DisableAnonymous: true,
APIServerStorageType: options.StorageTypeUnified, // use the entity api tables
DisableDataMigrations: true,
AppModeProduction: false, // required for unified storage
DisableAnonymous: true,
APIServerStorageType: options.StorageTypeUnified, // use the entity api tables
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
RESOURCEGROUP: {
DualWriterMode: grafanarest.Mode2,
@@ -233,9 +242,10 @@ func TestIntegrationPlaylist(t *testing.T) {
t.Run("with dual write (unified storage, mode 3)", func(t *testing.T) {
doPlaylistTests(t, apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: false, // required for unified storage
DisableAnonymous: true,
APIServerStorageType: options.StorageTypeUnified, // use the entity api tables
DisableDataMigrations: true,
AppModeProduction: false, // required for unified storage
DisableAnonymous: true,
APIServerStorageType: options.StorageTypeUnified, // use the entity api tables
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
RESOURCEGROUP: {
DualWriterMode: grafanarest.Mode3,
@@ -246,9 +256,10 @@ func TestIntegrationPlaylist(t *testing.T) {
t.Run("with dual write (unified storage, mode 5)", func(t *testing.T) {
doPlaylistTests(t, apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: false, // required for unified storage
DisableAnonymous: true,
APIServerStorageType: options.StorageTypeUnified, // use the entity api tables
DisableDataMigrations: true,
AppModeProduction: false, // required for unified storage
DisableAnonymous: true,
APIServerStorageType: options.StorageTypeUnified, // use the entity api tables
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
RESOURCEGROUP: {
DualWriterMode: grafanarest.Mode5,
@@ -262,9 +273,10 @@ func TestIntegrationPlaylist(t *testing.T) {
t.Skip("local etcd testing")
helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: options.StorageTypeEtcd, // requires etcd running on localhost:2379
DisableDataMigrations: true,
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: options.StorageTypeEtcd, // requires etcd running on localhost:2379
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
RESOURCEGROUP: {
DualWriterMode: grafanarest.Mode0,
@@ -288,9 +300,10 @@ func TestIntegrationPlaylist(t *testing.T) {
t.Skip("local etcd testing")
helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: options.StorageTypeEtcd, // requires etcd running on localhost:2379
DisableDataMigrations: true,
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: options.StorageTypeEtcd, // requires etcd running on localhost:2379
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
RESOURCEGROUP: {
DualWriterMode: grafanarest.Mode1,
@@ -314,9 +327,10 @@ func TestIntegrationPlaylist(t *testing.T) {
t.Skip("local etcd testing")
helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: options.StorageTypeEtcd, // requires etcd running on localhost:2379
DisableDataMigrations: true,
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: options.StorageTypeEtcd, // requires etcd running on localhost:2379
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
RESOURCEGROUP: {
DualWriterMode: grafanarest.Mode2,
@@ -340,9 +354,10 @@ func TestIntegrationPlaylist(t *testing.T) {
t.Skip("local etcd testing")
helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: options.StorageTypeEtcd, // requires etcd running on localhost:2379
DisableDataMigrations: true,
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: options.StorageTypeEtcd, // requires etcd running on localhost:2379
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
RESOURCEGROUP: {
DualWriterMode: grafanarest.Mode3,
@@ -366,9 +381,10 @@ func TestIntegrationPlaylist(t *testing.T) {
t.Skip("local etcd testing")
helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: options.StorageTypeEtcd, // requires etcd running on localhost:2379
DisableDataMigrations: true,
AppModeProduction: true,
DisableAnonymous: true,
APIServerStorageType: options.StorageTypeEtcd, // requires etcd running on localhost:2379
UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
RESOURCEGROUP: {
DualWriterMode: grafanarest.Mode5,
pkg/tests/testinfra/testinfra.go
+10
View File
@@ -362,6 +362,13 @@ func CreateGrafDir(t *testing.T, opts GrafanaOpts) (string, string) {
_, err = rbacSect.NewKey("permission_cache", "false")
require.NoError(t, err)
if opts.DisableAuthZClientCache {
authzSect, err := cfg.NewSection("authorization")
require.NoError(t, err)
_, err = authzSect.NewKey("cache_ttl", "0")
require.NoError(t, err)
}
analyticsSect, err := cfg.NewSection("analytics")
require.NoError(t, err)
_, err = analyticsSect.NewKey("intercom_secret", "intercom_secret_at_config")
@@ -547,6 +554,8 @@ func CreateGrafDir(t *testing.T, opts GrafanaOpts) (string, string) {
require.NoError(t, err)
_, err = section.NewKey("dualWriterMode", fmt.Sprintf("%d", v.DualWriterMode))
require.NoError(t, err)
_, err = section.NewKey("enableMigration", fmt.Sprintf("%t", v.EnableMigration))
require.NoError(t, err)
}
}
if opts.UnifiedStorageEnableSearch {
@@ -670,6 +679,7 @@ type GrafanaOpts struct {
DisableDataMigrations bool
SecretsManagerEnableDBMigrations bool
OpenFeatureAPIEnabled bool
DisableAuthZClientCache bool
// Allow creating grafana dir beforehand
Dir string