unified-storage: refactor Sql backend and sqlkv compat tests (#115849)

* move sql and sqlkv backends compatibility tests

* refactor compatibility tests

* run storage backend tests with and without rvmanager

* fix

* fix

* fmt

* fix

* address feedback

* fmt
This commit is contained in:
Will Assis
2026-01-08 15:06:44 -05:00
committed by GitHub
parent a79cda3328
commit f669bc4448
5 changed files with 319 additions and 296 deletions
+3 -2
View File
@@ -28,6 +28,7 @@ import (
"github.com/grafana/grafana/pkg/infra/log"
secrets "github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
"github.com/grafana/grafana/pkg/storage/unified/sql/rvmanager"
"github.com/grafana/grafana/pkg/util/scheduler"
)
@@ -815,7 +816,7 @@ func (s *server) update(ctx context.Context, user claims.AuthInfo, req *resource
// TODO: once we know the client is always sending the RV, require ResourceVersion > 0
// See: https://github.com/grafana/grafana/pull/111866
if req.ResourceVersion > 0 && latest.ResourceVersion != req.ResourceVersion {
if req.ResourceVersion > 0 && !rvmanager.IsRvEqual(latest.ResourceVersion, req.ResourceVersion) {
return &resourcepb.UpdateResponse{
Error: &ErrOptimisticLockingFailed,
}, nil
@@ -883,7 +884,7 @@ func (s *server) delete(ctx context.Context, user claims.AuthInfo, req *resource
rsp.Error = latest.Error
return rsp, nil
}
if req.ResourceVersion > 0 && latest.ResourceVersion != req.ResourceVersion {
if req.ResourceVersion > 0 && !rvmanager.IsRvEqual(latest.ResourceVersion, req.ResourceVersion) {
rsp.Error = &ErrOptimisticLockingFailed
return rsp, nil
}
@@ -107,16 +107,20 @@ func TestIntegrationSQLStorageAndSQLKVCompatibilityTests(t *testing.T) {
testutil.SkipIntegrationTestInShortMode(t)
t.Cleanup(db.CleanupTestDB)
newKvBackend := func(ctx context.Context) (resource.StorageBackend, sqldb.DB) {
return unitest.NewTestSqlKvBackend(t, ctx, true)
}
t.Run("IsHA (polling notifier)", func(t *testing.T) {
unitest.RunSQLStorageBackendCompatibilityTest(t, func(ctx context.Context) (resource.StorageBackend, sqldb.DB) {
return newTestBackend(t, true, 0)
}, nil)
}, newKvBackend, nil)
})
t.Run("NotHA (in process notifier)", func(t *testing.T) {
unitest.RunSQLStorageBackendCompatibilityTest(t, func(ctx context.Context) (resource.StorageBackend, sqldb.DB) {
return newTestBackend(t, false, 0)
}, nil)
}, newKvBackend, nil)
})
}
@@ -11,7 +11,6 @@ import (
"testing"
"time"
"github.com/bwmarrin/snowflake"
"github.com/go-jose/go-jose/v4/jwt"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
@@ -106,37 +105,6 @@ func RunStorageBackendTest(t *testing.T, newBackend NewBackendFunc, opts *TestOp
}
}
func RunSQLStorageBackendCompatibilityTest(t *testing.T, newBackend NewBackendWithDBFunc, opts *TestOptions) {
if opts == nil {
opts = &TestOptions{}
}
if opts.NSPrefix == "" {
opts.NSPrefix = GenerateRandomNSPrefix()
}
t.Logf("Running tests with namespace prefix: %s", opts.NSPrefix)
cases := []struct {
name string
fn func(*testing.T, resource.StorageBackend, string, sqldb.DB)
}{
{TestKeyPathGeneration, runTestIntegrationBackendKeyPathGeneration},
}
for _, tc := range cases {
if shouldSkip := opts.SkipTests[tc.name]; shouldSkip {
t.Logf("Skipping test: %s", tc.name)
continue
}
t.Run(tc.name, func(t *testing.T) {
backend, db := newBackend(context.Background())
tc.fn(t, backend, opts.NSPrefix, db)
})
}
}
func runTestIntegrationBackendHappyPath(t *testing.T, backend resource.StorageBackend, nsPrefix string) {
ctx := types.WithAuthInfo(context.Background(), authn.NewAccessTokenAuthInfo(authn.Claims[authn.AccessTokenClaims]{
Claims: jwt.Claims{
@@ -1759,222 +1727,3 @@ func runTestIntegrationBackendOptimisticLocking(t *testing.T, backend resource.S
require.LessOrEqual(t, successes, 1, "at most one create should succeed (errors: %v)", errorMessages)
})
}
func runTestIntegrationBackendKeyPathGeneration(t *testing.T, backend resource.StorageBackend, nsPrefix string, db sqldb.DB) {
ctx := testutil.NewDefaultTestContext(t)
t.Run("Create resource", func(t *testing.T) {
// Create a test resource
key := &resourcepb.ResourceKey{
Group: "playlist.grafana.app",
Resource: "playlists",
Namespace: nsPrefix + "-default",
Name: "test-playlist-crud",
}
// Create the K8s unstructured object
testObj := &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "playlist.grafana.app/v0alpha1",
"kind": "Playlist",
"metadata": map[string]interface{}{
"name": "test-playlist-crud",
"namespace": nsPrefix + "-default",
"uid": "test-uid-crud-123",
},
"spec": map[string]interface{}{
"title": "My Test Playlist",
},
},
}
// Get metadata accessor
metaAccessor, err := utils.MetaAccessor(testObj)
require.NoError(t, err)
// Serialize to JSON
jsonBytes, err := testObj.MarshalJSON()
require.NoError(t, err)
// Create WriteEvent
writeEvent := resource.WriteEvent{
Type: resourcepb.WatchEvent_ADDED,
Key: key,
Value: jsonBytes,
Object: metaAccessor,
PreviousRV: 0, // Always 0 for new resources
GUID: "create-guid-crud-123",
}
// Create the resource using WriteEvent
createRV, err := backend.WriteEvent(ctx, writeEvent)
require.NoError(t, err)
require.Greater(t, createRV, int64(0))
// Verify created resource key_path
verifyKeyPath(t, db, ctx, key, "created", createRV, "")
t.Run("Update resource", func(t *testing.T) {
// Update the resource
testObj.Object["spec"] = map[string]interface{}{
"title": "My Updated Playlist",
}
updatedMetaAccessor, err := utils.MetaAccessor(testObj)
require.NoError(t, err)
updatedJsonBytes, err := testObj.MarshalJSON()
require.NoError(t, err)
updateEvent := resource.WriteEvent{
Type: resourcepb.WatchEvent_MODIFIED,
Key: key,
Value: updatedJsonBytes,
Object: updatedMetaAccessor,
PreviousRV: createRV,
GUID: fmt.Sprintf("update-guid-%d", createRV),
}
// Update the resource
updateRV, err := backend.WriteEvent(ctx, updateEvent)
require.NoError(t, err)
require.Greater(t, updateRV, createRV)
// Verify updated resource key_path
verifyKeyPath(t, db, ctx, key, "updated", updateRV, "")
t.Run("Delete resource", func(t *testing.T) {
deleteEvent := resource.WriteEvent{
Type: resourcepb.WatchEvent_DELETED,
Key: key,
Value: updatedJsonBytes, // Keep the last known value
Object: updatedMetaAccessor,
PreviousRV: updateRV,
GUID: fmt.Sprintf("delete-guid-%d", updateRV),
}
// Delete the resource
deleteRV, err := backend.WriteEvent(ctx, deleteEvent)
require.NoError(t, err)
require.Greater(t, deleteRV, updateRV)
// Verify deleted resource key_path
verifyKeyPath(t, db, ctx, key, "deleted", deleteRV, "")
})
})
})
t.Run("Resource with folder", func(t *testing.T) {
// Create a resource in a folder
folderKey := &resourcepb.ResourceKey{
Group: "dashboard.grafana.app",
Resource: "dashboards",
Namespace: nsPrefix + "-default",
Name: "my-dashboard",
}
// Create dashboard object with folder
dashboardObj := &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "dashboard.grafana.app/v0alpha1",
"kind": "Dashboard",
"metadata": map[string]interface{}{
"name": "my-dashboard",
"namespace": nsPrefix + "-default",
"uid": "dash-uid-456",
"annotations": map[string]interface{}{
"grafana.app/folder": "test-folder",
},
},
"spec": map[string]interface{}{
"title": "My Dashboard",
},
},
}
folderMetaAccessor, err := utils.MetaAccessor(dashboardObj)
require.NoError(t, err)
folderJsonBytes, err := dashboardObj.MarshalJSON()
require.NoError(t, err)
folderWriteEvent := resource.WriteEvent{
Type: resourcepb.WatchEvent_ADDED,
Key: folderKey,
Value: folderJsonBytes,
Object: folderMetaAccessor,
PreviousRV: 0,
GUID: "folder-guid-456",
}
// Create the dashboard in folder
folderRV, err := backend.WriteEvent(ctx, folderWriteEvent)
require.NoError(t, err)
require.Greater(t, folderRV, int64(0))
// Verify folder resource key_path includes folder
verifyKeyPath(t, db, ctx, folderKey, "created", folderRV, "test-folder")
})
}
// verifyKeyPath is a helper function to verify key_path generation
func verifyKeyPath(t *testing.T, db sqldb.DB, ctx context.Context, key *resourcepb.ResourceKey, action string, resourceVersion int64, expectedFolder string) {
var query string
if db.DriverName() == "postgres" {
query = "SELECT key_path, resource_version, action, folder FROM resource_history WHERE namespace = $1 AND name = $2 AND resource_version = $3"
} else {
query = "SELECT key_path, resource_version, action, folder FROM resource_history WHERE namespace = ? AND name = ? AND resource_version = ?"
}
rows, err := db.QueryContext(ctx, query, key.Namespace, key.Name, resourceVersion)
require.NoError(t, err)
require.True(t, rows.Next())
var keyPath string
var actualRV int64
var actualAction int
var actualFolder string
err = rows.Scan(&keyPath, &actualRV, &actualAction, &actualFolder)
require.NoError(t, err)
err = rows.Close()
require.NoError(t, err)
// Verify basic key_path format
require.Contains(t, keyPath, "unified/data/")
require.Contains(t, keyPath, key.Group)
require.Contains(t, keyPath, key.Resource)
require.Contains(t, keyPath, key.Namespace)
require.Contains(t, keyPath, key.Name)
// Verify action suffix
require.Contains(t, keyPath, fmt.Sprintf("~%s~", action))
// Verify snowflake calculation
expectedSnowflake := (((resourceVersion / 1000) - snowflake.Epoch) << (snowflake.NodeBits + snowflake.StepBits)) + (resourceVersion % 1000)
require.Contains(t, keyPath, fmt.Sprintf("/%d~", expectedSnowflake), fmt.Sprintf("actual RV: %d", actualRV))
// Verify folder if specified
if expectedFolder != "" {
require.Equal(t, expectedFolder, actualFolder)
require.Contains(t, keyPath, expectedFolder)
}
// Verify action code matches
var expectedActionCode int
switch action {
case "created":
expectedActionCode = 1
case "updated":
expectedActionCode = 2
case "deleted":
expectedActionCode = 3
}
require.Equal(t, expectedActionCode, actualAction)
t.Logf("Action: %s, RV: %d, Snowflake: %d", action, resourceVersion, expectedSnowflake)
t.Logf("Key_path: %s", keyPath)
if expectedFolder != "" {
t.Logf("Folder: %s", actualFolder)
}
}
@@ -0,0 +1,286 @@
package test
import (
"context"
"fmt"
"testing"
"github.com/bwmarrin/snowflake"
"github.com/stretchr/testify/require"
claims "github.com/grafana/authlib/types"
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/storage/unified/resource"
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
sqldb "github.com/grafana/grafana/pkg/storage/unified/sql/db"
"github.com/grafana/grafana/pkg/storage/unified/sql/db/dbimpl"
"github.com/grafana/grafana/pkg/storage/unified/sql/rvmanager"
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
"github.com/grafana/grafana/pkg/util/testutil"
)
func NewTestSqlKvBackend(t *testing.T, ctx context.Context, withRvManager bool) (resource.KVBackend, sqldb.DB) {
dbstore := db.InitTestDB(t)
eDB, err := dbimpl.ProvideResourceDB(dbstore, setting.NewCfg(), nil)
require.NoError(t, err)
kv, err := resource.NewSQLKV(eDB)
require.NoError(t, err)
db, err := eDB.Init(ctx)
require.NoError(t, err)
kvOpts := resource.KVBackendOptions{
KvStore: kv,
}
if withRvManager {
dialect := sqltemplate.DialectForDriver(db.DriverName())
rvManager, err := rvmanager.NewResourceVersionManager(rvmanager.ResourceManagerOptions{
Dialect: dialect,
DB: db,
})
require.NoError(t, err)
kvOpts.RvManager = rvManager
}
backend, err := resource.NewKVStorageBackend(kvOpts)
require.NoError(t, err)
return backend, db
}
func RunSQLStorageBackendCompatibilityTest(t *testing.T, newSqlBackend, newKvBackend NewBackendWithDBFunc, opts *TestOptions) {
if opts == nil {
opts = &TestOptions{}
}
if opts.NSPrefix == "" {
opts.NSPrefix = GenerateRandomNSPrefix()
}
t.Logf("Running tests with namespace prefix: %s", opts.NSPrefix)
cases := []struct {
name string
fn func(*testing.T, resource.StorageBackend, resource.StorageBackend, string, sqldb.DB)
}{
{TestKeyPathGeneration, runTestIntegrationBackendKeyPathGeneration},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if opts.SkipTests[tc.name] {
t.Skip()
}
kvbackend, db := newKvBackend(t.Context())
sqlbackend, _ := newSqlBackend(t.Context())
tc.fn(t, sqlbackend, kvbackend, opts.NSPrefix, db)
})
}
}
func runTestIntegrationBackendKeyPathGeneration(t *testing.T, sqlBackend, kvBackend resource.StorageBackend, nsPrefix string, db sqldb.DB) {
ctx := testutil.NewDefaultTestContext(t)
// Test SQL backend with 3 writes, 3 updates, 3 deletes
t.Run("SQL Backend Operations", func(t *testing.T) {
runKeyPathTest(t, sqlBackend, nsPrefix+"-sql", db, ctx)
})
// Test SQL KV backend with 3 writes, 3 updates, 3 deletes
t.Run("SQL KV Backend Operations", func(t *testing.T) {
runKeyPathTest(t, kvBackend, nsPrefix+"-kv", db, ctx)
})
}
// runKeyPathTest performs 3 writes, 3 updates, and 3 deletes on a backend then verifies that key_path is properly
// generated across both backends
func runKeyPathTest(t *testing.T, backend resource.StorageBackend, nsPrefix string, db sqldb.DB, ctx context.Context) {
// Create storage server from backend
server, err := resource.NewResourceServer(resource.ResourceServerOptions{
Backend: backend,
AccessClient: claims.FixedAccessClient(true), // Allow all operations for testing
})
require.NoError(t, err)
// Track the current resource version for each resource (index 0, 1, 2 for resources 1, 2, 3)
currentRVs := make([]int64, 3)
// Create 3 resources
for i := 1; i <= 3; i++ {
key := &resourcepb.ResourceKey{
Group: "playlist.grafana.app",
Resource: "playlists",
Namespace: nsPrefix,
Name: fmt.Sprintf("test-playlist-%d", i),
}
// Create resource JSON with folder annotation for resource 2
resourceJSON := fmt.Sprintf(`{
"apiVersion": "playlist.grafana.app/v0alpha1",
"kind": "Playlist",
"metadata": {
"name": "test-playlist-%d",
"namespace": "%s",
"uid": "test-uid-%d"%s
},
"spec": {
"title": "My Test Playlist %d"
}
}`, i, nsPrefix, i, getAnnotationsJSON(i == 2), i)
// Create the resource using server.Create
created, err := server.Create(ctx, &resourcepb.CreateRequest{
Key: key,
Value: []byte(resourceJSON),
})
require.NoError(t, err)
require.Nil(t, created.Error)
require.Greater(t, created.ResourceVersion, int64(0))
currentRVs[i-1] = created.ResourceVersion
// Verify created resource key_path (with folder for resource 2)
if i == 2 {
verifyKeyPath(t, db, ctx, key, "created", created.ResourceVersion, "test-folder")
} else {
verifyKeyPath(t, db, ctx, key, "created", created.ResourceVersion, "")
}
}
// Update the 3 resources
for i := 1; i <= 3; i++ {
key := &resourcepb.ResourceKey{
Group: "playlist.grafana.app",
Resource: "playlists",
Namespace: nsPrefix,
Name: fmt.Sprintf("test-playlist-%d", i),
}
// Create updated resource JSON with folder annotation for resource 2
updatedResourceJSON := fmt.Sprintf(`{
"apiVersion": "playlist.grafana.app/v0alpha1",
"kind": "Playlist",
"metadata": {
"name": "test-playlist-%d",
"namespace": "%s",
"uid": "test-uid-%d"%s
},
"spec": {
"title": "My Updated Playlist %d"
}
}`, i, nsPrefix, i, getAnnotationsJSON(i == 2), i)
// Update the resource using server.Update
updated, err := server.Update(ctx, &resourcepb.UpdateRequest{
Key: key,
Value: []byte(updatedResourceJSON),
ResourceVersion: currentRVs[i-1], // Use the resource version returned by previous operation
})
require.NoError(t, err)
require.Nil(t, updated.Error)
require.Greater(t, updated.ResourceVersion, currentRVs[i-1])
currentRVs[i-1] = updated.ResourceVersion // Update to the latest resource version
// Verify updated resource key_path (with folder for resource 2)
if i == 2 {
verifyKeyPath(t, db, ctx, key, "updated", updated.ResourceVersion, "test-folder")
} else {
verifyKeyPath(t, db, ctx, key, "updated", updated.ResourceVersion, "")
}
}
// Delete the 3 resources
for i := 1; i <= 3; i++ {
key := &resourcepb.ResourceKey{
Group: "playlist.grafana.app",
Resource: "playlists",
Namespace: nsPrefix,
Name: fmt.Sprintf("test-playlist-%d", i),
}
// Delete the resource using server.Delete
deleted, err := server.Delete(ctx, &resourcepb.DeleteRequest{
Key: key,
ResourceVersion: currentRVs[i-1], // Use the resource version from previous operation
})
require.NoError(t, err)
require.Greater(t, deleted.ResourceVersion, currentRVs[i-1])
// Verify deleted resource key_path (with folder for resource 2)
if i == 2 {
verifyKeyPath(t, db, ctx, key, "deleted", deleted.ResourceVersion, "test-folder")
} else {
verifyKeyPath(t, db, ctx, key, "deleted", deleted.ResourceVersion, "")
}
}
}
// verifyKeyPath is a helper function to verify key_path generation
func verifyKeyPath(t *testing.T, db sqldb.DB, ctx context.Context, key *resourcepb.ResourceKey, action string, resourceVersion int64, expectedFolder string) {
var query string
if db.DriverName() == "postgres" {
query = "SELECT key_path, resource_version, action, folder FROM resource_history WHERE namespace = $1 AND name = $2 AND resource_version = $3"
} else {
query = "SELECT key_path, resource_version, action, folder FROM resource_history WHERE namespace = ? AND name = ? AND resource_version = ?"
}
rows, err := db.QueryContext(ctx, query, key.Namespace, key.Name, resourceVersion)
require.NoError(t, err)
require.True(t, rows.Next(), "Resource not found in resource_history table - both SQL and KV backends should write to this table")
var keyPath string
var actualRV int64
var actualAction int
var actualFolder string
err = rows.Scan(&keyPath, &actualRV, &actualAction, &actualFolder)
require.NoError(t, err)
// Ensure there's exactly one row and no errors
require.False(t, rows.Next())
require.NoError(t, rows.Err())
// Verify basic key_path format
require.Contains(t, keyPath, "unified/data/")
require.Contains(t, keyPath, key.Group)
require.Contains(t, keyPath, key.Resource)
require.Contains(t, keyPath, key.Namespace)
require.Contains(t, keyPath, key.Name)
// Verify action suffix
require.Contains(t, keyPath, fmt.Sprintf("~%s~", action))
// Verify snowflake calculation
expectedSnowflake := (((resourceVersion / 1000) - snowflake.Epoch) << (snowflake.NodeBits + snowflake.StepBits)) + (resourceVersion % 1000)
require.Contains(t, keyPath, fmt.Sprintf("/%d~", expectedSnowflake), "actual RV: %d", actualRV)
// Verify folder if specified
if expectedFolder != "" {
require.Equal(t, expectedFolder, actualFolder)
require.Contains(t, keyPath, expectedFolder)
}
// Verify action code matches
var expectedActionCode int
switch action {
case "created":
expectedActionCode = 1
case "updated":
expectedActionCode = 2
case "deleted":
expectedActionCode = 3
}
require.Equal(t, expectedActionCode, actualAction)
}
// getAnnotationsJSON returns the annotations JSON string for the folder annotation if needed
func getAnnotationsJSON(withFolder bool) string {
if withFolder {
return `,
"annotations": {
"grafana.app/folder": "test-folder"
}`
}
return ""
}
@@ -7,11 +7,7 @@ import (
badger "github.com/dgraph-io/badger/v4"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/storage/unified/resource"
sqldb "github.com/grafana/grafana/pkg/storage/unified/sql/db"
"github.com/grafana/grafana/pkg/storage/unified/sql/db/dbimpl"
)
func TestBadgerKVStorageBackend(t *testing.T) {
@@ -41,48 +37,35 @@ func TestBadgerKVStorageBackend(t *testing.T) {
}
func TestSQLKVStorageBackend(t *testing.T) {
newBackendFunc := func(ctx context.Context) (resource.StorageBackend, sqldb.DB) {
dbstore := db.InitTestDB(t)
eDB, err := dbimpl.ProvideResourceDB(dbstore, setting.NewCfg(), nil)
require.NoError(t, err)
kv, err := resource.NewSQLKV(eDB)
require.NoError(t, err)
kvOpts := resource.KVBackendOptions{
KvStore: kv,
}
backend, err := resource.NewKVStorageBackend(kvOpts)
require.NoError(t, err)
db, err := eDB.Init(ctx)
require.NoError(t, err)
return backend, db
skipTests := map[string]bool{
TestHappyPath: true,
TestWatchWriteEvents: true,
TestList: true,
TestBlobSupport: true,
TestGetResourceStats: true,
TestListHistory: true,
TestListHistoryErrorReporting: true,
TestListModifiedSince: true,
TestListTrash: true,
TestCreateNewResource: true,
TestGetResourceLastImportTime: true,
TestOptimisticLocking: true,
}
// without RvManager
RunStorageBackendTest(t, func(ctx context.Context) resource.StorageBackend {
backend, _ := newBackendFunc(ctx)
backend, _ := NewTestSqlKvBackend(t, ctx, false)
return backend
}, &TestOptions{
NSPrefix: "sqlkvstorage-test",
SkipTests: map[string]bool{
TestHappyPath: true,
TestWatchWriteEvents: true,
TestList: true,
TestBlobSupport: true,
TestGetResourceStats: true,
TestListHistory: true,
TestListHistoryErrorReporting: true,
TestListModifiedSince: true,
TestListTrash: true,
TestCreateNewResource: true,
TestGetResourceLastImportTime: true,
TestOptimisticLocking: true,
TestKeyPathGeneration: true,
},
NSPrefix: "sqlkvstorage-test",
SkipTests: skipTests,
})
RunSQLStorageBackendCompatibilityTest(t, newBackendFunc, &TestOptions{
NSPrefix: "sqlkvstorage-compatibility-test",
SkipTests: map[string]bool{
TestKeyPathGeneration: true,
},
// with RvManager
RunStorageBackendTest(t, func(ctx context.Context) resource.StorageBackend {
backend, _ := NewTestSqlKvBackend(t, ctx, true)
return backend
}, &TestOptions{
NSPrefix: "sqlkvstorage-withrvmanager-test",
SkipTests: skipTests,
})
}