Resource server improvements and fixes (#90715)
* cleanup dependencies and improve list method * Improve Resource Server API, remove unnecessary dependencies * Reduce the API footprint of ResourceDBInterface and its implementation * Improve LifecycleHooks to use context * Improve testing * reduce API size and improve code * sqltemplate: add DialectForDriver func and improve naming * improve lifecycle API * many small fixes after adding more tests
This commit is contained in:
Diego Augusto Molina
@@ -0,0 +1,296 @@
|
||||
package test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.opentelemetry.io/otel/trace/noop"
|
||||
|
||||
infraDB "github.com/grafana/grafana/pkg/infra/db"
|
||||
"github.com/grafana/grafana/pkg/services/featuremgmt"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/db/dbimpl"
|
||||
"github.com/grafana/grafana/pkg/tests/testsuite"
|
||||
"github.com/grafana/grafana/pkg/util/testutil"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
testsuite.Run(m)
|
||||
}
|
||||
|
||||
func newServer(t *testing.T) sql.Backend {
|
||||
t.Helper()
|
||||
|
||||
dbstore := infraDB.InitTestDB(t)
|
||||
cfg := setting.NewCfg()
|
||||
features := featuremgmt.WithFeatures(featuremgmt.FlagUnifiedStorage)
|
||||
tr := noop.NewTracerProvider().Tracer("integrationtests")
|
||||
|
||||
eDB, err := dbimpl.ProvideResourceDB(dbstore, cfg, features, tr)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, eDB)
|
||||
|
||||
ret, err := sql.NewBackend(sql.BackendOptions{
|
||||
DBProvider: eDB,
|
||||
Tracer: tr,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, ret)
|
||||
|
||||
err = ret.Init(testutil.NewDefaultTestContext(t))
|
||||
require.NoError(t, err)
|
||||
|
||||
return ret
|
||||
}
|
||||
|
||||
func TestBackendHappyPath(t *testing.T) {
|
||||
ctx := testutil.NewDefaultTestContext(t)
|
||||
store := newServer(t)
|
||||
|
||||
stream, err := store.WatchWriteEvents(ctx)
|
||||
assert.NoError(t, err)
|
||||
|
||||
t.Run("Add 3 resources", func(t *testing.T) {
|
||||
rv, err := writeEvent(ctx, store, "item1", resource.WatchEvent_ADDED)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int64(1), rv)
|
||||
|
||||
rv, err = writeEvent(ctx, store, "item2", resource.WatchEvent_ADDED)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int64(2), rv)
|
||||
|
||||
rv, err = writeEvent(ctx, store, "item3", resource.WatchEvent_ADDED)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int64(3), rv)
|
||||
})
|
||||
|
||||
t.Run("Update item2", func(t *testing.T) {
|
||||
rv, err := writeEvent(ctx, store, "item2", resource.WatchEvent_MODIFIED)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int64(4), rv)
|
||||
})
|
||||
|
||||
t.Run("Delete item1", func(t *testing.T) {
|
||||
rv, err := writeEvent(ctx, store, "item1", resource.WatchEvent_DELETED)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int64(5), rv)
|
||||
})
|
||||
|
||||
t.Run("Read latest item 2", func(t *testing.T) {
|
||||
resp, err := store.Read(ctx, &resource.ReadRequest{Key: resourceKey("item2")})
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int64(4), resp.ResourceVersion)
|
||||
assert.Equal(t, "item2 MODIFIED", string(resp.Value))
|
||||
})
|
||||
|
||||
t.Run("Read early verion of item2", func(t *testing.T) {
|
||||
resp, err := store.Read(ctx, &resource.ReadRequest{
|
||||
Key: resourceKey("item2"),
|
||||
ResourceVersion: 3, // item2 was created at rv=2 and updated at rv=4
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int64(2), resp.ResourceVersion)
|
||||
assert.Equal(t, "item2 ADDED", string(resp.Value))
|
||||
})
|
||||
|
||||
t.Run("PrepareList latest", func(t *testing.T) {
|
||||
resp, err := store.PrepareList(ctx, &resource.ListRequest{
|
||||
Options: &resource.ListOptions{
|
||||
Key: &resource.ResourceKey{
|
||||
Namespace: "namespace",
|
||||
Group: "group",
|
||||
Resource: "resource",
|
||||
},
|
||||
},
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, resp.Items, 2)
|
||||
assert.Equal(t, "item2 MODIFIED", string(resp.Items[0].Value))
|
||||
assert.Equal(t, "item3 ADDED", string(resp.Items[1].Value))
|
||||
assert.Equal(t, int64(5), resp.ResourceVersion)
|
||||
})
|
||||
|
||||
t.Run("Watch events", func(t *testing.T) {
|
||||
event := <-stream
|
||||
assert.Equal(t, "item1", event.Key.Name)
|
||||
assert.Equal(t, int64(1), event.ResourceVersion)
|
||||
assert.Equal(t, resource.WatchEvent_ADDED, event.Type)
|
||||
event = <-stream
|
||||
assert.Equal(t, "item2", event.Key.Name)
|
||||
assert.Equal(t, int64(2), event.ResourceVersion)
|
||||
assert.Equal(t, resource.WatchEvent_ADDED, event.Type)
|
||||
|
||||
event = <-stream
|
||||
assert.Equal(t, "item3", event.Key.Name)
|
||||
assert.Equal(t, int64(3), event.ResourceVersion)
|
||||
assert.Equal(t, resource.WatchEvent_ADDED, event.Type)
|
||||
|
||||
event = <-stream
|
||||
assert.Equal(t, "item2", event.Key.Name)
|
||||
assert.Equal(t, int64(4), event.ResourceVersion)
|
||||
assert.Equal(t, resource.WatchEvent_MODIFIED, event.Type)
|
||||
|
||||
event = <-stream
|
||||
assert.Equal(t, "item1", event.Key.Name)
|
||||
assert.Equal(t, int64(5), event.ResourceVersion)
|
||||
assert.Equal(t, resource.WatchEvent_DELETED, event.Type)
|
||||
})
|
||||
}
|
||||
|
||||
func TestBackendWatchWriteEventsFromLastest(t *testing.T) {
|
||||
ctx := testutil.NewDefaultTestContext(t)
|
||||
store := newServer(t)
|
||||
|
||||
// Create a few resources before initing the watch
|
||||
_, err := writeEvent(ctx, store, "item1", resource.WatchEvent_ADDED)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Start the watch
|
||||
stream, err := store.WatchWriteEvents(ctx)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Create one more event
|
||||
_, err = writeEvent(ctx, store, "item2", resource.WatchEvent_ADDED)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "item2", (<-stream).Key.Name)
|
||||
}
|
||||
|
||||
func TestBackendPrepareList(t *testing.T) {
|
||||
ctx := testutil.NewDefaultTestContext(t)
|
||||
store := newServer(t)
|
||||
|
||||
// Create a few resources before initing the watch
|
||||
_, _ = writeEvent(ctx, store, "item1", resource.WatchEvent_ADDED) // rv=1
|
||||
_, _ = writeEvent(ctx, store, "item2", resource.WatchEvent_ADDED) // rv=2 - will be modified at rv=6
|
||||
_, _ = writeEvent(ctx, store, "item3", resource.WatchEvent_ADDED) // rv=3 - will be deleted at rv=7
|
||||
_, _ = writeEvent(ctx, store, "item4", resource.WatchEvent_ADDED) // rv=4
|
||||
_, _ = writeEvent(ctx, store, "item5", resource.WatchEvent_ADDED) // rv=5
|
||||
_, _ = writeEvent(ctx, store, "item2", resource.WatchEvent_MODIFIED) // rv=6
|
||||
_, _ = writeEvent(ctx, store, "item3", resource.WatchEvent_DELETED) // rv=7
|
||||
_, _ = writeEvent(ctx, store, "item6", resource.WatchEvent_ADDED) // rv=8
|
||||
t.Run("fetch all latest", func(t *testing.T) {
|
||||
res, err := store.PrepareList(ctx, &resource.ListRequest{
|
||||
Options: &resource.ListOptions{
|
||||
Key: &resource.ResourceKey{
|
||||
Group: "group",
|
||||
Resource: "resource",
|
||||
},
|
||||
},
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, res.Items, 5)
|
||||
assert.Empty(t, res.NextPageToken)
|
||||
})
|
||||
|
||||
t.Run("list latest first page ", func(t *testing.T) {
|
||||
res, err := store.PrepareList(ctx, &resource.ListRequest{
|
||||
Limit: 3,
|
||||
Options: &resource.ListOptions{
|
||||
Key: &resource.ResourceKey{
|
||||
Group: "group",
|
||||
Resource: "resource",
|
||||
},
|
||||
},
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, res.Items, 3)
|
||||
continueToken, err := sql.GetContinueToken(res.NextPageToken)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int64(8), continueToken.ResourceVersion)
|
||||
assert.Equal(t, int64(3), continueToken.StartOffset)
|
||||
})
|
||||
|
||||
t.Run("list at revision", func(t *testing.T) {
|
||||
res, err := store.PrepareList(ctx, &resource.ListRequest{
|
||||
ResourceVersion: 4,
|
||||
Options: &resource.ListOptions{
|
||||
Key: &resource.ResourceKey{
|
||||
Group: "group",
|
||||
Resource: "resource",
|
||||
},
|
||||
},
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, res.Items, 4)
|
||||
assert.Equal(t, "item1 ADDED", string(res.Items[0].Value))
|
||||
assert.Equal(t, "item2 ADDED", string(res.Items[1].Value))
|
||||
assert.Equal(t, "item3 ADDED", string(res.Items[2].Value))
|
||||
assert.Equal(t, "item4 ADDED", string(res.Items[3].Value))
|
||||
assert.Empty(t, res.NextPageToken)
|
||||
})
|
||||
|
||||
t.Run("fetch first page at revision with limit", func(t *testing.T) {
|
||||
res, err := store.PrepareList(ctx, &resource.ListRequest{
|
||||
Limit: 3,
|
||||
ResourceVersion: 7,
|
||||
Options: &resource.ListOptions{
|
||||
Key: &resource.ResourceKey{
|
||||
Group: "group",
|
||||
Resource: "resource",
|
||||
},
|
||||
},
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, res.Items, 3)
|
||||
assert.Equal(t, "item1 ADDED", string(res.Items[0].Value))
|
||||
assert.Equal(t, "item4 ADDED", string(res.Items[1].Value))
|
||||
assert.Equal(t, "item5 ADDED", string(res.Items[2].Value))
|
||||
|
||||
continueToken, err := sql.GetContinueToken(res.NextPageToken)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int64(7), continueToken.ResourceVersion)
|
||||
assert.Equal(t, int64(3), continueToken.StartOffset)
|
||||
})
|
||||
|
||||
t.Run("fetch second page at revision", func(t *testing.T) {
|
||||
continueToken := &sql.ContinueToken{
|
||||
ResourceVersion: 8,
|
||||
StartOffset: 2,
|
||||
}
|
||||
res, err := store.PrepareList(ctx, &resource.ListRequest{
|
||||
NextPageToken: continueToken.String(),
|
||||
Limit: 2,
|
||||
Options: &resource.ListOptions{
|
||||
Key: &resource.ResourceKey{
|
||||
Group: "group",
|
||||
Resource: "resource",
|
||||
},
|
||||
},
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, res.Items, 2)
|
||||
assert.Equal(t, "item5 ADDED", string(res.Items[0].Value))
|
||||
assert.Equal(t, "item2 MODIFIED", string(res.Items[1].Value))
|
||||
|
||||
continueToken, err = sql.GetContinueToken(res.NextPageToken)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int64(8), continueToken.ResourceVersion)
|
||||
assert.Equal(t, int64(4), continueToken.StartOffset)
|
||||
})
|
||||
}
|
||||
|
||||
func writeEvent(ctx context.Context, store sql.Backend, name string, action resource.WatchEvent_Type) (int64, error) {
|
||||
return store.WriteEvent(ctx, resource.WriteEvent{
|
||||
Type: action,
|
||||
Value: []byte(name + " " + resource.WatchEvent_Type_name[int32(action)]),
|
||||
Key: &resource.ResourceKey{
|
||||
Namespace: "namespace",
|
||||
Group: "group",
|
||||
Resource: "resource",
|
||||
Name: name,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func resourceKey(name string) *resource.ResourceKey {
|
||||
return &resource.ResourceKey{
|
||||
Namespace: "namespace",
|
||||
Group: "group",
|
||||
Resource: "resource",
|
||||
Name: name,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
sqlmock "github.com/DATA-DOG/go-sqlmock"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/db"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/db/dbimpl"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
|
||||
)
|
||||
|
||||
// TestDBProvider is a stub for db.ResourceDBInterface.
|
||||
type TestDBProvider struct {
|
||||
Err error
|
||||
DB db.DB
|
||||
SQLMock sqlmock.Sqlmock
|
||||
}
|
||||
|
||||
func (d TestDBProvider) Init(context.Context) (db.DB, error) {
|
||||
return d.DB, d.Err
|
||||
}
|
||||
|
||||
var _ db.DBProvider = TestDBProvider{}
|
||||
|
||||
// NewDBProviderNopSQL returns a TestDBProvider with a sqlmock.Sqlmock that
|
||||
// doesn't validates SQL. This is only meant to be used to test wrapping
|
||||
// utilities where the actual SQL is not relevant to the unit tests, but rather
|
||||
// how the possible derived error conditions handled.
|
||||
func NewDBProviderNopSQL(t *testing.T) TestDBProvider {
|
||||
t.Helper()
|
||||
mockDB, mock, err := sqlmock.New(
|
||||
sqlmock.QueryMatcherOption(sqlmock.QueryMatcherFunc(
|
||||
func(string, string) error { return nil },
|
||||
)),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
return TestDBProvider{
|
||||
DB: dbimpl.NewDB(mockDB, "mysql"),
|
||||
SQLMock: mock,
|
||||
}
|
||||
}
|
||||
|
||||
// NewDBProviderWithPing requires that database pings have a matching
|
||||
// expectation, which are ignored by default. The SQL matching is the sqlmock
|
||||
// default.
|
||||
func NewDBProviderWithPing(t *testing.T) TestDBProvider {
|
||||
t.Helper()
|
||||
mockDB, mock, err := sqlmock.New(sqlmock.MonitorPingsOption(true))
|
||||
require.NoError(t, err)
|
||||
|
||||
return TestDBProvider{
|
||||
DB: dbimpl.NewDB(mockDB, "mysql"),
|
||||
SQLMock: mock,
|
||||
}
|
||||
}
|
||||
|
||||
// NewDBProviderMatchWords returns a TestDBProvider with a sqlmock.Sqlmock that
|
||||
// will match SQL by splitting the expected SQL string into words, and then try
|
||||
// to find all of them in the actual SQL, in the given order, case
|
||||
// insensitively. Prepend a word with a `!` to say that word should not be
|
||||
// found.
|
||||
func NewDBProviderMatchWords(t *testing.T) TestDBProvider {
|
||||
t.Helper()
|
||||
mockDB, mock, err := sqlmock.New(
|
||||
sqlmock.QueryMatcherOption(
|
||||
sqlmock.QueryMatcherFunc(wordsMatcherFunc),
|
||||
),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
return TestDBProvider{
|
||||
DB: dbimpl.NewDB(mockDB, "mysql"),
|
||||
SQLMock: mock,
|
||||
}
|
||||
}
|
||||
|
||||
func wordsMatcherFunc(expectedSQL, actualSQL string) error {
|
||||
actualSQL = strings.ToLower(sqltemplate.FormatSQL(actualSQL))
|
||||
expectedSQL = strings.ToLower(expectedSQL)
|
||||
|
||||
var offset int
|
||||
for _, vv := range matchWorsRE.FindAllStringSubmatch(expectedSQL, -1) {
|
||||
v := vv[1]
|
||||
|
||||
var shouldNotMatch bool
|
||||
if v != "" && v[0] == '!' {
|
||||
v = v[1:]
|
||||
shouldNotMatch = true
|
||||
}
|
||||
if v == "" {
|
||||
return fmt.Errorf("invalid expected word %q in %q", v,
|
||||
expectedSQL)
|
||||
}
|
||||
|
||||
reWord, err := regexp.Compile(`\b` + regexp.QuoteMeta(v) + `\b`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("compile word %q from expected SQL: %s", v,
|
||||
expectedSQL)
|
||||
}
|
||||
|
||||
if shouldNotMatch {
|
||||
if reWord.MatchString(actualSQL[offset:]) {
|
||||
return fmt.Errorf("actual SQL fragent should not cont"+
|
||||
"ain %q but it does\n\tFragment: %s\n\tFull SQL: %s",
|
||||
v, actualSQL[offset:], actualSQL)
|
||||
}
|
||||
} else {
|
||||
loc := reWord.FindStringIndex(actualSQL[offset:])
|
||||
if len(loc) == 0 {
|
||||
return fmt.Errorf("actual SQL fragment should contain "+
|
||||
"%q but it doesn't\n\tFragment: %s\n\tFull SQL: %s",
|
||||
v, actualSQL[offset:], actualSQL)
|
||||
}
|
||||
offset = loc[1] // advance the offset
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
var matchWorsRE = regexp.MustCompile(`(?:\W|\A)(!?\w+)\b`)
|
||||
Reference in New Issue
Block a user