Resource server improvements and fixes (#90715)

* cleanup dependencies and improve list method
* Improve Resource Server API, remove unnecessary dependencies
* Reduce the API footprint of ResourceDBInterface and its implementation
* Improve LifecycleHooks to use context
* Improve testing
* reduce API size and improve code
* sqltemplate: add DialectForDriver func and improve naming
* improve lifecycle API
* many small fixes after adding more tests
This commit is contained in:
Diego Augusto Molina
2024-07-22 20:08:30 +03:00
committed by GitHub
parent 5f367f05dc
commit 399d77a0fd
28 changed files with 2193 additions and 945 deletions
+209
View File
@@ -0,0 +1,209 @@
// Package dbutil provides utilities to perform common database operations and
// appropriate error handling.
package dbutil
import (
"context"
"database/sql"
"errors"
"fmt"
"strings"
"text/template"
"github.com/grafana/grafana/pkg/storage/unified/sql/db"
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
)
// SQLError is an error returned by the database, which includes additionally
// debugging information about what was sent to the database.
type SQLError struct {
Err error
CallType string // either Query, QueryRow or Exec
TemplateName string
Query string
RawQuery string
ScanDest []any
// potentially regulated information is not exported and only directly
// available for local testing and local debugging purposes, making sure it
// is never marshaled to JSON or any other serialization.
arguments []any
}
func (e SQLError) Unwrap() error {
return e.Err
}
func (e SQLError) Error() string {
return fmt.Sprintf("%s: %s with %d input arguments and %d output "+
"destination arguments: %v; query: %s", e.TemplateName, e.CallType,
len(e.arguments), len(e.ScanDest), e.Err, e.Query)
}
// Debug provides greater detail about the SQL error. It is defined on the same
// struct but on a test file so that the intention that its results should not
// be used in runtime code is very clear. The results could include PII or
// otherwise regulated information, hence this method is only available in
// tests, so that it can be used in local debugging only. Note that the error
// information may still be available through other means, like using the
// "reflect" package, so care must be taken not to ever expose these information
// in production.
func (e SQLError) Debug() string {
scanDestStr := "(none)"
if len(e.ScanDest) > 0 {
format := "[%T" + strings.Repeat(", %T", len(e.ScanDest)-1) + "]"
scanDestStr = fmt.Sprintf(format, e.ScanDest...)
}
return fmt.Sprintf("%s: %s: %v\n\tArguments (%d): %#v\n\tReturn Value "+
"Types (%d): %s\n\tExecuted Query: %s\n\tRaw SQL Template Output: %s",
e.TemplateName, e.CallType, e.Err, len(e.arguments), e.arguments,
len(e.ScanDest), scanDestStr, e.Query, e.RawQuery)
}
// Debug is meant to provide greater debugging detail about certain errors. The
// returned error will either provide more detailed information or be the same
// original error, suitable only for local debugging. The details provided are
// not meant to be logged, since they could include PII or otherwise
// sensitive/confidential information. These information should only be used for
// local debugging with fake or otherwise non-regulated information.
func Debug(err error) error {
var d interface{ Debug() string }
if errors.As(err, &d) {
return errors.New(d.Debug())
}
return err
}
// Exec uses `req` as input for a non-data returning query generated with
// `tmpl`, and executed in `x`.
func Exec(ctx context.Context, x db.ContextExecer, tmpl *template.Template, req sqltemplate.SQLTemplateIface) (sql.Result, error) {
if err := req.Validate(); err != nil {
return nil, fmt.Errorf("Exec: invalid request for template %q: %w",
tmpl.Name(), err)
}
rawQuery, err := sqltemplate.Execute(tmpl, req)
if err != nil {
return nil, fmt.Errorf("execute template: %w", err)
}
query := sqltemplate.FormatSQL(rawQuery)
res, err := x.ExecContext(ctx, query, req.GetArgs()...)
if err != nil {
return nil, SQLError{
Err: err,
CallType: "Exec",
TemplateName: tmpl.Name(),
arguments: req.GetArgs(),
Query: query,
RawQuery: rawQuery,
}
}
return res, nil
}
// Query uses `req` as input for a single-statement, set-returning query
// generated with `tmpl`, and executed in `x`. The `Results` method of `req`
// should return a deep copy since it will be used multiple times to decode each
// value. It returns an error if more than one result set is returned.
func Query[T any](ctx context.Context, x db.ContextExecer, tmpl *template.Template, req sqltemplate.WithResults[T]) ([]T, error) {
if err := req.Validate(); err != nil {
return nil, fmt.Errorf("Query: invalid request for template %q: %w",
tmpl.Name(), err)
}
rawQuery, err := sqltemplate.Execute(tmpl, req)
if err != nil {
return nil, fmt.Errorf("execute template %q: %w", tmpl.Name(), err)
}
query := sqltemplate.FormatSQL(rawQuery)
rows, err := x.QueryContext(ctx, query, req.GetArgs()...)
if err != nil {
return nil, SQLError{
Err: err,
CallType: "Query",
TemplateName: tmpl.Name(),
arguments: req.GetArgs(),
ScanDest: req.GetScanDest(),
Query: query,
RawQuery: rawQuery,
}
}
var ret []T
for rows.Next() {
v, err := scanRow(rows, req)
if err != nil {
return nil, fmt.Errorf("scan value #%d: %w", len(ret)+1, err)
}
ret = append(ret, v)
}
discardedResultSets, err := DiscardRows(rows)
if err != nil {
return nil, fmt.Errorf("closing rows: %w", err)
}
if discardedResultSets > 1 {
return nil, fmt.Errorf("too many result sets: %v", discardedResultSets)
}
return ret, nil
}
// QueryRow uses `req` as input and output for a single-statement, single-row
// returning query generated with `tmpl`, and executed in `x`. It returns
// sql.ErrNoRows if no rows are returned. It also returns an error if more than
// one row or result set is returned.
func QueryRow[T any](ctx context.Context, x db.ContextExecer, tmpl *template.Template, req sqltemplate.WithResults[T]) (T, error) {
var zero T
res, err := Query(ctx, x, tmpl, req)
if err != nil {
return zero, err
}
switch len(res) {
case 0:
return zero, sql.ErrNoRows
case 1:
return res[0], nil
default:
return zero, fmt.Errorf("expecting a single row, got %d", len(res))
}
}
// DiscardRows discards all the ResultSets in the given *sql.Rows and returns
// the final rows error and the number of times NextResultSet was called. This
// is useful to check for errors in queries with multiple SQL statements where
// there is no interesting output, since some drivers may omit an error returned
// by a SQL statement found in a statement that is not the first one. Note that
// not all drivers support multi-statement calls, though.
func DiscardRows(rows *sql.Rows) (int, error) {
discardedResultSets := 1
for ; rows.NextResultSet(); discardedResultSets++ {
}
return discardedResultSets, rows.Err()
}
type scanner interface {
Scan(dest ...any) error
}
// scanRow is used on *sql.Row and *sql.Rows, and is factored out here not to
// improving code reuse, but rather for ease of testing.
func scanRow[T any](sc scanner, req sqltemplate.WithResults[T]) (zero T, err error) {
if err = sc.Scan(req.GetScanDest()...); err != nil {
return zero, fmt.Errorf("row scan: %w", err)
}
res, err := req.Results()
if err != nil {
return zero, fmt.Errorf("row results: %w", err)
}
return res, nil
}
@@ -0,0 +1,513 @@
package dbutil
import (
"database/sql"
"errors"
"testing"
"text/template"
"github.com/DATA-DOG/go-sqlmock"
"github.com/stretchr/testify/require"
sqltemplateMocks "github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate/mocks"
"github.com/grafana/grafana/pkg/storage/unified/sql/test"
"github.com/grafana/grafana/pkg/util/testutil"
)
var (
validTestTmpl = template.Must(template.New("test").Parse("nothing special"))
invalidTestTmpl = template.New("no definition should fail to exec")
errTest = errors.New("because of reasons")
)
func TestSQLError(t *testing.T) {
t.Parallel()
const hiddenMessage = "obey, consume"
var err error = SQLError{
Err: errTest,
CallType: "Exec",
TemplateName: "some.sql",
Query: "SELECT name FROM movies WHERE quote LIKE ?",
RawQuery: "SELECT name FROM movies WHERE quote LIKE ?",
ScanDest: []any{new(string)},
arguments: []any{hiddenMessage},
}
require.Error(t, err)
require.ErrorIs(t, err, errTest)
require.NotContains(t, err.Error(), hiddenMessage)
err = Debug(err)
require.Error(t, err)
require.Contains(t, err.Error(), hiddenMessage)
err = Debug(errTest)
require.Error(t, err)
require.ErrorIs(t, err, errTest)
}
// expectRows is a testing helper to keep mocks in sync when adding rows to a
// mocked SQL result.
type expectRows[T any] struct {
*sqlmock.Rows
ExpectedResults []T
req *sqltemplateMocks.WithResults[T]
}
func newReturnsRow[T any](dbmock sqlmock.Sqlmock, req *sqltemplateMocks.WithResults[T]) *expectRows[T] {
return &expectRows[T]{
Rows: dbmock.NewRows(nil),
req: req,
}
}
// Add adds a new value that should be returned by the `Query` or `QueryRow`
// operation.
func (r *expectRows[T]) Add(value T, err error) *expectRows[T] {
r.req.EXPECT().GetScanDest().Return(nil).Once()
r.req.EXPECT().Results().Return(value, err).Once()
r.Rows.AddRow()
r.ExpectedResults = append(r.ExpectedResults, value)
return r
}
func TestQuery(t *testing.T) {
t.Parallel()
t.Run("happy path - no rows returned", func(t *testing.T) {
t.Parallel()
// test declarations
ctx := testutil.NewDefaultTestContext(t)
req := sqltemplateMocks.NewWithResults[int64](t)
rdb := test.NewDBProviderNopSQL(t)
// setup expectations
req.EXPECT().Validate().Return(nil).Once()
req.EXPECT().GetArgs().Return(nil)
req.EXPECT().GetScanDest().Return(nil).Maybe()
rdb.SQLMock.ExpectQuery("").WillReturnRows(rdb.SQLMock.NewRows(nil))
// execute and assert
res, err := Query(ctx, rdb.DB, validTestTmpl, req)
require.NoError(t, err)
require.Zero(t, res)
})
t.Run("happy path - multiple rows returned", func(t *testing.T) {
t.Parallel()
// test declarations
ctx := testutil.NewDefaultTestContext(t)
req := sqltemplateMocks.NewWithResults[int64](t)
rdb := test.NewDBProviderNopSQL(t)
rows := newReturnsRow(rdb.SQLMock, req)
// setup expectations
req.EXPECT().Validate().Return(nil).Once()
req.EXPECT().GetArgs().Return(nil).Once()
rows.Add(1, nil)
rows.Add(2, nil)
rows.Add(3, nil)
rdb.SQLMock.ExpectQuery("").WillReturnRows(rows.Rows)
// execute and assert
res, err := Query(ctx, rdb.DB, validTestTmpl, req)
require.NoError(t, err)
require.NotZero(t, res)
require.Equal(t, rows.ExpectedResults, res)
})
t.Run("invalid request", func(t *testing.T) {
t.Parallel()
// test declarations
ctx := testutil.NewDefaultTestContext(t)
req := sqltemplateMocks.NewWithResults[int64](t)
rdb := test.NewDBProviderNopSQL(t)
// setup expectations
req.EXPECT().Validate().Return(errTest).Once()
// execute and assert
res, err := Query(ctx, rdb.DB, invalidTestTmpl, req)
require.Zero(t, res)
require.Error(t, err)
require.ErrorContains(t, err, "invalid request")
})
t.Run("error executing template", func(t *testing.T) {
t.Parallel()
// test declarations
ctx := testutil.NewDefaultTestContext(t)
req := sqltemplateMocks.NewWithResults[int64](t)
rdb := test.NewDBProviderNopSQL(t)
// setup expectations
req.EXPECT().Validate().Return(nil).Once()
// execute and assert
res, err := Query(ctx, rdb.DB, invalidTestTmpl, req)
require.Zero(t, res)
require.Error(t, err)
require.ErrorContains(t, err, "execute template")
})
t.Run("error executing query", func(t *testing.T) {
t.Parallel()
// test declarations
ctx := testutil.NewDefaultTestContext(t)
req := sqltemplateMocks.NewWithResults[int64](t)
rdb := test.NewDBProviderNopSQL(t)
// setup expectations
req.EXPECT().Validate().Return(nil).Once()
req.EXPECT().GetArgs().Return(nil)
req.EXPECT().GetScanDest().Return(nil).Maybe()
rdb.SQLMock.ExpectQuery("").WillReturnError(errTest)
// execute and assert
res, err := Query(ctx, rdb.DB, validTestTmpl, req)
require.Zero(t, res)
require.Error(t, err)
require.ErrorAs(t, err, new(SQLError))
})
t.Run("error decoding row", func(t *testing.T) {
t.Parallel()
// test declarations
ctx := testutil.NewDefaultTestContext(t)
req := sqltemplateMocks.NewWithResults[int64](t)
rdb := test.NewDBProviderNopSQL(t)
rows := newReturnsRow(rdb.SQLMock, req)
// setup expectations
req.EXPECT().Validate().Return(nil).Once()
req.EXPECT().GetArgs().Return(nil).Once()
rows.Add(0, errTest)
rdb.SQLMock.ExpectQuery("").WillReturnRows(rows.Rows)
// execute and assert
res, err := Query(ctx, rdb.DB, validTestTmpl, req)
require.Zero(t, res)
require.Error(t, err)
require.ErrorContains(t, err, "scan value")
})
t.Run("error iterating rows", func(t *testing.T) {
t.Parallel()
// test declarations
ctx := testutil.NewDefaultTestContext(t)
req := sqltemplateMocks.NewWithResults[int64](t)
rdb := test.NewDBProviderNopSQL(t)
rows := newReturnsRow(rdb.SQLMock, req)
// setup expectations
req.EXPECT().Validate().Return(nil).Once()
req.EXPECT().GetArgs().Return(nil).Once()
rows.Rows.AddRow() // we don't expect GetScanDest or Results here
rows.Rows.RowError(0, errTest)
rdb.SQLMock.ExpectQuery("").WillReturnRows(rows.Rows)
// execute and assert
res, err := Query(ctx, rdb.DB, validTestTmpl, req)
require.Zero(t, res)
require.Error(t, err)
require.ErrorContains(t, err, "closing rows")
})
t.Run("too many result sets", func(t *testing.T) {
t.Parallel()
// test declarations
ctx := testutil.NewDefaultTestContext(t)
req := sqltemplateMocks.NewWithResults[int64](t)
rdb := test.NewDBProviderNopSQL(t)
rows1 := newReturnsRow(rdb.SQLMock, req)
rows2 := newReturnsRow(rdb.SQLMock, req)
// setup expectations
req.EXPECT().Validate().Return(nil).Once()
req.EXPECT().GetArgs().Return(nil).Once()
rows1.Add(1, nil)
rows2.Rows.AddRow() // we don't expect GetScanDest or Results here
rdb.SQLMock.ExpectQuery("").WillReturnRows(rows1.Rows, rows2.Rows)
// execute and assert
res, err := Query(ctx, rdb.DB, validTestTmpl, req)
require.Zero(t, res)
require.Error(t, err)
require.ErrorContains(t, err, "too many result sets")
})
}
func TestQueryRow(t *testing.T) {
t.Parallel()
t.Run("happy path", func(t *testing.T) {
t.Parallel()
// test declarations
ctx := testutil.NewDefaultTestContext(t)
req := sqltemplateMocks.NewWithResults[int64](t)
rdb := test.NewDBProviderNopSQL(t)
rows := newReturnsRow(rdb.SQLMock, req)
// setup expectations
req.EXPECT().Validate().Return(nil).Once()
req.EXPECT().GetArgs().Return(nil).Once()
rows.Add(1, nil)
rdb.SQLMock.ExpectQuery("").WillReturnRows(rows.Rows)
// execute and assert
res, err := QueryRow(ctx, rdb.DB, validTestTmpl, req)
require.NoError(t, err)
require.Equal(t, rows.ExpectedResults[0], res)
})
t.Run("no rows returned", func(t *testing.T) {
t.Parallel()
// test declarations
ctx := testutil.NewDefaultTestContext(t)
req := sqltemplateMocks.NewWithResults[int64](t)
rdb := test.NewDBProviderNopSQL(t)
// setup expectations
req.EXPECT().Validate().Return(nil).Once()
req.EXPECT().GetArgs().Return(nil).Once()
rdb.SQLMock.ExpectQuery("").WillReturnRows(rdb.SQLMock.NewRows(nil))
// execute and assert
res, err := QueryRow(ctx, rdb.DB, validTestTmpl, req)
require.Zero(t, res)
require.Error(t, err)
require.ErrorIs(t, err, sql.ErrNoRows)
})
t.Run("error executing query", func(t *testing.T) {
t.Parallel()
// test declarations
ctx := testutil.NewDefaultTestContext(t)
req := sqltemplateMocks.NewWithResults[int64](t)
rdb := test.NewDBProviderNopSQL(t)
// setup expectations
req.EXPECT().Validate().Return(nil).Once()
req.EXPECT().GetArgs().Return(nil)
req.EXPECT().GetScanDest().Return(nil).Maybe()
rdb.SQLMock.ExpectQuery("").WillReturnError(errTest)
// execute and assert
res, err := QueryRow(ctx, rdb.DB, validTestTmpl, req)
require.Zero(t, res)
require.Error(t, err)
require.ErrorAs(t, err, new(SQLError))
})
t.Run("too many rows returned", func(t *testing.T) {
t.Parallel()
// test declarations
ctx := testutil.NewDefaultTestContext(t)
req := sqltemplateMocks.NewWithResults[int64](t)
rdb := test.NewDBProviderNopSQL(t)
rows := newReturnsRow(rdb.SQLMock, req)
// setup expectations
req.EXPECT().Validate().Return(nil).Once()
req.EXPECT().GetArgs().Return(nil).Once()
rows.Add(1, nil)
rows.Add(2, nil)
rdb.SQLMock.ExpectQuery("").WillReturnRows(rows.Rows)
// execute and assert
res, err := QueryRow(ctx, rdb.DB, validTestTmpl, req)
require.Zero(t, res)
require.Error(t, err)
require.ErrorContains(t, err, "expecting a single row")
})
t.Run("too many result sets", func(t *testing.T) {
t.Parallel()
// test declarations
ctx := testutil.NewDefaultTestContext(t)
req := sqltemplateMocks.NewWithResults[int64](t)
rdb := test.NewDBProviderNopSQL(t)
rows1 := newReturnsRow(rdb.SQLMock, req)
rows2 := newReturnsRow(rdb.SQLMock, req)
// setup expectations
req.EXPECT().Validate().Return(nil).Once()
req.EXPECT().GetArgs().Return(nil).Once()
rows1.Add(1, nil)
rows2.Rows.AddRow() // we don't expect GetScanDest or Results here
rdb.SQLMock.ExpectQuery("").WillReturnRows(rows1.Rows, rows2.Rows)
// execute and assert
res, err := QueryRow(ctx, rdb.DB, validTestTmpl, req)
require.Zero(t, res)
require.Error(t, err)
require.ErrorContains(t, err, "too many result sets")
})
}
// scannerFunc is an adapter for the `scanner` interface.
type scannerFunc func(dest ...any) error
func (f scannerFunc) Scan(dest ...any) error {
return f(dest...)
}
func TestScanRow(t *testing.T) {
t.Parallel()
const value int64 = 1
t.Run("happy path", func(t *testing.T) {
t.Parallel()
// test declarations
req := sqltemplateMocks.NewWithResults[int64](t)
sc := scannerFunc(func(dest ...any) error {
return nil
})
// setup expectations
req.EXPECT().GetScanDest().Return(nil).Once()
req.EXPECT().Results().Return(value, nil).Once()
// execute and assert
res, err := scanRow(sc, req)
require.NoError(t, err)
require.Equal(t, value, res)
})
t.Run("scan error", func(t *testing.T) {
t.Parallel()
// test declarations
req := sqltemplateMocks.NewWithResults[int64](t)
sc := scannerFunc(func(dest ...any) error {
return errTest
})
// setup expectations
req.EXPECT().GetScanDest().Return(nil).Once()
// execute and assert
res, err := scanRow(sc, req)
require.Zero(t, res)
require.Error(t, err)
require.ErrorIs(t, err, errTest)
})
t.Run("results error", func(t *testing.T) {
t.Parallel()
// test declarations
req := sqltemplateMocks.NewWithResults[int64](t)
sc := scannerFunc(func(dest ...any) error {
return nil
})
// setup expectations
req.EXPECT().GetScanDest().Return(nil).Once()
req.EXPECT().Results().Return(0, errTest).Once()
// execute and assert
res, err := scanRow(sc, req)
require.Zero(t, res)
require.Error(t, err)
require.ErrorIs(t, err, errTest)
})
}
func TestExec(t *testing.T) {
t.Parallel()
t.Run("happy path", func(t *testing.T) {
t.Parallel()
// test declarations
ctx := testutil.NewDefaultTestContext(t)
req := sqltemplateMocks.NewSQLTemplateIface(t)
rdb := test.NewDBProviderNopSQL(t)
// setup expectations
req.EXPECT().Validate().Return(nil).Once()
req.EXPECT().GetArgs().Return(nil).Once()
rdb.SQLMock.ExpectExec("").WillReturnResult(sqlmock.NewResult(0, 0))
// execute and assert
res, err := Exec(ctx, rdb.DB, validTestTmpl, req)
require.NoError(t, err)
require.NotZero(t, res)
})
t.Run("invalid request", func(t *testing.T) {
t.Parallel()
// test declarations
ctx := testutil.NewDefaultTestContext(t)
req := sqltemplateMocks.NewSQLTemplateIface(t)
rdb := test.NewDBProviderNopSQL(t)
// setup expectations
req.EXPECT().Validate().Return(errTest).Once()
// execute and assert
res, err := Exec(ctx, rdb.DB, invalidTestTmpl, req)
require.Zero(t, res)
require.Error(t, err)
require.ErrorContains(t, err, "invalid request")
})
t.Run("error executing template", func(t *testing.T) {
t.Parallel()
// test declarations
ctx := testutil.NewDefaultTestContext(t)
req := sqltemplateMocks.NewSQLTemplateIface(t)
rdb := test.NewDBProviderNopSQL(t)
// setup expectations
req.EXPECT().Validate().Return(nil).Once()
// execute and assert
res, err := Exec(ctx, rdb.DB, invalidTestTmpl, req)
require.Zero(t, res)
require.Error(t, err)
require.ErrorContains(t, err, "execute template")
})
t.Run("error executing SQL", func(t *testing.T) {
t.Parallel()
// test declarations
ctx := testutil.NewDefaultTestContext(t)
req := sqltemplateMocks.NewSQLTemplateIface(t)
rdb := test.NewDBProviderNopSQL(t)
// setup expectations
req.EXPECT().Validate().Return(nil).Once()
req.EXPECT().GetArgs().Return(nil)
rdb.SQLMock.ExpectExec("").WillReturnError(errTest)
// execute and assert
res, err := Exec(ctx, rdb.DB, validTestTmpl, req)
require.Zero(t, res)
require.Error(t, err)
require.ErrorAs(t, err, new(SQLError))
})
}