public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SQL: Migrate to use SDK contracts (#36635)

Browse Source 
* convert SQLs to use sdk contracts

* make draft

* postgres

* intermedia

* get datasourceinfo filled at the beginning of the service

* move the interval into package because of cyclict  import and fix all postgres tests

* fix mysql test

* fix mssql

* fix the test for pr https://github.com/grafana/grafana/issues/35839

* fix some issue about intervalv2 package

* update sql test

* wire migration for SQLs

* add sqls to the background process

* make it register instead of register and start

* revert formatting

* fix tests

* fix linter

* remove integration test

* Postgres test fix

Co-authored-by: Marcus Efraimsson <marcus.efraimsson@gmail.com>
This commit is contained in:
ying-jeanne
2021-09-07 09:35:37 +02:00
committed by GitHub
co-authored by Marcus Efraimsson
parent 51aeb717a4
commit c989f0fbbe
40 changed files with 1632 additions and 1495 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/tsdb/mssql/macros.go
+10 -10
View File
@@ -6,8 +6,8 @@ import (
"strings"
"time"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/components/gtime"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/tsdb/sqleng"
)
@@ -22,7 +22,7 @@ func newMssqlMacroEngine() sqleng.SQLMacroEngine {
return &msSQLMacroEngine{SQLMacroEngineBase: sqleng.NewSQLMacroEngineBase()}
}
func (m *msSQLMacroEngine) Interpolate(query plugins.DataSubQuery, timeRange plugins.DataTimeRange,
func (m *msSQLMacroEngine) Interpolate(query *backend.DataQuery, timeRange backend.TimeRange,
sql string) (string, error) {
// TODO: Return any error
rExp, _ := regexp.Compile(sExpr)
@@ -48,7 +48,7 @@ func (m *msSQLMacroEngine) Interpolate(query plugins.DataSubQuery, timeRange plu
return sql, nil
}
func (m *msSQLMacroEngine) evaluateMacro(timeRange plugins.DataTimeRange, query plugins.DataSubQuery, name string, args []string) (string, error) {
func (m *msSQLMacroEngine) evaluateMacro(timeRange backend.TimeRange, query *backend.DataQuery, name string, args []string) (string, error) {
switch name {
case "__time":
if len(args) == 0 {
@@ -65,11 +65,11 @@ func (m *msSQLMacroEngine) evaluateMacro(timeRange plugins.DataTimeRange, query
return "", fmt.Errorf("missing time column argument for macro %v", name)
}
return fmt.Sprintf("%s BETWEEN '%s' AND '%s'", args[0], timeRange.GetFromAsTimeUTC().Format(time.RFC3339), timeRange.GetToAsTimeUTC().Format(time.RFC3339)), nil
return fmt.Sprintf("%s BETWEEN '%s' AND '%s'", args[0], timeRange.From.UTC().Format(time.RFC3339), timeRange.To.UTC().Format(time.RFC3339)), nil
case "__timeFrom":
return fmt.Sprintf("'%s'", timeRange.GetFromAsTimeUTC().Format(time.RFC3339)), nil
return fmt.Sprintf("'%s'", timeRange.From.UTC().Format(time.RFC3339)), nil
case "__timeTo":
return fmt.Sprintf("'%s'", timeRange.GetToAsTimeUTC().Format(time.RFC3339)), nil
return fmt.Sprintf("'%s'", timeRange.To.UTC().Format(time.RFC3339)), nil
case "__timeGroup":
if len(args) < 2 {
return "", fmt.Errorf("macro %v needs time column and interval", name)
@@ -95,16 +95,16 @@ func (m *msSQLMacroEngine) evaluateMacro(timeRange plugins.DataTimeRange, query
if len(args) == 0 {
return "", fmt.Errorf("missing time column argument for macro %v", name)
}
return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], timeRange.GetFromAsSecondsEpoch(), args[0], timeRange.GetToAsSecondsEpoch()), nil
return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], timeRange.From.UTC().Unix(), args[0], timeRange.To.UTC().Unix()), nil
case "__unixEpochNanoFilter":
if len(args) == 0 {
return "", fmt.Errorf("missing time column argument for macro %v", name)
}
return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], timeRange.GetFromAsTimeUTC().UnixNano(), args[0], timeRange.GetToAsTimeUTC().UnixNano()), nil
return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], timeRange.From.UTC().UnixNano(), args[0], timeRange.To.UTC().UnixNano()), nil
case "__unixEpochNanoFrom":
return fmt.Sprintf("%d", timeRange.GetFromAsTimeUTC().UnixNano()), nil
return fmt.Sprintf("%d", timeRange.From.UTC().UnixNano()), nil
case "__unixEpochNanoTo":
return fmt.Sprintf("%d", timeRange.GetToAsTimeUTC().UnixNano()), nil
return fmt.Sprintf("%d", timeRange.To.UTC().UnixNano()), nil
case "__unixEpochGroup":
if len(args) < 2 {
return "", fmt.Errorf("macro %v needs time column and interval and optional fill value", name)
pkg/tsdb/mssql/macros_test.go
+34 -46
View File
@@ -2,14 +2,12 @@ package mssql
import (
"fmt"
"strconv"
"sync"
"testing"
"time"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana-plugin-sdk-go/backend"
. "github.com/smartystreets/goconvey/convey"
"github.com/stretchr/testify/require"
)
@@ -17,16 +15,16 @@ import (
func TestMacroEngine(t *testing.T) {
Convey("MacroEngine", t, func() {
engine := &msSQLMacroEngine{}
query := plugins.DataSubQuery{
Model: simplejson.New(),
query := &backend.DataQuery{
JSON: []byte("{}"),
}
dfltTimeRange := plugins.DataTimeRange{}
dfltTimeRange := backend.TimeRange{}
Convey("Given a time range between 2018-04-12 00:00 and 2018-04-12 00:05", func() {
from := time.Date(2018, 4, 12, 18, 0, 0, 0, time.UTC)
to := from.Add(5 * time.Minute)
timeRange := plugins.DataTimeRange{From: "5m", Now: to, To: "now"}
timeRange := backend.TimeRange{From: from, To: to}
Convey("interpolate __time function", func() {
sql, err := engine.Interpolate(query, dfltTimeRange, "select $__time(time_column)")
@@ -92,41 +90,26 @@ func TestMacroEngine(t *testing.T) {
Convey("interpolate __timeGroup function with fill (value = NULL)", func() {
_, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column,'5m', NULL)")
fill := query.Model.Get("fill").MustBool()
fillMode := query.Model.Get("fillMode").MustString()
fillInterval := query.Model.Get("fillInterval").MustInt()
So(err, ShouldBeNil)
So(fill, ShouldBeTrue)
So(fillMode, ShouldEqual, "null")
So(fillInterval, ShouldEqual, 5*time.Minute.Seconds())
queryJson, err := query.JSON.MarshalJSON()
So(err, ShouldBeNil)
So(string(queryJson), ShouldEqual, `{"fill":true,"fillInterval":300,"fillMode":"null"}`)
})
Convey("interpolate __timeGroup function with fill (value = previous)", func() {
_, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column,'5m', previous)")
fill := query.Model.Get("fill").MustBool()
fillMode := query.Model.Get("fillMode").MustString()
fillInterval := query.Model.Get("fillInterval").MustInt()
So(err, ShouldBeNil)
So(fill, ShouldBeTrue)
So(fillMode, ShouldEqual, "previous")
So(fillInterval, ShouldEqual, 5*time.Minute.Seconds())
queryJson, err := query.JSON.MarshalJSON()
So(err, ShouldBeNil)
So(string(queryJson), ShouldEqual, `{"fill":true,"fillInterval":300,"fillMode":"previous"}`)
})
Convey("interpolate __timeGroup function with fill (value = float)", func() {
_, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column,'5m', 1.5)")
fill := query.Model.Get("fill").MustBool()
fillValue := query.Model.Get("fillValue").MustFloat64()
fillInterval := query.Model.Get("fillInterval").MustInt()
So(err, ShouldBeNil)
So(fill, ShouldBeTrue)
So(fillValue, ShouldEqual, 1.5)
So(fillInterval, ShouldEqual, 5*time.Minute.Seconds())
queryJson, err := query.JSON.MarshalJSON()
So(err, ShouldBeNil)
So(string(queryJson), ShouldEqual, `{"fill":true,"fillInterval":300,"fillMode":"value","fillValue":1.5}`)
})
Convey("interpolate __unixEpochFilter function", func() {
@@ -170,9 +153,10 @@ func TestMacroEngine(t *testing.T) {
Convey("Given a time range between 1960-02-01 07:00 and 1965-02-03 08:00", func() {
from := time.Date(1960, 2, 1, 7, 0, 0, 0, time.UTC)
to := time.Date(1965, 2, 3, 8, 0, 0, 0, time.UTC)
timeRange := plugins.NewDataTimeRange(
strconv.FormatInt(from.UnixNano()/int64(time.Millisecond), 10),
strconv.FormatInt(to.UnixNano()/int64(time.Millisecond), 10))
timeRange := backend.TimeRange{
From: from,
To: to,
}
Convey("interpolate __timeFilter function", func() {
sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)")
@@ -199,9 +183,10 @@ func TestMacroEngine(t *testing.T) {
Convey("Given a time range between 1960-02-01 07:00 and 1980-02-03 08:00", func() {
from := time.Date(1960, 2, 1, 7, 0, 0, 0, time.UTC)
to := time.Date(1980, 2, 3, 8, 0, 0, 0, time.UTC)
timeRange := plugins.NewDataTimeRange(
strconv.FormatInt(from.UnixNano()/int64(time.Millisecond), 10),
strconv.FormatInt(to.UnixNano()/int64(time.Millisecond), 10))
timeRange := backend.TimeRange{
From: from,
To: to,
}
Convey("interpolate __timeFilter function", func() {
sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)")
@@ -229,27 +214,30 @@ func TestMacroEngine(t *testing.T) {
func TestMacroEngineConcurrency(t *testing.T) {
engine := newMssqlMacroEngine()
query1 := plugins.DataSubQuery{
Model: simplejson.New(),
query1 := backend.DataQuery{
JSON: []byte{},
}
query2 := plugins.DataSubQuery{
Model: simplejson.New(),
query2 := backend.DataQuery{
JSON: []byte{},
}
from := time.Date(2018, 4, 12, 18, 0, 0, 0, time.UTC)
to := from.Add(5 * time.Minute)
timeRange := plugins.DataTimeRange{From: "5m", To: "now", Now: to}
timeRange := backend.TimeRange{
From: from,
To: to,
}
var wg sync.WaitGroup
wg.Add(2)
go func(query plugins.DataSubQuery) {
go func(query backend.DataQuery) {
defer wg.Done()
_, err := engine.Interpolate(query, timeRange, "SELECT $__timeGroup(time_column,'5m')")
_, err := engine.Interpolate(&query, timeRange, "SELECT $__timeGroup(time_column,'5m')")
require.NoError(t, err)
}(query1)
go func(query plugins.DataSubQuery) {
_, err := engine.Interpolate(query, timeRange, "SELECT $__timeGroup(time_column,'5m')")
go func(query backend.DataQuery) {
_, err := engine.Interpolate(&query, timeRange, "SELECT $__timeGroup(time_column,'5m')")
require.NoError(t, err)
defer wg.Done()
}(query2)
pkg/tsdb/mssql/mssql.go
+94 -28
View File
@@ -1,6 +1,8 @@
package mssql
import (
"context"
"encoding/json"
"fmt"
"net/url"
"reflect"
@@ -8,43 +10,103 @@ import (
"strconv"
"strings"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/backend/datasource"
"github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana-plugin-sdk-go/data/sqlutil"
"github.com/grafana/grafana/pkg/plugins/backendplugin"
"github.com/grafana/grafana/pkg/plugins/backendplugin/coreplugin"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util"
mssql "github.com/denisenkom/go-mssqldb"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/tsdb/sqleng"
)
var logger = log.New("tsdb.mssql")
//nolint: staticcheck // plugins.DataPlugin deprecated
func NewExecutor(datasource *models.DataSource) (plugins.DataPlugin, error) {
cnnstr, err := generateConnectionString(datasource)
type Service struct {
im instancemgmt.InstanceManager
}
func ProvideService(manager backendplugin.Manager) (*Service, error) {
s := &Service{
im: datasource.NewInstanceManager(newInstanceSettings()),
}
factory := coreplugin.New(backend.ServeOpts{
QueryDataHandler: s,
})
if err := manager.Register("mssql", factory); err != nil {
logger.Error("Failed to register plugin", "error", err)
}
return s, nil
}
func (s *Service) getDataSourceHandler(pluginCtx backend.PluginContext) (*sqleng.DataSourceHandler, error) {
i, err := s.im.Get(pluginCtx)
if err != nil {
return nil, err
}
// TODO: Don't use global
if setting.Env == setting.Dev {
logger.Debug("getEngine", "connection", cnnstr)
}
instance := i.(*sqleng.DataSourceHandler)
return instance, nil
}
config := sqleng.DataPluginConfiguration{
DriverName: "mssql",
ConnectionString: cnnstr,
Datasource: datasource,
MetricColumnTypes: []string{"VARCHAR", "CHAR", "NVARCHAR", "NCHAR"},
func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) {
dsHandler, err := s.getDataSourceHandler(req.PluginContext)
if err != nil {
return nil, err
}
return dsHandler.QueryData(ctx, req)
}
queryResultTransformer := mssqlQueryResultTransformer{
log: logger,
func newInstanceSettings() datasource.InstanceFactoryFunc {
return func(settings backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
jsonData := sqleng.JsonData{
MaxOpenConns: 0,
MaxIdleConns: 2,
ConnMaxLifetime: 14400,
Encrypt: "false",
}
err := json.Unmarshal(settings.JSONData, &jsonData)
if err != nil {
return nil, fmt.Errorf("error reading settings: %w", err)
}
dsInfo := sqleng.DataSourceInfo{
JsonData: jsonData,
URL: settings.URL,
User: settings.User,
Database: settings.Database,
ID: settings.ID,
Updated: settings.Updated,
UID: settings.UID,
DecryptedSecureJSONData: settings.DecryptedSecureJSONData,
}
cnnstr, err := generateConnectionString(dsInfo)
if err != nil {
return nil, err
}
// TODO: Don't use global
if setting.Env == setting.Dev {
logger.Debug("getEngine", "connection", cnnstr)
}
config := sqleng.DataPluginConfiguration{
DriverName: "mssql",
ConnectionString: cnnstr,
DSInfo: dsInfo,
MetricColumnTypes: []string{"VARCHAR", "CHAR", "NVARCHAR", "NCHAR"},
}
queryResultTransformer := mssqlQueryResultTransformer{
log: logger,
}
return sqleng.NewQueryDataHandler(config, &queryResultTransformer, newMssqlMacroEngine(), logger)
}
return sqleng.NewDataPlugin(config, &queryResultTransformer, newMssqlMacroEngine(), logger)
}
// ParseURL tries to parse an MSSQL URL string into a URL object.
@@ -68,11 +130,11 @@ func ParseURL(u string) (*url.URL, error) {
}, nil
}
func generateConnectionString(dataSource *models.DataSource) (string, error) {
func generateConnectionString(dsInfo sqleng.DataSourceInfo) (string, error) {
const dfltPort = "0"
var addr util.NetworkAddress
if dataSource.Url != "" {
u, err := ParseURL(dataSource.Url)
if dsInfo.URL != "" {
u, err := ParseURL(dsInfo.URL)
if err != nil {
return "", err
}
@@ -88,26 +150,30 @@ func generateConnectionString(dataSource *models.DataSource) (string, error) {
}
args := []interface{}{
"url", dataSource.Url, "host", addr.Host,
"url", dsInfo.URL, "host", addr.Host,
}
if addr.Port != "0" {
args = append(args, "port", addr.Port)
}
logger.Debug("Generating connection string", args...)
encrypt := dataSource.JsonData.Get("encrypt").MustString("false")
connStr := fmt.Sprintf("server=%s;database=%s;user id=%s;password=%s;",
addr.Host,
dataSource.Database,
dataSource.User,
dataSource.DecryptedPassword(),
dsInfo.Database,
dsInfo.User,
dsInfo.DecryptedSecureJSONData["password"],
)
// Port number 0 means to determine the port automatically, so we can let the driver choose
if addr.Port != "0" {
connStr += fmt.Sprintf("port=%s;", addr.Port)
}
if encrypt != "false" {
connStr += fmt.Sprintf("encrypt=%s;", encrypt)
if dsInfo.JsonData.Encrypt == "" {
dsInfo.JsonData.Encrypt = "false"
}
if dsInfo.JsonData.Encrypt != "false" {
connStr += fmt.Sprintf("encrypt=%s;", dsInfo.JsonData.Encrypt)
}
return connStr, nil
}
pkg/tsdb/mssql/mssql_test.go
+292 -332
View File
File diff suppressed because it is too large Load Diff