SQL: Migrate to use SDK contracts (#36635)

* convert SQLs to use sdk contracts

* make draft

* postgres

* intermedia

* get datasourceinfo filled at the beginning of the service

* move the interval into package because of cyclict  import and fix all postgres tests

* fix mysql test

* fix mssql

* fix the test for pr https://github.com/grafana/grafana/issues/35839

* fix some issue about intervalv2 package

* update sql test

* wire migration for SQLs

* add sqls to the background process

* make it register instead of register and start

* revert formatting

* fix tests

* fix linter

* remove integration test

* Postgres test fix

Co-authored-by: Marcus Efraimsson <marcus.efraimsson@gmail.com>

pkg/tsdb/mysql/macros.go

@@ -6,9 +6,9 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	"github.com/grafana/grafana/pkg/components/gtime"
 	"github.com/grafana/grafana/pkg/infra/log"
-	"github.com/grafana/grafana/pkg/plugins"
 	"github.com/grafana/grafana/pkg/tsdb/sqleng"
 )
 
@@ -26,7 +26,7 @@ func newMysqlMacroEngine(logger log.Logger) sqleng.SQLMacroEngine {
 	return &mySQLMacroEngine{SQLMacroEngineBase: sqleng.NewSQLMacroEngineBase(), logger: logger}
 }
 
-func (m *mySQLMacroEngine) Interpolate(query plugins.DataSubQuery, timeRange plugins.DataTimeRange, sql string) (string, error) {
+func (m *mySQLMacroEngine) Interpolate(query *backend.DataQuery, timeRange backend.TimeRange, sql string) (string, error) {
 	matches := restrictedRegExp.FindAllStringSubmatch(sql, 1)
 	if len(matches) > 0 {
 		m.logger.Error("show grants, session_user(), current_user(), system_user() or user() not allowed in query")
@@ -57,7 +57,7 @@ func (m *mySQLMacroEngine) Interpolate(query plugins.DataSubQuery, timeRange plu
 	return sql, nil
 }
 
-func (m *mySQLMacroEngine) evaluateMacro(timeRange plugins.DataTimeRange, query plugins.DataSubQuery, name string, args []string) (string, error) {
+func (m *mySQLMacroEngine) evaluateMacro(timeRange backend.TimeRange, query *backend.DataQuery, name string, args []string) (string, error) {
 	switch name {
 	case "__timeEpoch", "__time":
 		if len(args) == 0 {
@@ -69,11 +69,11 @@ func (m *mySQLMacroEngine) evaluateMacro(timeRange plugins.DataTimeRange, query
 			return "", fmt.Errorf("missing time column argument for macro %v", name)
 		}
 
-		return fmt.Sprintf("%s BETWEEN FROM_UNIXTIME(%d) AND FROM_UNIXTIME(%d)", args[0], timeRange.GetFromAsSecondsEpoch(), timeRange.GetToAsSecondsEpoch()), nil
+		return fmt.Sprintf("%s BETWEEN FROM_UNIXTIME(%d) AND FROM_UNIXTIME(%d)", args[0], timeRange.From.UTC().Unix(), timeRange.To.UTC().Unix()), nil
 	case "__timeFrom":
-		return fmt.Sprintf("FROM_UNIXTIME(%d)", timeRange.GetFromAsSecondsEpoch()), nil
+		return fmt.Sprintf("FROM_UNIXTIME(%d)", timeRange.From.UTC().Unix()), nil
 	case "__timeTo":
-		return fmt.Sprintf("FROM_UNIXTIME(%d)", timeRange.GetToAsSecondsEpoch()), nil
+		return fmt.Sprintf("FROM_UNIXTIME(%d)", timeRange.To.UTC().Unix()), nil
 	case "__timeGroup":
 		if len(args) < 2 {
 			return "", fmt.Errorf("macro %v needs time column and interval", name)
@@ -99,16 +99,16 @@ func (m *mySQLMacroEngine) evaluateMacro(timeRange plugins.DataTimeRange, query
 		if len(args) == 0 {
 			return "", fmt.Errorf("missing time column argument for macro %v", name)
 		}
-		return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], timeRange.GetFromAsSecondsEpoch(), args[0], timeRange.GetToAsSecondsEpoch()), nil
+		return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], timeRange.From.UTC().Unix(), args[0], timeRange.To.UTC().Unix()), nil
 	case "__unixEpochNanoFilter":
 		if len(args) == 0 {
 			return "", fmt.Errorf("missing time column argument for macro %v", name)
 		}
-		return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], timeRange.GetFromAsTimeUTC().UnixNano(), args[0], timeRange.GetToAsTimeUTC().UnixNano()), nil
+		return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], timeRange.From.UTC().UnixNano(), args[0], timeRange.To.UTC().UnixNano()), nil
 	case "__unixEpochNanoFrom":
-		return fmt.Sprintf("%d", timeRange.GetFromAsTimeUTC().UnixNano()), nil
+		return fmt.Sprintf("%d", timeRange.From.UTC().UnixNano()), nil
 	case "__unixEpochNanoTo":
-		return fmt.Sprintf("%d", timeRange.GetToAsTimeUTC().UnixNano()), nil
+		return fmt.Sprintf("%d", timeRange.To.UTC().UnixNano()), nil
 	case "__unixEpochGroup":
 		if len(args) < 2 {
 			return "", fmt.Errorf("macro %v needs time column and interval and optional fill value", name)

pkg/tsdb/mysql/macros_test.go

@@ -2,14 +2,12 @@ package mysql
 
 import (
 	"fmt"
-	"strconv"
 	"sync"
 	"testing"
 	"time"
 
-	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	"github.com/grafana/grafana/pkg/infra/log"
-	"github.com/grafana/grafana/pkg/plugins"
 	. "github.com/smartystreets/goconvey/convey"
 	"github.com/stretchr/testify/require"
 )
@@ -19,12 +17,12 @@ func TestMacroEngine(t *testing.T) {
 		engine := &mySQLMacroEngine{
 			logger: log.New("test"),
 		}
-		query := plugins.DataSubQuery{}
+		query := &backend.DataQuery{}
 
 		Convey("Given a time range between 2018-04-12 00:00 and 2018-04-12 00:05", func() {
 			from := time.Date(2018, 4, 12, 18, 0, 0, 0, time.UTC)
 			to := from.Add(5 * time.Minute)
-			timeRange := plugins.DataTimeRange{From: "5m", Now: to, To: "now"}
+			timeRange := backend.TimeRange{From: from, To: to}
 
 			Convey("interpolate __time function", func() {
 				sql, err := engine.Interpolate(query, timeRange, "select $__time(time_column)")
@@ -123,8 +121,10 @@ func TestMacroEngine(t *testing.T) {
 		Convey("Given a time range between 1960-02-01 07:00 and 1965-02-03 08:00", func() {
 			from := time.Date(1960, 2, 1, 7, 0, 0, 0, time.UTC)
 			to := time.Date(1965, 2, 3, 8, 0, 0, 0, time.UTC)
-			timeRange := plugins.NewDataTimeRange(
-				strconv.FormatInt(from.UnixNano()/int64(time.Millisecond), 10), strconv.FormatInt(to.UnixNano()/int64(time.Millisecond), 10))
+			timeRange := backend.TimeRange{
+				From: from,
+				To:   to,
+			}
 
 			Convey("interpolate __timeFilter function", func() {
 				sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)")
@@ -144,8 +144,10 @@ func TestMacroEngine(t *testing.T) {
 		Convey("Given a time range between 1960-02-01 07:00 and 1980-02-03 08:00", func() {
 			from := time.Date(1960, 2, 1, 7, 0, 0, 0, time.UTC)
 			to := time.Date(1980, 2, 3, 8, 0, 0, 0, time.UTC)
-			timeRange := plugins.NewDataTimeRange(
-				strconv.FormatInt(from.UnixNano()/int64(time.Millisecond), 10), strconv.FormatInt(to.UnixNano()/int64(time.Millisecond), 10))
+			timeRange := backend.TimeRange{
+				From: from,
+				To:   to,
+			}
 
 			Convey("interpolate __timeFilter function", func() {
 				sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)")
@@ -185,7 +187,7 @@ func TestMacroEngine(t *testing.T) {
 			}
 
 			for _, tc := range tcs {
-				_, err := engine.Interpolate(plugins.DataSubQuery{}, plugins.DataTimeRange{}, tc)
+				_, err := engine.Interpolate(&backend.DataQuery{}, backend.TimeRange{}, tc)
 				So(err.Error(), ShouldEqual, "invalid query - inspect Grafana server log for details")
 			}
 		})
@@ -194,27 +196,27 @@ func TestMacroEngine(t *testing.T) {
 
 func TestMacroEngineConcurrency(t *testing.T) {
 	engine := newMysqlMacroEngine(log.New("test"))
-	query1 := plugins.DataSubQuery{
-		Model: simplejson.New(),
+	query1 := backend.DataQuery{
+		JSON: []byte{},
 	}
-	query2 := plugins.DataSubQuery{
-		Model: simplejson.New(),
+	query2 := backend.DataQuery{
+		JSON: []byte{},
 	}
 	from := time.Date(2018, 4, 12, 18, 0, 0, 0, time.UTC)
 	to := from.Add(5 * time.Minute)
-	timeRange := plugins.DataTimeRange{From: "5m", To: "now", Now: to}
+	timeRange := backend.TimeRange{From: from, To: to}
 
 	var wg sync.WaitGroup
 	wg.Add(2)
 
-	go func(query plugins.DataSubQuery) {
+	go func(query backend.DataQuery) {
 		defer wg.Done()
-		_, err := engine.Interpolate(query, timeRange, "SELECT $__timeGroup(time_column,'5m')")
+		_, err := engine.Interpolate(&query, timeRange, "SELECT $__timeGroup(time_column,'5m')")
 		require.NoError(t, err)
 	}(query1)
 
-	go func(query plugins.DataSubQuery) {
-		_, err := engine.Interpolate(query, timeRange, "SELECT $__timeGroup(time_column,'5m')")
+	go func(query backend.DataQuery) {
+		_, err := engine.Interpolate(&query, timeRange, "SELECT $__timeGroup(time_column,'5m')")
 		require.NoError(t, err)
 		defer wg.Done()
 	}(query2)

pkg/tsdb/mysql/mysql.go

@@ -1,6 +1,8 @@
 package mysql
 
 import (
+	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/url"
@@ -10,15 +12,18 @@ import (
 	"time"
 
 	"github.com/VividCortex/mysqlerr"
+	"github.com/grafana/grafana-plugin-sdk-go/backend"
+	"github.com/grafana/grafana-plugin-sdk-go/backend/datasource"
+	"github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
 	"github.com/grafana/grafana-plugin-sdk-go/data/sqlutil"
 	"github.com/grafana/grafana/pkg/infra/httpclient"
+	"github.com/grafana/grafana/pkg/plugins/backendplugin"
+	"github.com/grafana/grafana/pkg/plugins/backendplugin/coreplugin"
 	"github.com/grafana/grafana/pkg/setting"
 
 	"github.com/go-sql-driver/mysql"
 	"github.com/grafana/grafana/pkg/infra/log"
-	"github.com/grafana/grafana/pkg/models"
-	"github.com/grafana/grafana/pkg/plugins"
 	"github.com/grafana/grafana/pkg/tsdb/sqleng"
 )
 
@@ -28,47 +33,88 @@ const (
 	dateTimeFormat2 = "2006-01-02T15:04:05Z"
 )
 
+var logger = log.New("tsdb.mysql")
+
+type Service struct {
+	Cfg *setting.Cfg
+	im  instancemgmt.InstanceManager
+}
+
 func characterEscape(s string, escapeChar string) string {
 	return strings.ReplaceAll(s, escapeChar, url.QueryEscape(escapeChar))
 }
 
-//nolint: staticcheck // plugins.DataPlugin deprecated
-func New(httpClientProvider httpclient.Provider) func(datasource *models.DataSource) (plugins.DataPlugin, error) {
-	//nolint: staticcheck // plugins.DataPlugin deprecated
-	return func(datasource *models.DataSource) (plugins.DataPlugin, error) {
-		logger := log.New("tsdb.mysql")
+func ProvideService(cfg *setting.Cfg, manager backendplugin.Manager, httpClientProvider httpclient.Provider) (*Service, error) {
+	s := &Service{
+		Cfg: cfg,
+		im:  datasource.NewInstanceManager(newInstanceSettings(httpClientProvider)),
+	}
+	factory := coreplugin.New(backend.ServeOpts{
+		QueryDataHandler: s,
+	})
+
+	if err := manager.Register("mysql", factory); err != nil {
+		logger.Error("Failed to register plugin", "error", err)
+	}
+	return s, nil
+}
+
+func newInstanceSettings(httpClientProvider httpclient.Provider) datasource.InstanceFactoryFunc {
+	return func(settings backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
+		jsonData := sqleng.JsonData{
+			MaxOpenConns:    0,
+			MaxIdleConns:    2,
+			ConnMaxLifetime: 14400,
+		}
+
+		err := json.Unmarshal(settings.JSONData, &jsonData)
+		if err != nil {
+			return nil, fmt.Errorf("error reading settings: %w", err)
+		}
+		dsInfo := sqleng.DataSourceInfo{
+			JsonData:                jsonData,
+			URL:                     settings.URL,
+			User:                    settings.User,
+			Database:                settings.Database,
+			ID:                      settings.ID,
+			Updated:                 settings.Updated,
+			UID:                     settings.UID,
+			DecryptedSecureJSONData: settings.DecryptedSecureJSONData,
+		}
 
 		protocol := "tcp"
-		if strings.HasPrefix(datasource.Url, "/") {
+		if strings.HasPrefix(dsInfo.URL, "/") {
 			protocol = "unix"
 		}
 
 		cnnstr := fmt.Sprintf("%s:%s@%s(%s)/%s?collation=utf8mb4_unicode_ci&parseTime=true&loc=UTC&allowNativePasswords=true",
-			characterEscape(datasource.User, ":"),
-			datasource.DecryptedPassword(),
+			characterEscape(dsInfo.User, ":"),
+			dsInfo.DecryptedSecureJSONData["password"],
 			protocol,
-			characterEscape(datasource.Url, ")"),
-			characterEscape(datasource.Database, "?"),
+			characterEscape(dsInfo.URL, ")"),
+			characterEscape(dsInfo.Database, "?"),
 		)
 
-		tlsConfig, err := datasource.GetTLSConfig(httpClientProvider)
+		opts, err := settings.HTTPClientOptions()
+		if err != nil {
+			return nil, err
+		}
+
+		tlsConfig, err := httpClientProvider.GetTLSConfig(opts)
 		if err != nil {
 			return nil, err
 		}
 
 		if tlsConfig.RootCAs != nil || len(tlsConfig.Certificates) > 0 {
-			tlsConfigString := fmt.Sprintf("ds%d", datasource.Id)
+			tlsConfigString := fmt.Sprintf("ds%d", settings.ID)
 			if err := mysql.RegisterTLSConfig(tlsConfigString, tlsConfig); err != nil {
 				return nil, err
 			}
 			cnnstr += "&tls=" + tlsConfigString
 		}
 
-		if datasource.JsonData != nil {
-			timezone, hasTimezone := datasource.JsonData.CheckGet("timezone")
-			if hasTimezone && timezone.MustString() != "" {
-				cnnstr += fmt.Sprintf("&time_zone='%s'", url.QueryEscape(timezone.MustString()))
-			}
+		if dsInfo.JsonData.Timezone != "" {
+			cnnstr += fmt.Sprintf("&time_zone='%s'", url.QueryEscape(dsInfo.JsonData.Timezone))
 		}
 
 		if setting.Env == setting.Dev {
@@ -78,7 +124,7 @@ func New(httpClientProvider httpclient.Provider) func(datasource *models.DataSou
 		config := sqleng.DataPluginConfiguration{
 			DriverName:        "mysql",
 			ConnectionString:  cnnstr,
-			Datasource:        datasource,
+			DSInfo:            dsInfo,
 			TimeColumnNames:   []string{"time", "time_sec"},
 			MetricColumnTypes: []string{"CHAR", "VARCHAR", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT"},
 		}
@@ -87,10 +133,27 @@ func New(httpClientProvider httpclient.Provider) func(datasource *models.DataSou
 			log: logger,
 		}
 
-		return sqleng.NewDataPlugin(config, &rowTransformer, newMysqlMacroEngine(logger), logger)
+		return sqleng.NewQueryDataHandler(config, &rowTransformer, newMysqlMacroEngine(logger), logger)
 	}
 }
 
+func (s *Service) getDataSourceHandler(pluginCtx backend.PluginContext) (*sqleng.DataSourceHandler, error) {
+	i, err := s.im.Get(pluginCtx)
+	if err != nil {
+		return nil, err
+	}
+	instance := i.(*sqleng.DataSourceHandler)
+	return instance, nil
+}
+
+func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) {
+	dsHandler, err := s.getDataSourceHandler(req.PluginContext)
+	if err != nil {
+		return nil, err
+	}
+	return dsHandler.QueryData(ctx, req)
+}
+
 type mysqlQueryResultTransformer struct {
 	log log.Logger
 }