SQL data sources: Convert to return data frames (#32257)

Convert SQL data sources to return data frames. Co-authored-by: Marcus Efraimsson <marcus.efraimsson@gmail.com> Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com> Co-authored-by: Will Browne <will.browne@grafana.com> Co-authored-by: Hugo Häggmark <hugo.haggmark@gmail.com>
2021-05-05 16:46:07 +02:00
parent 06c24476dc
commit bd66c8dde3
36 changed files with 4947 additions and 3711 deletions
@@ -1,15 +1,17 @@
 package mysql

 import (
-	"database/sql"
 	"errors"
 	"fmt"
 	"net/url"
 	"reflect"
 	"strconv"
 	"strings"
+	"time"

 	"github.com/VividCortex/mysqlerr"
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/grafana/grafana-plugin-sdk-go/data/sqlutil"
 	"github.com/grafana/grafana/pkg/setting"

 	"github.com/go-sql-driver/mysql"
@@ -17,7 +19,12 @@ import (
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins"
 	"github.com/grafana/grafana/pkg/tsdb/sqleng"
-	"xorm.io/core"
+)
+
+const (
+	dateFormat      = "2006-01-02"
+	dateTimeFormat1 = "2006-01-02 15:04:05"
+	dateTimeFormat2 = "2006-01-02T15:04:05Z"
 )

 func characterEscape(s string, escapeChar string) string {
@@ -77,66 +84,6 @@ type mysqlQueryResultTransformer struct {
 	log log.Logger
 }

-func (t *mysqlQueryResultTransformer) TransformQueryResult(columnTypes []*sql.ColumnType, rows *core.Rows) (
-	plugins.DataRowValues, error) {
-	values := make([]interface{}, len(columnTypes))
-
-	for i := range values {
-		scanType := columnTypes[i].ScanType()
-		values[i] = reflect.New(scanType).Interface()
-
-		if columnTypes[i].DatabaseTypeName() == "BIT" {
-			values[i] = new([]byte)
-		}
-	}
-
-	if err := rows.Scan(values...); err != nil {
-		return nil, err
-	}
-
-	for i := 0; i < len(columnTypes); i++ {
-		typeName := reflect.ValueOf(values[i]).Type().String()
-
-		switch typeName {
-		case "*sql.RawBytes":
-			values[i] = string(*values[i].(*sql.RawBytes))
-		case "*mysql.NullTime":
-			sqlTime := (*values[i].(*mysql.NullTime))
-			if sqlTime.Valid {
-				values[i] = sqlTime.Time
-			} else {
-				values[i] = nil
-			}
-		case "*sql.NullInt64":
-			nullInt64 := (*values[i].(*sql.NullInt64))
-			if nullInt64.Valid {
-				values[i] = nullInt64.Int64
-			} else {
-				values[i] = nil
-			}
-		case "*sql.NullFloat64":
-			nullFloat64 := (*values[i].(*sql.NullFloat64))
-			if nullFloat64.Valid {
-				values[i] = nullFloat64.Float64
-			} else {
-				values[i] = nil
-			}
-		}
-
-		if columnTypes[i].DatabaseTypeName() == "DECIMAL" {
-			f, err := strconv.ParseFloat(values[i].(string), 64)
-
-			if err == nil {
-				values[i] = f
-			} else {
-				values[i] = nil
-			}
-		}
-	}
-
-	return values, nil
-}
-
 func (t *mysqlQueryResultTransformer) TransformQueryError(err error) error {
 	var driverErr *mysql.MySQLError
 	if errors.As(err, &driverErr) {
@@ -151,3 +98,199 @@ func (t *mysqlQueryResultTransformer) TransformQueryError(err error) error {
 }

 var errQueryFailed = errors.New("query failed - please inspect Grafana server log for details")
+
+func (t *mysqlQueryResultTransformer) GetConverterList() []sqlutil.StringConverter {
+	// For the MySQL driver , we have these possible data types:
+	// https://www.w3schools.com/sql/sql_datatypes.asp#:~:text=In%20MySQL%20there%20are%20three,numeric%2C%20and%20date%20and%20time.
+	// Since by default, we convert all into String, we need only to handle the Numeric data types
+	return []sqlutil.StringConverter{
+		{
+			Name:           "handle DOUBLE",
+			InputScanKind:  reflect.Struct,
+			InputTypeName:  "DOUBLE",
+			ConversionFunc: func(in *string) (*string, error) { return in, nil },
+			Replacer: &sqlutil.StringFieldReplacer{
+				OutputFieldType: data.FieldTypeNullableFloat64,
+				ReplaceFunc: func(in *string) (interface{}, error) {
+					if in == nil {
+						return nil, nil
+					}
+					v, err := strconv.ParseFloat(*in, 64)
+					if err != nil {
+						return nil, err
+					}
+					return &v, nil
+				},
+			},
+		},
+		{
+			Name:           "handle BIGINT",
+			InputScanKind:  reflect.Struct,
+			InputTypeName:  "BIGINT",
+			ConversionFunc: func(in *string) (*string, error) { return in, nil },
+			Replacer: &sqlutil.StringFieldReplacer{
+				OutputFieldType: data.FieldTypeNullableInt64,
+				ReplaceFunc: func(in *string) (interface{}, error) {
+					if in == nil {
+						return nil, nil
+					}
+					v, err := strconv.ParseInt(*in, 10, 64)
+					if err != nil {
+						return nil, err
+					}
+					return &v, nil
+				},
+			},
+		},
+		{
+			Name:           "handle DECIMAL",
+			InputScanKind:  reflect.Slice,
+			InputTypeName:  "DECIMAL",
+			ConversionFunc: func(in *string) (*string, error) { return in, nil },
+			Replacer: &sqlutil.StringFieldReplacer{
+				OutputFieldType: data.FieldTypeNullableFloat64,
+				ReplaceFunc: func(in *string) (interface{}, error) {
+					if in == nil {
+						return nil, nil
+					}
+					v, err := strconv.ParseFloat(*in, 64)
+					if err != nil {
+						return nil, err
+					}
+					return &v, nil
+				},
+			},
+		},
+		{
+			Name:           "handle DATETIME",
+			InputScanKind:  reflect.Struct,
+			InputTypeName:  "DATETIME",
+			ConversionFunc: func(in *string) (*string, error) { return in, nil },
+			Replacer: &sqlutil.StringFieldReplacer{
+				OutputFieldType: data.FieldTypeNullableTime,
+				ReplaceFunc: func(in *string) (interface{}, error) {
+					if in == nil {
+						return nil, nil
+					}
+					v, err := time.Parse(dateTimeFormat1, *in)
+					if err == nil {
+						return &v, nil
+					}
+					v, err = time.Parse(dateTimeFormat2, *in)
+					if err == nil {
+						return &v, nil
+					}
+
+					return nil, err
+				},
+			},
+		},
+		{
+			Name:           "handle DATE",
+			InputScanKind:  reflect.Struct,
+			InputTypeName:  "DATE",
+			ConversionFunc: func(in *string) (*string, error) { return in, nil },
+			Replacer: &sqlutil.StringFieldReplacer{
+				OutputFieldType: data.FieldTypeNullableTime,
+				ReplaceFunc: func(in *string) (interface{}, error) {
+					if in == nil {
+						return nil, nil
+					}
+					v, err := time.Parse(dateFormat, *in)
+					if err == nil {
+						return &v, nil
+					}
+					v, err = time.Parse(dateTimeFormat1, *in)
+					if err == nil {
+						return &v, nil
+					}
+					v, err = time.Parse(dateTimeFormat2, *in)
+					if err == nil {
+						return &v, nil
+					}
+					return nil, err
+				},
+			},
+		},
+		{
+			Name:           "handle TIMESTAMP",
+			InputScanKind:  reflect.Struct,
+			InputTypeName:  "TIMESTAMP",
+			ConversionFunc: func(in *string) (*string, error) { return in, nil },
+			Replacer: &sqlutil.StringFieldReplacer{
+				OutputFieldType: data.FieldTypeNullableTime,
+				ReplaceFunc: func(in *string) (interface{}, error) {
+					if in == nil {
+						return nil, nil
+					}
+					v, err := time.Parse(dateTimeFormat1, *in)
+					if err == nil {
+						return &v, nil
+					}
+					v, err = time.Parse(dateTimeFormat2, *in)
+					if err == nil {
+						return &v, nil
+					}
+					return nil, err
+				},
+			},
+		},
+		{
+			Name:           "handle YEAR",
+			InputScanKind:  reflect.Struct,
+			InputTypeName:  "YEAR",
+			ConversionFunc: func(in *string) (*string, error) { return in, nil },
+			Replacer: &sqlutil.StringFieldReplacer{
+				OutputFieldType: data.FieldTypeNullableInt64,
+				ReplaceFunc: func(in *string) (interface{}, error) {
+					if in == nil {
+						return nil, nil
+					}
+					v, err := strconv.ParseInt(*in, 10, 64)
+					if err != nil {
+						return nil, err
+					}
+					return &v, nil
+				},
+			},
+		},
+		{
+			Name:           "handle INT",
+			InputScanKind:  reflect.Struct,
+			InputTypeName:  "INT",
+			ConversionFunc: func(in *string) (*string, error) { return in, nil },
+			Replacer: &sqlutil.StringFieldReplacer{
+				OutputFieldType: data.FieldTypeNullableInt64,
+				ReplaceFunc: func(in *string) (interface{}, error) {
+					if in == nil {
+						return nil, nil
+					}
+					v, err := strconv.ParseInt(*in, 10, 64)
+					if err != nil {
+						return nil, err
+					}
+					return &v, nil
+				},
+			},
+		},
+		{
+			Name:           "handle FLOAT",
+			InputScanKind:  reflect.Struct,
+			InputTypeName:  "FLOAT",
+			ConversionFunc: func(in *string) (*string, error) { return in, nil },
+			Replacer: &sqlutil.StringFieldReplacer{
+				OutputFieldType: data.FieldTypeNullableFloat64,
+				ReplaceFunc: func(in *string) (interface{}, error) {
+					if in == nil {
+						return nil, nil
+					}
+					v, err := strconv.ParseFloat(*in, 64)
+					if err != nil {
+						return nil, err
+					}
+					return &v, nil
+				},
+			},
+		},
+	}
+}