Spanner support enhancements (#101634)

* Adds ability to run integration tests against spanner (by using GRAFANA_TEST_DB=spanner env variable. SPANNER_DB variable then specifies database to use: spannertest, emulator or string like /projects/<project>/instances/<instance>/databases/<db>) * Adds feature to migration dialects to create database from a snapshot, instead of running individual migrations. * Adds first version of Spanner snapshot, prepared from "OSS" migrations. * Uses generated bit-reversed-positive values instead of auto_increment. (As an experiment)
2025-03-06 16:11:20 +01:00
parent 43f56c5ca1
commit 9e690bd620
18 changed files with 1477 additions and 86 deletions
@@ -3,6 +3,9 @@
 package xorm

 import (
+	"database/sql"
+	"fmt"
+	"strconv"
 	"strings"

 	_ "github.com/googleapis/go-sql-spanner"
@@ -131,11 +134,14 @@ func (s *spanner) IsReserved(name string) bool {
 	_, exists := spannerReservedKeywords[name]
 	return exists
 }
-func (s *spanner) AndStr() string            { return "AND" }
-func (s *spanner) OrStr() string             { return "OR" }
-func (s *spanner) EqStr() string             { return "=" }
-func (s *spanner) RollBackStr() string       { return "ROLL BACK" }
-func (s *spanner) AutoIncrStr() string       { return "" }    // Spanner does not support auto-increment
+func (s *spanner) AndStr() string      { return "AND" }
+func (s *spanner) OrStr() string       { return "OR" }
+func (s *spanner) EqStr() string       { return "=" }
+func (s *spanner) RollBackStr() string { return "ROLL BACK" }
+func (s *spanner) AutoIncrStr() string {
+	// Spanner does not support auto-increment, but supports unique generated IDs (not sequential!).
+	return "GENERATED BY DEFAULT AS IDENTITY (BIT_REVERSED_POSITIVE)"
+}
 func (s *spanner) SupportInsertMany() bool   { return false } // Needs manual transaction batching
 func (s *spanner) SupportEngine() bool       { return false } // No support for engine selection
 func (s *spanner) SupportCharset() bool      { return false } // ...or charsets
@@ -145,26 +151,43 @@ func (s *spanner) ShowCreateNull() bool      { return false }
 func (s *spanner) Quote(name string) string  { return "`" + name + "`" }
 func (s *spanner) SqlType(col *core.Column) string {
 	switch col.SQLType.Name {
-	case core.Int, core.BigInt:
+	case core.Int, core.SmallInt, core.BigInt:
 		return "INT64"
-	case core.Varchar, core.Text:
+	case core.Varchar, core.Text, core.MediumText, core.LongText, core.Char, core.NVarchar, core.NChar, core.NText:
+		l := col.Length
+		if l == 0 {
+			l = col.SQLType.DefaultLength
+		}
+		if l > 0 {
+			return fmt.Sprintf("STRING(%d)", l)
+		}
 		return "STRING(MAX)"
-	case core.Bool:
+	case core.Bool, core.TinyInt:
 		return "BOOL"
 	case core.Float, core.Double:
 		return "FLOAT64"
-	case core.Bytea:
+	case core.Bytea, core.Blob, core.MediumBlob, core.LongBlob:
+		l := col.Length
+		if l == 0 {
+			l = col.SQLType.DefaultLength
+		}
+		if l > 0 {
+			return fmt.Sprintf("BYTES(%d)", l)
+		}
 		return "BYTES(MAX)"
 	case core.DateTime, core.TimeStamp:
 		return "TIMESTAMP"
 	default:
-		return "STRING(MAX)" // XXX: more types to add
+		panic("unknown column type: " + col.SQLType.Name)
+		//default:
+		//	return "STRING(MAX)" // XXX: more types to add
 	}
 }

 func (s *spanner) GetColumns(tableName string) ([]string, map[string]*core.Column, error) {
-	query := `SELECT COLUMN_NAME, SPANNER_TYPE, IS_NULLABLE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = @tableName`
-	rows, err := s.DB().Query(query, map[string]any{"tableName": tableName})
+	query := `SELECT COLUMN_NAME, SPANNER_TYPE, IS_NULLABLE, IS_IDENTITY, IDENTITY_GENERATION, IDENTITY_KIND, COLUMN_DEFAULT
+				FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = ? AND TABLE_SCHEMA="" ORDER BY ORDINAL_POSITION`
+	rows, err := s.DB().Query(query, tableName)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -174,21 +197,68 @@ func (s *spanner) GetColumns(tableName string) ([]string, map[string]*core.Colum
 	var colNames []string

 	var name, sqlType, isNullable string
+	var isIdentity, identityGeneration, identityKind, columnDefault sql.NullString
 	for rows.Next() {
-		if err := rows.Scan(&name, &sqlType, &isNullable); err != nil {
+		if err := rows.Scan(&name, &sqlType, &isNullable, &isIdentity, &identityGeneration, &identityKind, &columnDefault); err != nil {
 			return nil, nil, err
 		}

+		var length int
+		switch {
+		case sqlType == "INT64":
+			sqlType = core.Int
+		case sqlType == "FLOAT32" || sqlType == "FLOAT64":
+			sqlType = core.Float
+		case sqlType == "BOOL":
+			sqlType = core.Bool
+		case sqlType == "BYTES(MAX)":
+			sqlType = core.Blob
+		case sqlType == "STRING(MAX)":
+			sqlType = core.NVarchar
+		case sqlType == "TIMESTAMP":
+			sqlType = core.DateTime
+		case strings.HasPrefix(sqlType, "BYTES("):
+			// 6 == len(`BYTES(`), we also remove ")" from the end.
+			if l, err := strconv.Atoi(sqlType[6 : len(sqlType)-1]); err == nil {
+				length = l
+			}
+			sqlType = core.Blob
+		case strings.HasPrefix(sqlType, "STRING("):
+			// 7 == len(`STRING(`), we also remove ")" from the end.
+			if l, err := strconv.Atoi(sqlType[7 : len(sqlType)-1]); err == nil {
+				length = l
+			}
+			sqlType = core.Varchar
+		default:
+			panic("unknown column type: " + sqlType)
+		}
+
+		autoincrement := isIdentity.Valid && isIdentity.String == "YES" &&
+			identityGeneration.Valid && identityGeneration.String == "BY DEFAULT" &&
+			identityKind.Valid && identityKind.String == "BIT_REVERSED_POSITIVE_SEQUENCE"
+
+		defValue := ""
+		defEmpty := true
+		if columnDefault.Valid {
+			defValue = columnDefault.String
+			defEmpty = false
+		}
+
 		col := &core.Column{
-			Name:     name,
-			SQLType:  core.SQLType{Name: sqlType},
-			Nullable: isNullable == "YES",
+			Name:            name,
+			SQLType:         core.SQLType{Name: sqlType},
+			Length:          length,
+			Nullable:        isNullable == "YES",
+			IsAutoIncrement: autoincrement,
+			Indexes:         map[string]int{},
+			Default:         defValue,
+			DefaultIsEmpty:  defEmpty,
 		}
 		columns[name] = col
 		colNames = append(colNames, name)
 	}

-	return colNames, columns, nil
+	return colNames, columns, rows.Err()
 }

 func (s *spanner) CreateTableSql(table *core.Table, tableName, _, charset string) string {
@@ -198,9 +268,16 @@ func (s *spanner) CreateTableSql(table *core.Table, tableName, _, charset string
 		if i > 0 {
 			sql += ", "
 		}
+
 		sql += s.Quote(col.Name) + " " + s.SqlType(col)
-		if col.IsPrimaryKey {
-			sql += " PRIMARY KEY"
+		if !col.Nullable {
+			sql += " NOT NULL"
+		}
+		if col.Default != "" {
+			sql += " DEFAULT (" + col.Default + ")"
+		}
+		if col.IsAutoIncrement {
+			sql += " GENERATED BY DEFAULT AS IDENTITY (BIT_REVERSED_POSITIVE)"
 		}
 	}

@@ -230,10 +307,7 @@ func (s *spanner) TableCheckSql(tableName string) (string, []any) {
 }

 func (s *spanner) GetTables() ([]*core.Table, error) {
-	res, err := s.DB().Query(`
-    SELECT table_name FROM information_schema.tables
-    WHERE table_schema = ""
-	`)
+	res, err := s.DB().Query(`SELECT table_name FROM information_schema.tables WHERE table_schema = ""`)
 	if err != nil {
 		return nil, err
 	}
@@ -249,36 +323,50 @@ func (s *spanner) GetTables() ([]*core.Table, error) {
 		t.Name = name
 		tables = append(tables, t)
 	}
-	return tables, nil
+	return tables, res.Err()
 }

 func (s *spanner) GetIndexes(tableName string) (map[string]*core.Index, error) {
-	res, err := s.DB().Query(`
-		SELECT index_name, index_type, is_unique FROM information_schema.tables
-		WHERE table_name = ? AND table_schema = ""
-  `, []any{tableName})
+	res, err := s.DB().Query(`SELECT ix.INDEX_NAME, ix.INDEX_TYPE, ix.IS_UNIQUE, c.COLUMN_NAME
+		FROM INFORMATION_SCHEMA.INDEXES ix
+		JOIN INFORMATION_SCHEMA.INDEX_COLUMNS c ON (ix.TABLE_NAME=c.TABLE_NAME AND ix.INDEX_NAME=c.INDEX_NAME)
+		WHERE ix.TABLE_SCHEMA = "" AND ix.TABLE_NAME=?
+		ORDER BY ix.INDEX_NAME, c.ORDINAL_POSITION`, tableName)
 	if err != nil {
 		return nil, err
 	}
 	defer res.Close()

-	indices := map[string]*core.Index{}
+	indexes := map[string]*core.Index{}
+	var ixName, ixType, colName string
+	var isUnique bool
 	for res.Next() {
-		index := struct {
-			Name     string `xorm:"index_name"`
-			Type     string `xorm:"index_type"`
-			IsUnqiue bool   `xorm:"is_unique"`
-		}{}
-		err := res.Scan(&index)
+		err := res.Scan(&ixName, &ixType, &isUnique, &colName)
 		if err != nil {
 			return nil, err
 		}
-		switch {
-		case index.Type == "INDEX":
-			indices[index.Name] = core.NewIndex(index.Name, core.IndexType)
-		case index.Type == "PRIMARY_KEY", index.IsUnqiue:
-			indices[index.Name] = core.NewIndex(index.Name, core.UniqueType)
+
+		isRegular := false
+		if strings.HasPrefix(ixName, "IDX_"+tableName) || strings.HasPrefix(ixName, "UQE_"+tableName) {
+			ixName = ixName[5+len(tableName):]
+			isRegular = true
 		}
+
+		var index *core.Index
+		var ok bool
+		if index, ok = indexes[ixName]; !ok {
+			t := core.IndexType // ixType == "INDEX" && !isUnique
+			if ixType == "PRIMARY KEY" || isUnique {
+				t = core.UniqueType
+			}
+
+			index = &core.Index{}
+			index.IsRegular = isRegular
+			index.Type = t
+			index.Name = ixName
+			indexes[ixName] = index
+		}
+		index.AddColumn(colName)
 	}
-	return indices, nil
+	return indexes, res.Err()
 }