public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Postgres Data Source (#9475)

Browse Source 
* add postgresql datasource

* add rest of files for postgres datasource

* fix timeseries query, remove unused code

* consistent naming, refactoring

* s/mysql/postgres/

* s/mysql/postgres/

* couple more tests

* tests for more datatypes

* fix macros for postgres

* add __timeSec macro

* add frontend for postgres datasource

* adjust documentation

* fix formatting

* add proper plugin description

* merge editor changes from mysql

* port changes from mysql datasource

* set proper defaultQuery for postgres

* add time_sec to timeseries query
accept int for value for timeseries query

* revert allowing time_sec and handle int or float values as unix
timestamp for "time" column

* fix tslint error

* handle decimal values in timeseries query

* allow setting sslmode for postgres datasource

* use type switch for handling data types

* fix value for timeseries query

* refactor timeseries queries to make them more flexible

* remove debug statement from inner loop in type conversion

* use plain for loop in getTypedRowData

* fix timeseries queries

* adjust postgres datasource to tsdb refactoring

* adjust postgres datasource to frontend changes

* update lib/pq to latest version

* move type conversion to getTypedRowData

* handle address types cidr, inet and macaddr

* adjust response parser and docs for annotations

* convert unknown types to string

* add documentation for postgres datasource

* add another example query with metric column

* set more helpful default query

* update help text in query editor

* handle NULL in value column of timeseries query

* add __timeGroup macro

* add test for __timeGroup macro

* document __timeGroup and set proper default query for annotations

* fix typos in docs

* add postgres to list of datasources

* add postgres to builtInPlugins

* mysql: refactoring as prep for merging postgres

Refactors out the initialization of the xorm engine and the query logic
for an sql data source.

* mysql: rename refactoring + test update

* postgres:refactor to use SqlEngine(same as mysql)

Refactored to use a common base class with the MySql data source.

Other changes from the original PR:
- Changed time column to be time_sec to allow other time units in the
future and to be the same as MySQL
- Changed integration test to test the main Query method rather than
the private transformToTable method
- Changed the __timeSec macro name to __timeEpoch
- Renamed PostgresExecutor to PostgresQueryEndpoint

Fixes #9209 (the original PR)

* postgres: encrypt password on config page

With some other cosmetic changes to the config page:
- placeholder texts
- reset button for the password after it has been encrypted.
- default value for the sslmode field.

* postgres: change back col name to time from time_sec

* postgres mysql: remove annotation title

Title has been removed from annotations

* postgres: fix images for docs page

* postgres mysql: fix specs
This commit is contained in:
Daniel Lee
2017-10-10 15:19:14 +02:00
committed by GitHub
parent 630e6f5da6
commit d1c9760fa8
36 changed files with 2231 additions and 478 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/tsdb/mysql/macros.go
+8 -13
View File
@@ -11,26 +11,21 @@ import (
const rsIdentifier = `([_a-zA-Z0-9]+)`
const sExpr = `\$` + rsIdentifier + `\(([^\)]*)\)`
type SqlMacroEngine interface {
Interpolate(sql string) (string, error)
}
type MySqlMacroEngine struct {
TimeRange *tsdb.TimeRange
}
func NewMysqlMacroEngine(timeRange *tsdb.TimeRange) SqlMacroEngine {
return &MySqlMacroEngine{
TimeRange: timeRange,
}
func NewMysqlMacroEngine() tsdb.SqlMacroEngine {
return &MySqlMacroEngine{}
}
func (m *MySqlMacroEngine) Interpolate(sql string) (string, error) {
func (m *MySqlMacroEngine) Interpolate(timeRange *tsdb.TimeRange, sql string) (string, error) {
m.TimeRange = timeRange
rExp, _ := regexp.Compile(sExpr)
var macroError error
sql = ReplaceAllStringSubmatchFunc(rExp, sql, func(groups []string) string {
res, err := m.EvaluateMacro(groups[1], groups[2:])
sql = replaceAllStringSubmatchFunc(rExp, sql, func(groups []string) string {
res, err := m.evaluateMacro(groups[1], groups[2:])
if err != nil && macroError == nil {
macroError = err
return "macro_error()"
@@ -45,7 +40,7 @@ func (m *MySqlMacroEngine) Interpolate(sql string) (string, error) {
return sql, nil
}
func ReplaceAllStringSubmatchFunc(re *regexp.Regexp, str string, repl func([]string) string) string {
func replaceAllStringSubmatchFunc(re *regexp.Regexp, str string, repl func([]string) string) string {
result := ""
lastIndex := 0
@@ -62,7 +57,7 @@ func ReplaceAllStringSubmatchFunc(re *regexp.Regexp, str string, repl func([]str
return result + str[lastIndex:]
}
func (m *MySqlMacroEngine) EvaluateMacro(name string, args []string) (string, error) {
func (m *MySqlMacroEngine) evaluateMacro(name string, args []string) (string, error) {
switch name {
case "__time":
if len(args) == 0 {
pkg/tsdb/mysql/macros_test.go
+10 -36
View File
@@ -9,86 +9,60 @@ import (
func TestMacroEngine(t *testing.T) {
Convey("MacroEngine", t, func() {
engine := &MySqlMacroEngine{}
timeRange := &tsdb.TimeRange{From: "5m", To: "now"}
Convey("interpolate __time function", func() {
engine := &MySqlMacroEngine{}
sql, err := engine.Interpolate("select $__time(time_column)")
sql, err := engine.Interpolate(nil, "select $__time(time_column)")
So(err, ShouldBeNil)
So(sql, ShouldEqual, "select UNIX_TIMESTAMP(time_column) as time_sec")
})
Convey("interpolate __time function wrapped in aggregation", func() {
engine := &MySqlMacroEngine{}
sql, err := engine.Interpolate("select min($__time(time_column))")
sql, err := engine.Interpolate(nil, "select min($__time(time_column))")
So(err, ShouldBeNil)
So(sql, ShouldEqual, "select min(UNIX_TIMESTAMP(time_column) as time_sec)")
})
Convey("interpolate __timeFilter function", func() {
engine := &MySqlMacroEngine{
TimeRange: &tsdb.TimeRange{From: "5m", To: "now"},
}
sql, err := engine.Interpolate("WHERE $__timeFilter(time_column)")
sql, err := engine.Interpolate(timeRange, "WHERE $__timeFilter(time_column)")
So(err, ShouldBeNil)
So(sql, ShouldEqual, "WHERE time_column >= FROM_UNIXTIME(18446744066914186738) AND time_column <= FROM_UNIXTIME(18446744066914187038)")
})
Convey("interpolate __timeFrom function", func() {
engine := &MySqlMacroEngine{
TimeRange: &tsdb.TimeRange{From: "5m", To: "now"},
}
sql, err := engine.Interpolate("select $__timeFrom(time_column)")
sql, err := engine.Interpolate(timeRange, "select $__timeFrom(time_column)")
So(err, ShouldBeNil)
So(sql, ShouldEqual, "select FROM_UNIXTIME(18446744066914186738)")
})
Convey("interpolate __timeTo function", func() {
engine := &MySqlMacroEngine{
TimeRange: &tsdb.TimeRange{From: "5m", To: "now"},
}
sql, err := engine.Interpolate("select $__timeTo(time_column)")
sql, err := engine.Interpolate(timeRange, "select $__timeTo(time_column)")
So(err, ShouldBeNil)
So(sql, ShouldEqual, "select FROM_UNIXTIME(18446744066914187038)")
})
Convey("interpolate __unixEpochFilter function", func() {
engine := &MySqlMacroEngine{
TimeRange: &tsdb.TimeRange{From: "5m", To: "now"},
}
sql, err := engine.Interpolate("select $__unixEpochFilter(18446744066914186738)")
sql, err := engine.Interpolate(timeRange, "select $__unixEpochFilter(18446744066914186738)")
So(err, ShouldBeNil)
So(sql, ShouldEqual, "select 18446744066914186738 >= 18446744066914186738 AND 18446744066914186738 <= 18446744066914187038")
})
Convey("interpolate __unixEpochFrom function", func() {
engine := &MySqlMacroEngine{
TimeRange: &tsdb.TimeRange{From: "5m", To: "now"},
}
sql, err := engine.Interpolate("select $__unixEpochFrom()")
sql, err := engine.Interpolate(timeRange, "select $__unixEpochFrom()")
So(err, ShouldBeNil)
So(sql, ShouldEqual, "select 18446744066914186738")
})
Convey("interpolate __unixEpochTo function", func() {
engine := &MySqlMacroEngine{
TimeRange: &tsdb.TimeRange{From: "5m", To: "now"},
}
sql, err := engine.Interpolate("select $__unixEpochTo()")
sql, err := engine.Interpolate(timeRange, "select $__unixEpochTo()")
So(err, ShouldBeNil)
So(sql, ShouldEqual, "select 18446744066914187038")
pkg/tsdb/mysql/mysql.go
+25 -110
View File
@@ -6,142 +6,57 @@ import (
"database/sql"
"fmt"
"strconv"
"sync"
"time"
"github.com/go-sql-driver/mysql"
"github.com/go-xorm/core"
"github.com/go-xorm/xorm"
"github.com/grafana/grafana/pkg/components/null"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/tsdb"
)
type MysqlExecutor struct {
engine *xorm.Engine
log log.Logger
}
type engineCacheType struct {
cache map[int64]*xorm.Engine
versions map[int64]int
sync.Mutex
}
var engineCache = engineCacheType{
cache: make(map[int64]*xorm.Engine),
versions: make(map[int64]int),
type MysqlQueryEndpoint struct {
sqlEngine tsdb.SqlEngine
log log.Logger
}
func init() {
tsdb.RegisterTsdbQueryEndpoint("mysql", NewMysqlExecutor)
tsdb.RegisterTsdbQueryEndpoint("mysql", NewMysqlQueryEndpoint)
}
func NewMysqlExecutor(datasource *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
executor := &MysqlExecutor{
func NewMysqlQueryEndpoint(datasource *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
endpoint := &MysqlQueryEndpoint{
log: log.New("tsdb.mysql"),
}
err := executor.initEngine(datasource)
if err != nil {
return nil, err
}
return executor, nil
}
func (e *MysqlExecutor) initEngine(dsInfo *models.DataSource) error {
engineCache.Lock()
defer engineCache.Unlock()
if engine, present := engineCache.cache[dsInfo.Id]; present {
if version, _ := engineCache.versions[dsInfo.Id]; version == dsInfo.Version {
e.engine = engine
return nil
}
endpoint.sqlEngine = &tsdb.DefaultSqlEngine{
MacroEngine: NewMysqlMacroEngine(),
}
cnnstr := fmt.Sprintf("%s:%s@%s(%s)/%s?collation=utf8mb4_unicode_ci&parseTime=true&loc=UTC",
dsInfo.User,
dsInfo.Password,
datasource.User,
datasource.Password,
"tcp",
dsInfo.Url,
dsInfo.Database)
datasource.Url,
datasource.Database,
)
endpoint.log.Debug("getEngine", "connection", cnnstr)
e.log.Debug("getEngine", "connection", cnnstr)
engine, err := xorm.NewEngine("mysql", cnnstr)
engine.SetMaxOpenConns(10)
engine.SetMaxIdleConns(10)
if err != nil {
return err
if err := endpoint.sqlEngine.InitEngine("mysql", datasource, cnnstr); err != nil {
return nil, err
}
engineCache.cache[dsInfo.Id] = engine
e.engine = engine
return nil
return endpoint, nil
}
func (e *MysqlExecutor) Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) {
result := &tsdb.Response{
Results: make(map[string]*tsdb.QueryResult),
}
macroEngine := NewMysqlMacroEngine(tsdbQuery.TimeRange)
session := e.engine.NewSession()
defer session.Close()
db := session.DB()
for _, query := range tsdbQuery.Queries {
rawSql := query.Model.Get("rawSql").MustString()
if rawSql == "" {
continue
}
queryResult := &tsdb.QueryResult{Meta: simplejson.New(), RefId: query.RefId}
result.Results[query.RefId] = queryResult
rawSql, err := macroEngine.Interpolate(rawSql)
if err != nil {
queryResult.Error = err
continue
}
queryResult.Meta.Set("sql", rawSql)
rows, err := db.Query(rawSql)
if err != nil {
queryResult.Error = err
continue
}
defer rows.Close()
format := query.Model.Get("format").MustString("time_series")
switch format {
case "time_series":
err := e.TransformToTimeSeries(query, rows, queryResult)
if err != nil {
queryResult.Error = err
continue
}
case "table":
err := e.TransformToTable(query, rows, queryResult)
if err != nil {
queryResult.Error = err
continue
}
}
}
return result, nil
// Query is the main function for the MysqlExecutor
func (e *MysqlQueryEndpoint) Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) {
return e.sqlEngine.Query(ctx, dsInfo, tsdbQuery, e.transformToTimeSeries, e.transformToTable)
}
func (e MysqlExecutor) TransformToTable(query *tsdb.Query, rows *core.Rows, result *tsdb.QueryResult) error {
func (e MysqlQueryEndpoint) transformToTable(query *tsdb.Query, rows *core.Rows, result *tsdb.QueryResult) error {
columnNames, err := rows.Columns()
columnCount := len(columnNames)
@@ -166,7 +81,7 @@ func (e MysqlExecutor) TransformToTable(query *tsdb.Query, rows *core.Rows, resu
rowLimit := 1000000
rowCount := 0
for ; rows.Next(); rowCount += 1 {
for ; rows.Next(); rowCount++ {
if rowCount > rowLimit {
return fmt.Errorf("MySQL query row limit exceeded, limit %d", rowLimit)
}
@@ -184,7 +99,7 @@ func (e MysqlExecutor) TransformToTable(query *tsdb.Query, rows *core.Rows, resu
return nil
}
func (e MysqlExecutor) getTypedRowData(types []*sql.ColumnType, rows *core.Rows) (tsdb.RowValues, error) {
func (e MysqlQueryEndpoint) getTypedRowData(types []*sql.ColumnType, rows *core.Rows) (tsdb.RowValues, error) {
values := make([]interface{}, len(types))
for i, stype := range types {
@@ -248,7 +163,7 @@ func (e MysqlExecutor) getTypedRowData(types []*sql.ColumnType, rows *core.Rows)
return values, nil
}
func (e MysqlExecutor) TransformToTimeSeries(query *tsdb.Query, rows *core.Rows, result *tsdb.QueryResult) error {
func (e MysqlQueryEndpoint) transformToTimeSeries(query *tsdb.Query, rows *core.Rows, result *tsdb.QueryResult) error {
pointsBySeries := make(map[string]*tsdb.TimeSeries)
seriesByQueryOrder := list.New()
columnNames, err := rows.Columns()
@@ -261,7 +176,7 @@ func (e MysqlExecutor) TransformToTimeSeries(query *tsdb.Query, rows *core.Rows,
rowLimit := 1000000
rowCount := 0
for ; rows.Next(); rowCount += 1 {
for ; rows.Next(); rowCount++ {
if rowCount > rowLimit {
return fmt.Errorf("MySQL query row limit exceeded, limit %d", rowLimit)
}
pkg/tsdb/mysql/mysql_test.go
+21 -10
View File
@@ -18,14 +18,16 @@ func TestMySQL(t *testing.T) {
SkipConvey("MySQL", t, func() {
x := InitMySQLTestDB(t)
executor := &MysqlExecutor{
engine: x,
log: log.New("tsdb.mysql"),
endpoint := &MysqlQueryEndpoint{
sqlEngine: &tsdb.DefaultSqlEngine{
MacroEngine: NewMysqlMacroEngine(),
XormEngine: x,
},
log: log.New("tsdb.mysql"),
}
sess := x.NewSession()
defer sess.Close()
db := sess.DB()
sql := "CREATE TABLE `mysql_types` ("
sql += "`atinyint` tinyint(1),"
@@ -70,14 +72,23 @@ func TestMySQL(t *testing.T) {
_, err = sess.Exec(sql)
So(err, ShouldBeNil)
Convey("TransformToTable should map MySQL column types to Go types", func() {
rows, err := db.Query("SELECT * FROM mysql_types")
defer rows.Close()
Convey("Query with Table format should map MySQL column types to Go types", func() {
query := &tsdb.TsdbQuery{
Queries: []*tsdb.Query{
{
Model: simplejson.NewFromAny(map[string]interface{}{
"rawSql": "SELECT * FROM mysql_types",
"format": "table",
}),
RefId: "A",
},
},
}
resp, err := endpoint.Query(nil, nil, query)
queryResult := resp.Results["A"]
So(err, ShouldBeNil)
queryResult := &tsdb.QueryResult{Meta: simplejson.New()}
err = executor.TransformToTable(nil, rows, queryResult)
So(err, ShouldBeNil)
column := queryResult.Tables[0].Rows[0]
So(*column[0].(*int8), ShouldEqual, 1)
So(*column[1].(*string), ShouldEqual, "abc")