Postgres Data Source (#9475)

* add postgresql datasource

* add rest of files for postgres datasource

* fix timeseries query, remove unused code

* consistent naming, refactoring

* s/mysql/postgres/

* s/mysql/postgres/

* couple more tests

* tests for more datatypes

* fix macros for postgres

* add __timeSec macro

* add frontend for postgres datasource

* adjust documentation

* fix formatting

* add proper plugin description

* merge editor changes from mysql

* port changes from mysql datasource

* set proper defaultQuery for postgres

* add time_sec to timeseries query
accept int for value for timeseries query

* revert allowing time_sec and handle int or float values as unix
timestamp for "time" column

* fix tslint error

* handle decimal values in timeseries query

* allow setting sslmode for postgres datasource

* use type switch for handling data types

* fix value for timeseries query

* refactor timeseries queries to make them more flexible

* remove debug statement from inner loop in type conversion

* use plain for loop in getTypedRowData

* fix timeseries queries

* adjust postgres datasource to tsdb refactoring

* adjust postgres datasource to frontend changes

* update lib/pq to latest version

* move type conversion to getTypedRowData

* handle address types cidr, inet and macaddr

* adjust response parser and docs for annotations

* convert unknown types to string

* add documentation for postgres datasource

* add another example query with metric column

* set more helpful default query

* update help text in query editor

* handle NULL in value column of timeseries query

* add __timeGroup macro

* add test for __timeGroup macro

* document __timeGroup and set proper default query for annotations

* fix typos in docs

* add postgres to list of datasources

* add postgres to builtInPlugins

* mysql: refactoring as prep for merging postgres

Refactors out the initialization of the xorm engine and the query logic
for an sql data source.

* mysql: rename refactoring + test update

* postgres:refactor to use SqlEngine(same as mysql)

Refactored to use a common base class with the MySql data source.

Other changes from the original PR:
- Changed time column to be time_sec to allow other time units in the
future and to be the same as MySQL
- Changed integration test to test the main Query method rather than
the private transformToTable method
- Changed the __timeSec macro name to __timeEpoch
- Renamed PostgresExecutor to PostgresQueryEndpoint

Fixes #9209 (the original PR)

* postgres: encrypt password on config page

With some other cosmetic changes to the config page:
- placeholder texts
- reset button for the password after it has been encrypted.
- default value for the sslmode field.

* postgres: change back col name to time from time_sec

* postgres mysql: remove annotation title

Title has been removed from annotations

* postgres: fix images for docs page

* postgres mysql: fix specs

pkg/cmd/grafana-server/main.go

@@ -26,7 +26,7 @@ import (
 	_ "github.com/grafana/grafana/pkg/tsdb/influxdb"
 	_ "github.com/grafana/grafana/pkg/tsdb/mysql"
 	_ "github.com/grafana/grafana/pkg/tsdb/opentsdb"
-
+	_ "github.com/grafana/grafana/pkg/tsdb/postgres"
 	_ "github.com/grafana/grafana/pkg/tsdb/prometheus"
 	_ "github.com/grafana/grafana/pkg/tsdb/testdata"
 )

pkg/models/datasource.go

@@ -17,6 +17,7 @@ const (
 	DS_CLOUDWATCH    = "cloudwatch"
 	DS_KAIROSDB      = "kairosdb"
 	DS_PROMETHEUS    = "prometheus"
+	DS_POSTGRES      = "postgres"
 	DS_ACCESS_DIRECT = "direct"
 	DS_ACCESS_PROXY  = "proxy"
 )
@@ -62,6 +63,7 @@ var knownDatasourcePlugins map[string]bool = map[string]bool{
 	DS_CLOUDWATCH:                         true,
 	DS_PROMETHEUS:                         true,
 	DS_OPENTSDB:                           true,
+	DS_POSTGRES:                           true,
 	"opennms":                             true,
 	"druid":                               true,
 	"dalmatinerdb":                        true,

pkg/tsdb/mysql/macros.go

@@ -11,26 +11,21 @@ import (
 const rsIdentifier = `([_a-zA-Z0-9]+)`
 const sExpr = `\$` + rsIdentifier + `\(([^\)]*)\)`
 
-type SqlMacroEngine interface {
-	Interpolate(sql string) (string, error)
-}
-
 type MySqlMacroEngine struct {
 	TimeRange *tsdb.TimeRange
 }
 
-func NewMysqlMacroEngine(timeRange *tsdb.TimeRange) SqlMacroEngine {
-	return &MySqlMacroEngine{
-		TimeRange: timeRange,
-	}
+func NewMysqlMacroEngine() tsdb.SqlMacroEngine {
+	return &MySqlMacroEngine{}
 }
 
-func (m *MySqlMacroEngine) Interpolate(sql string) (string, error) {
+func (m *MySqlMacroEngine) Interpolate(timeRange *tsdb.TimeRange, sql string) (string, error) {
+	m.TimeRange = timeRange
 	rExp, _ := regexp.Compile(sExpr)
 	var macroError error
 
-	sql = ReplaceAllStringSubmatchFunc(rExp, sql, func(groups []string) string {
-		res, err := m.EvaluateMacro(groups[1], groups[2:])
+	sql = replaceAllStringSubmatchFunc(rExp, sql, func(groups []string) string {
+		res, err := m.evaluateMacro(groups[1], groups[2:])
 		if err != nil && macroError == nil {
 			macroError = err
 			return "macro_error()"
@@ -45,7 +40,7 @@ func (m *MySqlMacroEngine) Interpolate(sql string) (string, error) {
 	return sql, nil
 }
 
-func ReplaceAllStringSubmatchFunc(re *regexp.Regexp, str string, repl func([]string) string) string {
+func replaceAllStringSubmatchFunc(re *regexp.Regexp, str string, repl func([]string) string) string {
 	result := ""
 	lastIndex := 0
 
@@ -62,7 +57,7 @@ func ReplaceAllStringSubmatchFunc(re *regexp.Regexp, str string, repl func([]str
 	return result + str[lastIndex:]
 }
 
-func (m *MySqlMacroEngine) EvaluateMacro(name string, args []string) (string, error) {
+func (m *MySqlMacroEngine) evaluateMacro(name string, args []string) (string, error) {
 	switch name {
 	case "__time":
 		if len(args) == 0 {

pkg/tsdb/mysql/macros_test.go

@@ -9,86 +9,60 @@ import (
 
 func TestMacroEngine(t *testing.T) {
 	Convey("MacroEngine", t, func() {
+		engine := &MySqlMacroEngine{}
+		timeRange := &tsdb.TimeRange{From: "5m", To: "now"}
 
 		Convey("interpolate __time function", func() {
-			engine := &MySqlMacroEngine{}
-
-			sql, err := engine.Interpolate("select $__time(time_column)")
+			sql, err := engine.Interpolate(nil, "select $__time(time_column)")
 			So(err, ShouldBeNil)
 
 			So(sql, ShouldEqual, "select UNIX_TIMESTAMP(time_column) as time_sec")
 		})
 
 		Convey("interpolate __time function wrapped in aggregation", func() {
-			engine := &MySqlMacroEngine{}
-
-			sql, err := engine.Interpolate("select min($__time(time_column))")
+			sql, err := engine.Interpolate(nil, "select min($__time(time_column))")
 			So(err, ShouldBeNil)
 
 			So(sql, ShouldEqual, "select min(UNIX_TIMESTAMP(time_column) as time_sec)")
 		})
 
 		Convey("interpolate __timeFilter function", func() {
-			engine := &MySqlMacroEngine{
-				TimeRange: &tsdb.TimeRange{From: "5m", To: "now"},
-			}
-
-			sql, err := engine.Interpolate("WHERE $__timeFilter(time_column)")
+			sql, err := engine.Interpolate(timeRange, "WHERE $__timeFilter(time_column)")
 			So(err, ShouldBeNil)
 
 			So(sql, ShouldEqual, "WHERE time_column >= FROM_UNIXTIME(18446744066914186738) AND time_column <= FROM_UNIXTIME(18446744066914187038)")
 		})
 
 		Convey("interpolate __timeFrom function", func() {
-			engine := &MySqlMacroEngine{
-				TimeRange: &tsdb.TimeRange{From: "5m", To: "now"},
-			}
-
-			sql, err := engine.Interpolate("select $__timeFrom(time_column)")
+			sql, err := engine.Interpolate(timeRange, "select $__timeFrom(time_column)")
 			So(err, ShouldBeNil)
 
 			So(sql, ShouldEqual, "select FROM_UNIXTIME(18446744066914186738)")
 		})
 
 		Convey("interpolate __timeTo function", func() {
-			engine := &MySqlMacroEngine{
-				TimeRange: &tsdb.TimeRange{From: "5m", To: "now"},
-			}
-
-			sql, err := engine.Interpolate("select $__timeTo(time_column)")
+			sql, err := engine.Interpolate(timeRange, "select $__timeTo(time_column)")
 			So(err, ShouldBeNil)
 
 			So(sql, ShouldEqual, "select FROM_UNIXTIME(18446744066914187038)")
 		})
 
 		Convey("interpolate __unixEpochFilter function", func() {
-			engine := &MySqlMacroEngine{
-				TimeRange: &tsdb.TimeRange{From: "5m", To: "now"},
-			}
-
-			sql, err := engine.Interpolate("select $__unixEpochFilter(18446744066914186738)")
+			sql, err := engine.Interpolate(timeRange, "select $__unixEpochFilter(18446744066914186738)")
 			So(err, ShouldBeNil)
 
 			So(sql, ShouldEqual, "select 18446744066914186738 >= 18446744066914186738 AND 18446744066914186738 <= 18446744066914187038")
 		})
 
 		Convey("interpolate __unixEpochFrom function", func() {
-			engine := &MySqlMacroEngine{
-				TimeRange: &tsdb.TimeRange{From: "5m", To: "now"},
-			}
-
-			sql, err := engine.Interpolate("select $__unixEpochFrom()")
+			sql, err := engine.Interpolate(timeRange, "select $__unixEpochFrom()")
 			So(err, ShouldBeNil)
 
 			So(sql, ShouldEqual, "select 18446744066914186738")
 		})
 
 		Convey("interpolate __unixEpochTo function", func() {
-			engine := &MySqlMacroEngine{
-				TimeRange: &tsdb.TimeRange{From: "5m", To: "now"},
-			}
-
-			sql, err := engine.Interpolate("select $__unixEpochTo()")
+			sql, err := engine.Interpolate(timeRange, "select $__unixEpochTo()")
 			So(err, ShouldBeNil)
 
 			So(sql, ShouldEqual, "select 18446744066914187038")

pkg/tsdb/mysql/mysql.go

@@ -6,142 +6,57 @@ import (
 	"database/sql"
 	"fmt"
 	"strconv"
-	"sync"
 
 	"time"
 
 	"github.com/go-sql-driver/mysql"
 	"github.com/go-xorm/core"
-	"github.com/go-xorm/xorm"
 	"github.com/grafana/grafana/pkg/components/null"
-	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/log"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/tsdb"
 )
 
-type MysqlExecutor struct {
-	engine *xorm.Engine
-	log    log.Logger
-}
-
-type engineCacheType struct {
-	cache    map[int64]*xorm.Engine
-	versions map[int64]int
-	sync.Mutex
-}
-
-var engineCache = engineCacheType{
-	cache:    make(map[int64]*xorm.Engine),
-	versions: make(map[int64]int),
+type MysqlQueryEndpoint struct {
+	sqlEngine tsdb.SqlEngine
+	log       log.Logger
 }
 
 func init() {
-	tsdb.RegisterTsdbQueryEndpoint("mysql", NewMysqlExecutor)
+	tsdb.RegisterTsdbQueryEndpoint("mysql", NewMysqlQueryEndpoint)
 }
 
-func NewMysqlExecutor(datasource *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
-	executor := &MysqlExecutor{
+func NewMysqlQueryEndpoint(datasource *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
+	endpoint := &MysqlQueryEndpoint{
 		log: log.New("tsdb.mysql"),
 	}
 
-	err := executor.initEngine(datasource)
-	if err != nil {
-		return nil, err
-	}
-
-	return executor, nil
-}
-
-func (e *MysqlExecutor) initEngine(dsInfo *models.DataSource) error {
-	engineCache.Lock()
-	defer engineCache.Unlock()
-
-	if engine, present := engineCache.cache[dsInfo.Id]; present {
-		if version, _ := engineCache.versions[dsInfo.Id]; version == dsInfo.Version {
-			e.engine = engine
-			return nil
-		}
+	endpoint.sqlEngine = &tsdb.DefaultSqlEngine{
+		MacroEngine: NewMysqlMacroEngine(),
 	}
 
 	cnnstr := fmt.Sprintf("%s:%s@%s(%s)/%s?collation=utf8mb4_unicode_ci&parseTime=true&loc=UTC",
-		dsInfo.User,
-		dsInfo.Password,
+		datasource.User,
+		datasource.Password,
 		"tcp",
-		dsInfo.Url,
-		dsInfo.Database)
+		datasource.Url,
+		datasource.Database,
+	)
+	endpoint.log.Debug("getEngine", "connection", cnnstr)
 
-	e.log.Debug("getEngine", "connection", cnnstr)
-
-	engine, err := xorm.NewEngine("mysql", cnnstr)
-	engine.SetMaxOpenConns(10)
-	engine.SetMaxIdleConns(10)
-	if err != nil {
-		return err
+	if err := endpoint.sqlEngine.InitEngine("mysql", datasource, cnnstr); err != nil {
+		return nil, err
 	}
 
-	engineCache.cache[dsInfo.Id] = engine
-	e.engine = engine
-	return nil
+	return endpoint, nil
 }
 
-func (e *MysqlExecutor) Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) {
-	result := &tsdb.Response{
-		Results: make(map[string]*tsdb.QueryResult),
-	}
-
-	macroEngine := NewMysqlMacroEngine(tsdbQuery.TimeRange)
-	session := e.engine.NewSession()
-	defer session.Close()
-	db := session.DB()
-
-	for _, query := range tsdbQuery.Queries {
-		rawSql := query.Model.Get("rawSql").MustString()
-		if rawSql == "" {
-			continue
-		}
-
-		queryResult := &tsdb.QueryResult{Meta: simplejson.New(), RefId: query.RefId}
-		result.Results[query.RefId] = queryResult
-
-		rawSql, err := macroEngine.Interpolate(rawSql)
-		if err != nil {
-			queryResult.Error = err
-			continue
-		}
-
-		queryResult.Meta.Set("sql", rawSql)
-
-		rows, err := db.Query(rawSql)
-		if err != nil {
-			queryResult.Error = err
-			continue
-		}
-
-		defer rows.Close()
-
-		format := query.Model.Get("format").MustString("time_series")
-
-		switch format {
-		case "time_series":
-			err := e.TransformToTimeSeries(query, rows, queryResult)
-			if err != nil {
-				queryResult.Error = err
-				continue
-			}
-		case "table":
-			err := e.TransformToTable(query, rows, queryResult)
-			if err != nil {
-				queryResult.Error = err
-				continue
-			}
-		}
-	}
-
-	return result, nil
+// Query is the main function for the MysqlExecutor
+func (e *MysqlQueryEndpoint) Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) {
+	return e.sqlEngine.Query(ctx, dsInfo, tsdbQuery, e.transformToTimeSeries, e.transformToTable)
 }
 
-func (e MysqlExecutor) TransformToTable(query *tsdb.Query, rows *core.Rows, result *tsdb.QueryResult) error {
+func (e MysqlQueryEndpoint) transformToTable(query *tsdb.Query, rows *core.Rows, result *tsdb.QueryResult) error {
 	columnNames, err := rows.Columns()
 	columnCount := len(columnNames)
 
@@ -166,7 +81,7 @@ func (e MysqlExecutor) TransformToTable(query *tsdb.Query, rows *core.Rows, resu
 	rowLimit := 1000000
 	rowCount := 0
 
-	for ; rows.Next(); rowCount += 1 {
+	for ; rows.Next(); rowCount++ {
 		if rowCount > rowLimit {
 			return fmt.Errorf("MySQL query row limit exceeded, limit %d", rowLimit)
 		}
@@ -184,7 +99,7 @@ func (e MysqlExecutor) TransformToTable(query *tsdb.Query, rows *core.Rows, resu
 	return nil
 }
 
-func (e MysqlExecutor) getTypedRowData(types []*sql.ColumnType, rows *core.Rows) (tsdb.RowValues, error) {
+func (e MysqlQueryEndpoint) getTypedRowData(types []*sql.ColumnType, rows *core.Rows) (tsdb.RowValues, error) {
 	values := make([]interface{}, len(types))
 
 	for i, stype := range types {
@@ -248,7 +163,7 @@ func (e MysqlExecutor) getTypedRowData(types []*sql.ColumnType, rows *core.Rows)
 	return values, nil
 }
 
-func (e MysqlExecutor) TransformToTimeSeries(query *tsdb.Query, rows *core.Rows, result *tsdb.QueryResult) error {
+func (e MysqlQueryEndpoint) transformToTimeSeries(query *tsdb.Query, rows *core.Rows, result *tsdb.QueryResult) error {
 	pointsBySeries := make(map[string]*tsdb.TimeSeries)
 	seriesByQueryOrder := list.New()
 	columnNames, err := rows.Columns()
@@ -261,7 +176,7 @@ func (e MysqlExecutor) TransformToTimeSeries(query *tsdb.Query, rows *core.Rows,
 	rowLimit := 1000000
 	rowCount := 0
 
-	for ; rows.Next(); rowCount += 1 {
+	for ; rows.Next(); rowCount++ {
 		if rowCount > rowLimit {
 			return fmt.Errorf("MySQL query row limit exceeded, limit %d", rowLimit)
 		}

pkg/tsdb/mysql/mysql_test.go

@@ -18,14 +18,16 @@ func TestMySQL(t *testing.T) {
 	SkipConvey("MySQL", t, func() {
 		x := InitMySQLTestDB(t)
 
-		executor := &MysqlExecutor{
-			engine: x,
-			log:    log.New("tsdb.mysql"),
+		endpoint := &MysqlQueryEndpoint{
+			sqlEngine: &tsdb.DefaultSqlEngine{
+				MacroEngine: NewMysqlMacroEngine(),
+				XormEngine:  x,
+			},
+			log: log.New("tsdb.mysql"),
 		}
 
 		sess := x.NewSession()
 		defer sess.Close()
-		db := sess.DB()
 
 		sql := "CREATE TABLE `mysql_types` ("
 		sql += "`atinyint` tinyint(1),"
@@ -70,14 +72,23 @@ func TestMySQL(t *testing.T) {
 		_, err = sess.Exec(sql)
 		So(err, ShouldBeNil)
 
-		Convey("TransformToTable should map MySQL column types to Go types", func() {
-			rows, err := db.Query("SELECT * FROM mysql_types")
-			defer rows.Close()
+		Convey("Query with Table format should map MySQL column types to Go types", func() {
+			query := &tsdb.TsdbQuery{
+				Queries: []*tsdb.Query{
+					{
+						Model: simplejson.NewFromAny(map[string]interface{}{
+							"rawSql": "SELECT * FROM mysql_types",
+							"format": "table",
+						}),
+						RefId: "A",
+					},
+				},
+			}
+
+			resp, err := endpoint.Query(nil, nil, query)
+			queryResult := resp.Results["A"]
 			So(err, ShouldBeNil)
 
-			queryResult := &tsdb.QueryResult{Meta: simplejson.New()}
-			err = executor.TransformToTable(nil, rows, queryResult)
-			So(err, ShouldBeNil)
 			column := queryResult.Tables[0].Rows[0]
 			So(*column[0].(*int8), ShouldEqual, 1)
 			So(*column[1].(*string), ShouldEqual, "abc")

pkg/tsdb/postgres/macros.go

@@ -0,0 +1,99 @@
+package postgres
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+
+	"github.com/grafana/grafana/pkg/tsdb"
+)
+
+//const rsString = `(?:"([^"]*)")`;
+const rsIdentifier = `([_a-zA-Z0-9]+)`
+const sExpr = `\$` + rsIdentifier + `\(([^\)]*)\)`
+
+type PostgresMacroEngine struct {
+	TimeRange *tsdb.TimeRange
+}
+
+func NewPostgresMacroEngine() tsdb.SqlMacroEngine {
+	return &PostgresMacroEngine{}
+}
+
+func (m *PostgresMacroEngine) Interpolate(timeRange *tsdb.TimeRange, sql string) (string, error) {
+	m.TimeRange = timeRange
+	rExp, _ := regexp.Compile(sExpr)
+	var macroError error
+
+	sql = replaceAllStringSubmatchFunc(rExp, sql, func(groups []string) string {
+		res, err := m.evaluateMacro(groups[1], strings.Split(groups[2], ","))
+		if err != nil && macroError == nil {
+			macroError = err
+			return "macro_error()"
+		}
+		return res
+	})
+
+	if macroError != nil {
+		return "", macroError
+	}
+
+	return sql, nil
+}
+
+func replaceAllStringSubmatchFunc(re *regexp.Regexp, str string, repl func([]string) string) string {
+	result := ""
+	lastIndex := 0
+
+	for _, v := range re.FindAllSubmatchIndex([]byte(str), -1) {
+		groups := []string{}
+		for i := 0; i < len(v); i += 2 {
+			groups = append(groups, str[v[i]:v[i+1]])
+		}
+
+		result += str[lastIndex:v[0]] + repl(groups)
+		lastIndex = v[1]
+	}
+
+	return result + str[lastIndex:]
+}
+
+func (m *PostgresMacroEngine) evaluateMacro(name string, args []string) (string, error) {
+	switch name {
+	case "__time":
+		if len(args) == 0 {
+			return "", fmt.Errorf("missing time column argument for macro %v", name)
+		}
+		return fmt.Sprintf("%s AS \"time\"", args[0]), nil
+	case "__timeEpoch":
+		if len(args) == 0 {
+			return "", fmt.Errorf("missing time column argument for macro %v", name)
+		}
+		return fmt.Sprintf("extract(epoch from %s) as \"time\"", args[0]), nil
+	case "__timeFilter":
+		if len(args) == 0 {
+			return "", fmt.Errorf("missing time column argument for macro %v", name)
+		}
+		return fmt.Sprintf("%s >= to_timestamp(%d) AND %s <= to_timestamp(%d)", args[0], uint64(m.TimeRange.GetFromAsMsEpoch()/1000), args[0], uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil
+	case "__timeFrom":
+		return fmt.Sprintf("to_timestamp(%d)", uint64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil
+	case "__timeTo":
+		return fmt.Sprintf("to_timestamp(%d)", uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil
+	case "__timeGroup":
+		if len(args) < 2 {
+			return "", fmt.Errorf("macro %v needs time column and interval", name)
+		}
+		return fmt.Sprintf("(extract(epoch from \"%s\")/extract(epoch from %s::interval))::int", args[0], args[1]), nil
+	case "__unixEpochFilter":
+		if len(args) == 0 {
+			return "", fmt.Errorf("missing time column argument for macro %v", name)
+		}
+		return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], uint64(m.TimeRange.GetFromAsMsEpoch()/1000), args[0], uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil
+	case "__unixEpochFrom":
+		return fmt.Sprintf("%d", uint64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil
+	case "__unixEpochTo":
+		return fmt.Sprintf("%d", uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil
+	default:
+		return "", fmt.Errorf("Unknown macro %v", name)
+	}
+}

pkg/tsdb/postgres/macros_test.go

@@ -0,0 +1,80 @@
+package postgres
+
+import (
+	"testing"
+
+	"github.com/grafana/grafana/pkg/tsdb"
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestMacroEngine(t *testing.T) {
+	Convey("MacroEngine", t, func() {
+		engine := &PostgresMacroEngine{}
+		timeRange := &tsdb.TimeRange{From: "5m", To: "now"}
+
+		Convey("interpolate __time function", func() {
+			sql, err := engine.Interpolate(nil, "select $__time(time_column)")
+			So(err, ShouldBeNil)
+
+			So(sql, ShouldEqual, "select time_column AS \"time\"")
+		})
+
+		Convey("interpolate __time function wrapped in aggregation", func() {
+			sql, err := engine.Interpolate(nil, "select min($__time(time_column))")
+			So(err, ShouldBeNil)
+
+			So(sql, ShouldEqual, "select min(time_column AS \"time\")")
+		})
+
+		Convey("interpolate __timeFilter function", func() {
+			sql, err := engine.Interpolate(timeRange, "WHERE $__timeFilter(time_column)")
+			So(err, ShouldBeNil)
+
+			So(sql, ShouldEqual, "WHERE time_column >= to_timestamp(18446744066914186738) AND time_column <= to_timestamp(18446744066914187038)")
+		})
+
+		Convey("interpolate __timeFrom function", func() {
+			sql, err := engine.Interpolate(timeRange, "select $__timeFrom(time_column)")
+			So(err, ShouldBeNil)
+
+			So(sql, ShouldEqual, "select to_timestamp(18446744066914186738)")
+		})
+
+		Convey("interpolate __timeGroup function", func() {
+
+			sql, err := engine.Interpolate(timeRange, "GROUP BY $__timeGroup(time_column,'5m')")
+			So(err, ShouldBeNil)
+
+			So(sql, ShouldEqual, "GROUP BY (extract(epoch from \"time_column\")/extract(epoch from '5m'::interval))::int")
+		})
+
+		Convey("interpolate __timeTo function", func() {
+			sql, err := engine.Interpolate(timeRange, "select $__timeTo(time_column)")
+			So(err, ShouldBeNil)
+
+			So(sql, ShouldEqual, "select to_timestamp(18446744066914187038)")
+		})
+
+		Convey("interpolate __unixEpochFilter function", func() {
+			sql, err := engine.Interpolate(timeRange, "select $__unixEpochFilter(18446744066914186738)")
+			So(err, ShouldBeNil)
+
+			So(sql, ShouldEqual, "select 18446744066914186738 >= 18446744066914186738 AND 18446744066914186738 <= 18446744066914187038")
+		})
+
+		Convey("interpolate __unixEpochFrom function", func() {
+			sql, err := engine.Interpolate(timeRange, "select $__unixEpochFrom()")
+			So(err, ShouldBeNil)
+
+			So(sql, ShouldEqual, "select 18446744066914186738")
+		})
+
+		Convey("interpolate __unixEpochTo function", func() {
+			sql, err := engine.Interpolate(timeRange, "select $__unixEpochTo()")
+			So(err, ShouldBeNil)
+
+			So(sql, ShouldEqual, "select 18446744066914187038")
+		})
+
+	})
+}

pkg/tsdb/postgres/postgres.go

@@ -0,0 +1,245 @@
+package postgres
+
+import (
+	"container/list"
+	"context"
+	"fmt"
+	"strconv"
+	"time"
+
+	"github.com/go-xorm/core"
+	"github.com/grafana/grafana/pkg/components/null"
+	"github.com/grafana/grafana/pkg/log"
+	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/tsdb"
+)
+
+type PostgresQueryEndpoint struct {
+	sqlEngine tsdb.SqlEngine
+	log       log.Logger
+}
+
+func init() {
+	tsdb.RegisterTsdbQueryEndpoint("postgres", NewPostgresQueryEndpoint)
+}
+
+func NewPostgresQueryEndpoint(datasource *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
+	endpoint := &PostgresQueryEndpoint{
+		log: log.New("tsdb.postgres"),
+	}
+
+	endpoint.sqlEngine = &tsdb.DefaultSqlEngine{
+		MacroEngine: NewPostgresMacroEngine(),
+	}
+
+	cnnstr := generateConnectionString(datasource)
+	endpoint.log.Debug("getEngine", "connection", cnnstr)
+
+	if err := endpoint.sqlEngine.InitEngine("postgres", datasource, cnnstr); err != nil {
+		return nil, err
+	}
+
+	return endpoint, nil
+}
+
+func generateConnectionString(datasource *models.DataSource) string {
+	password := ""
+	for key, value := range datasource.SecureJsonData.Decrypt() {
+		if key == "password" {
+			password = value
+			break
+		}
+	}
+
+	sslmode := datasource.JsonData.Get("sslmode").MustString("require")
+	return fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=%s", datasource.User, password, datasource.Url, datasource.Database, sslmode)
+}
+
+func (e *PostgresQueryEndpoint) Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) {
+	return e.sqlEngine.Query(ctx, dsInfo, tsdbQuery, e.transformToTimeSeries, e.transformToTable)
+}
+
+func (e PostgresQueryEndpoint) transformToTable(query *tsdb.Query, rows *core.Rows, result *tsdb.QueryResult) error {
+
+	columnNames, err := rows.Columns()
+	if err != nil {
+		return err
+	}
+
+	table := &tsdb.Table{
+		Columns: make([]tsdb.TableColumn, len(columnNames)),
+		Rows:    make([]tsdb.RowValues, 0),
+	}
+
+	for i, name := range columnNames {
+		table.Columns[i].Text = name
+	}
+
+	rowLimit := 1000000
+	rowCount := 0
+
+	for ; rows.Next(); rowCount++ {
+		if rowCount > rowLimit {
+			return fmt.Errorf("PostgreSQL query row limit exceeded, limit %d", rowLimit)
+		}
+
+		values, err := e.getTypedRowData(rows)
+		if err != nil {
+			return err
+		}
+
+		table.Rows = append(table.Rows, values)
+	}
+
+	result.Tables = append(result.Tables, table)
+	result.Meta.Set("rowCount", rowCount)
+	return nil
+}
+
+func (e PostgresQueryEndpoint) getTypedRowData(rows *core.Rows) (tsdb.RowValues, error) {
+
+	types, err := rows.ColumnTypes()
+	if err != nil {
+		return nil, err
+	}
+
+	values := make([]interface{}, len(types))
+	valuePtrs := make([]interface{}, len(types))
+
+	for i := 0; i < len(types); i++ {
+		valuePtrs[i] = &values[i]
+	}
+
+	if err := rows.Scan(valuePtrs...); err != nil {
+		return nil, err
+	}
+
+	// convert types not handled by lib/pq
+	// unhandled types are returned as []byte
+	for i := 0; i < len(types); i++ {
+		if value, ok := values[i].([]byte); ok == true {
+			switch types[i].DatabaseTypeName() {
+			case "NUMERIC":
+				if v, err := strconv.ParseFloat(string(value), 64); err == nil {
+					values[i] = v
+				} else {
+					e.log.Debug("Rows", "Error converting numeric to float", value)
+				}
+			case "UNKNOWN", "CIDR", "INET", "MACADDR":
+				// char literals have type UNKNOWN
+				values[i] = string(value)
+			default:
+				e.log.Debug("Rows", "Unknown database type", types[i].DatabaseTypeName(), "value", value)
+				values[i] = string(value)
+			}
+		}
+	}
+
+	return values, nil
+}
+
+func (e PostgresQueryEndpoint) transformToTimeSeries(query *tsdb.Query, rows *core.Rows, result *tsdb.QueryResult) error {
+	pointsBySeries := make(map[string]*tsdb.TimeSeries)
+	seriesByQueryOrder := list.New()
+	columnNames, err := rows.Columns()
+
+	if err != nil {
+		return err
+	}
+
+	rowLimit := 1000000
+	rowCount := 0
+	timeIndex := -1
+	metricIndex := -1
+
+	// check columns of resultset
+	for i, col := range columnNames {
+		switch col {
+		case "time":
+			timeIndex = i
+		case "metric":
+			metricIndex = i
+		}
+	}
+
+	if timeIndex == -1 {
+		return fmt.Errorf("Found no column named time")
+	}
+
+	for rows.Next() {
+		var timestamp float64
+		var value null.Float
+		var metric string
+
+		if rowCount > rowLimit {
+			return fmt.Errorf("PostgreSQL query row limit exceeded, limit %d", rowLimit)
+		}
+
+		values, err := e.getTypedRowData(rows)
+		if err != nil {
+			return err
+		}
+
+		switch columnValue := values[timeIndex].(type) {
+		case int64:
+			timestamp = float64(columnValue * 1000)
+		case float64:
+			timestamp = columnValue * 1000
+		case time.Time:
+			timestamp = float64(columnValue.Unix() * 1000)
+		default:
+			return fmt.Errorf("Invalid type for column time, must be of type timestamp or unix timestamp")
+		}
+
+		if metricIndex >= 0 {
+			if columnValue, ok := values[metricIndex].(string); ok == true {
+				metric = columnValue
+			} else {
+				return fmt.Errorf("Column metric must be of type char,varchar or text")
+			}
+		}
+
+		for i, col := range columnNames {
+			if i == timeIndex || i == metricIndex {
+				continue
+			}
+
+			switch columnValue := values[i].(type) {
+			case int64:
+				value = null.FloatFrom(float64(columnValue))
+			case float64:
+				value = null.FloatFrom(columnValue)
+			case nil:
+				value.Valid = false
+			default:
+				return fmt.Errorf("Value column must have numeric datatype, column: %s type: %T value: %v", col, columnValue, columnValue)
+			}
+			if metricIndex == -1 {
+				metric = col
+			}
+			e.appendTimePoint(pointsBySeries, seriesByQueryOrder, metric, timestamp, value)
+			rowCount++
+
+		}
+	}
+
+	for elem := seriesByQueryOrder.Front(); elem != nil; elem = elem.Next() {
+		key := elem.Value.(string)
+		result.Series = append(result.Series, pointsBySeries[key])
+	}
+
+	result.Meta.Set("rowCount", rowCount)
+	return nil
+}
+
+func (e PostgresQueryEndpoint) appendTimePoint(pointsBySeries map[string]*tsdb.TimeSeries, seriesByQueryOrder *list.List, metric string, timestamp float64, value null.Float) {
+	if series, exist := pointsBySeries[metric]; exist {
+		series.Points = append(series.Points, tsdb.TimePoint{value, null.FloatFrom(timestamp)})
+	} else {
+		series := &tsdb.TimeSeries{Name: metric}
+		series.Points = append(series.Points, tsdb.TimePoint{value, null.FloatFrom(timestamp)})
+		pointsBySeries[metric] = series
+		seriesByQueryOrder.PushBack(metric)
+	}
+	e.log.Debug("Rows", "metric", metric, "time", timestamp, "value", value)
+}

pkg/tsdb/postgres/postgres_test.go

@@ -0,0 +1,125 @@
+package postgres
+
+import (
+	"testing"
+	"time"
+
+	"github.com/go-xorm/xorm"
+	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/log"
+	"github.com/grafana/grafana/pkg/services/sqlstore/sqlutil"
+	"github.com/grafana/grafana/pkg/tsdb"
+	_ "github.com/lib/pq"
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+// To run this test, remove the Skip from SkipConvey
+// and set up a PostgreSQL db named grafanatest and a user/password grafanatest/grafanatest
+func TestPostgres(t *testing.T) {
+	SkipConvey("PostgreSQL", t, func() {
+		x := InitPostgresTestDB(t)
+
+		endpoint := &PostgresQueryEndpoint{
+			sqlEngine: &tsdb.DefaultSqlEngine{
+				MacroEngine: NewPostgresMacroEngine(),
+				XormEngine:  x,
+			},
+			log: log.New("tsdb.postgres"),
+		}
+
+		sess := x.NewSession()
+		defer sess.Close()
+
+		sql := `
+      CREATE TABLE postgres_types(
+        c00_smallint smallint,
+        c01_integer integer,
+        c02_bigint bigint,
+
+        c03_real real,
+        c04_double double precision,
+        c05_decimal decimal(10,2),
+        c06_numeric numeric(10,2),
+
+        c07_char char(10),
+        c08_varchar varchar(10),
+        c09_text text,
+
+        c10_timestamp timestamp without time zone,
+        c11_timestamptz timestamp with time zone,
+        c12_date date,
+        c13_time time without time zone,
+        c14_timetz time with time zone,
+        c15_interval interval
+      );
+    `
+		_, err := sess.Exec(sql)
+		So(err, ShouldBeNil)
+
+		sql = `
+      INSERT INTO postgres_types VALUES(
+        1,2,3,
+        4.5,6.7,1.1,1.2,
+        'char10','varchar10','text',
+
+        now(),now(),now(),now(),now(),'15m'::interval
+      );
+    `
+		_, err = sess.Exec(sql)
+		So(err, ShouldBeNil)
+
+		Convey("Query with Table format should map PostgreSQL column types to Go types", func() {
+			query := &tsdb.TsdbQuery{
+				Queries: []*tsdb.Query{
+					{
+						Model: simplejson.NewFromAny(map[string]interface{}{
+							"rawSql": "SELECT * FROM postgres_types",
+							"format": "table",
+						}),
+						RefId: "A",
+					},
+				},
+			}
+
+			resp, err := endpoint.Query(nil, nil, query)
+			queryResult := resp.Results["A"]
+			So(err, ShouldBeNil)
+
+			column := queryResult.Tables[0].Rows[0]
+			So(column[0].(int64), ShouldEqual, 1)
+			So(column[1].(int64), ShouldEqual, 2)
+			So(column[2].(int64), ShouldEqual, 3)
+			So(column[3].(float64), ShouldEqual, 4.5)
+			So(column[4].(float64), ShouldEqual, 6.7)
+			// libpq doesnt properly convert decimal, numeric and char to go types but returns []uint8 instead
+			//			So(column[5].(float64), ShouldEqual, 1.1)
+			//			So(column[6].(float64), ShouldEqual, 1.2)
+			//			So(column[7].(string), ShouldEqual, "char")
+			So(column[8].(string), ShouldEqual, "varchar10")
+			So(column[9].(string), ShouldEqual, "text")
+
+			So(column[10].(time.Time), ShouldHaveSameTypeAs, time.Now())
+			So(column[11].(time.Time), ShouldHaveSameTypeAs, time.Now())
+			So(column[12].(time.Time), ShouldHaveSameTypeAs, time.Now())
+			So(column[13].(time.Time), ShouldHaveSameTypeAs, time.Now())
+			So(column[14].(time.Time), ShouldHaveSameTypeAs, time.Now())
+
+			// libpq doesnt properly convert interval to go types but returns []uint8 instead
+			//			So(column[15].(time.Time), ShouldHaveSameTypeAs, time.Now())
+		})
+	})
+}
+
+func InitPostgresTestDB(t *testing.T) *xorm.Engine {
+	x, err := xorm.NewEngine(sqlutil.TestDB_Postgres.DriverName, sqlutil.TestDB_Postgres.ConnStr)
+
+	// x.ShowSQL()
+
+	if err != nil {
+		t.Fatalf("Failed to init postgres db %v", err)
+	}
+
+	sqlutil.CleanDB(x)
+
+	return x
+}

pkg/tsdb/sql_engine.go

@@ -0,0 +1,134 @@
+package tsdb
+
+import (
+	"context"
+	"sync"
+
+	"github.com/go-xorm/core"
+	"github.com/go-xorm/xorm"
+	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/models"
+)
+
+// SqlEngine is a wrapper class around xorm for relational database data sources.
+type SqlEngine interface {
+	InitEngine(driverName string, dsInfo *models.DataSource, cnnstr string) error
+	Query(
+		ctx context.Context,
+		ds *models.DataSource,
+		query *TsdbQuery,
+		transformToTimeSeries func(query *Query, rows *core.Rows, result *QueryResult) error,
+		transformToTable func(query *Query, rows *core.Rows, result *QueryResult) error,
+	) (*Response, error)
+}
+
+// SqlMacroEngine interpolates macros into sql. It takes in the timeRange to be able to
+// generate queries that use from and to.
+type SqlMacroEngine interface {
+	Interpolate(timeRange *TimeRange, sql string) (string, error)
+}
+
+type DefaultSqlEngine struct {
+	MacroEngine SqlMacroEngine
+	XormEngine  *xorm.Engine
+}
+
+type engineCacheType struct {
+	cache    map[int64]*xorm.Engine
+	versions map[int64]int
+	sync.Mutex
+}
+
+var engineCache = engineCacheType{
+	cache:    make(map[int64]*xorm.Engine),
+	versions: make(map[int64]int),
+}
+
+// InitEngine creates the db connection and inits the xorm engine or loads it from the engine cache
+func (e *DefaultSqlEngine) InitEngine(driverName string, dsInfo *models.DataSource, cnnstr string) error {
+	engineCache.Lock()
+	defer engineCache.Unlock()
+
+	if engine, present := engineCache.cache[dsInfo.Id]; present {
+		if version, _ := engineCache.versions[dsInfo.Id]; version == dsInfo.Version {
+			e.XormEngine = engine
+			return nil
+		}
+	}
+
+	engine, err := xorm.NewEngine(driverName, cnnstr)
+	engine.SetMaxOpenConns(10)
+	engine.SetMaxIdleConns(10)
+	if err != nil {
+		return err
+	}
+
+	engineCache.cache[dsInfo.Id] = engine
+	e.XormEngine = engine
+
+	return nil
+}
+
+// Query is a default implementation of the Query method for an SQL data source.
+// The caller of this function must implement transformToTimeSeries and transformToTable and
+// pass them in as parameters.
+func (e *DefaultSqlEngine) Query(
+	ctx context.Context,
+	dsInfo *models.DataSource,
+	tsdbQuery *TsdbQuery,
+	transformToTimeSeries func(query *Query, rows *core.Rows, result *QueryResult) error,
+	transformToTable func(query *Query, rows *core.Rows, result *QueryResult) error,
+) (*Response, error) {
+	result := &Response{
+		Results: make(map[string]*QueryResult),
+	}
+
+	session := e.XormEngine.NewSession()
+	defer session.Close()
+	db := session.DB()
+
+	for _, query := range tsdbQuery.Queries {
+		rawSql := query.Model.Get("rawSql").MustString()
+		if rawSql == "" {
+			continue
+		}
+
+		queryResult := &QueryResult{Meta: simplejson.New(), RefId: query.RefId}
+		result.Results[query.RefId] = queryResult
+
+		rawSql, err := e.MacroEngine.Interpolate(tsdbQuery.TimeRange, rawSql)
+		if err != nil {
+			queryResult.Error = err
+			continue
+		}
+
+		queryResult.Meta.Set("sql", rawSql)
+
+		rows, err := db.Query(rawSql)
+		if err != nil {
+			queryResult.Error = err
+			continue
+		}
+
+		defer rows.Close()
+
+		format := query.Model.Get("format").MustString("time_series")
+
+		switch format {
+		case "time_series":
+			err := transformToTimeSeries(query, rows, queryResult)
+			if err != nil {
+				queryResult.Error = err
+				continue
+			}
+		case "table":
+			err := transformToTable(query, rows, queryResult)
+			if err != nil {
+				queryResult.Error = err
+				continue
+			}
+		}
+	}
+
+	return result, nil
+}