From b987aee7cffe873a01380b4ed6dbf60838f97736 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sat, 21 Jul 2018 20:00:26 +0200 Subject: [PATCH 01/11] add timescaledb option to postgres datasource This adds an option to the postgres datasource config for timescaledb support. When set to auto it will check for timescaledb when testing the datasource. When this option is enabled the $__timeGroup macro will use the time_bucket function from timescaledb to group times by an interval. This also passes the datasource edit control to testDatasource to allow for setting additional settings, this might be useful for other datasources aswell which have optional or version dependant features which can be queried. --- pkg/tsdb/postgres/macros.go | 8 +++- pkg/tsdb/postgres/macros_test.go | 22 +++++++++- pkg/tsdb/postgres/postgres_test.go | 23 ++++++++++- public/app/features/plugins/ds_edit_ctrl.ts | 2 +- .../plugins/datasource/postgres/datasource.ts | 40 +++++++++---------- .../datasource/postgres/partials/config.html | 14 +++++++ 6 files changed, 85 insertions(+), 24 deletions(-) diff --git a/pkg/tsdb/postgres/macros.go b/pkg/tsdb/postgres/macros.go index aebdc55d1d7..4f1d3f72558 100644 --- a/pkg/tsdb/postgres/macros.go +++ b/pkg/tsdb/postgres/macros.go @@ -130,13 +130,19 @@ func (m *postgresMacroEngine) evaluateMacro(name string, args []string) (string, m.query.Model.Set("fillValue", floatVal) } } - return fmt.Sprintf("floor(extract(epoch from %s)/%v)*%v", args[0], interval.Seconds(), interval.Seconds()), nil + + if m.query.DataSource.JsonData.Get("timescaledb").MustString("auto") == "enabled" { + return fmt.Sprintf("time_bucket('%vs',%s) AS time", interval.Seconds(), args[0]), nil + } else { + return fmt.Sprintf("floor(extract(epoch from %s)/%v)*%v AS time", args[0], interval.Seconds(), interval.Seconds()), nil + } case "__timeGroupAlias": tg, err := m.evaluateMacro("__timeGroup", args) if err == nil { return tg + " AS \"time\"", err } return "", err + case "__unixEpochFilter": if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) diff --git a/pkg/tsdb/postgres/macros_test.go b/pkg/tsdb/postgres/macros_test.go index beeea93893b..6c4ba8305b1 100644 --- a/pkg/tsdb/postgres/macros_test.go +++ b/pkg/tsdb/postgres/macros_test.go @@ -6,6 +6,8 @@ import ( "testing" "time" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/tsdb" . "github.com/smartystreets/goconvey/convey" ) @@ -13,7 +15,9 @@ import ( func TestMacroEngine(t *testing.T) { Convey("MacroEngine", t, func() { engine := newPostgresMacroEngine() - query := &tsdb.Query{} + query := &tsdb.Query{DataSource: &models.DataSource{JsonData: simplejson.New()}} + queryTS := &tsdb.Query{DataSource: &models.DataSource{JsonData: simplejson.New()}} + queryTS.DataSource.JsonData.Set("timescaledb", "enabled") Convey("Given a time range between 2018-04-12 00:00 and 2018-04-12 00:05", func() { from := time.Date(2018, 4, 12, 18, 0, 0, 0, time.UTC) @@ -83,6 +87,22 @@ func TestMacroEngine(t *testing.T) { So(sql2, ShouldEqual, sql+" AS \"time\"") }) + Convey("interpolate __timeGroup function with TimescaleDB enabled", func() { + + sql, err := engine.Interpolate(queryTS, timeRange, "GROUP BY $__timeGroup(time_column,'5m')") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "GROUP BY time_bucket('300s',time_column) AS time") + }) + + Convey("interpolate __timeGroup function with spaces between args and TimescaleDB enabled", func() { + + sql, err := engine.Interpolate(queryTS, timeRange, "GROUP BY $__timeGroup(time_column , '5m')") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "GROUP BY time_bucket('300s',time_column) AS time") + }) + Convey("interpolate __timeTo function", func() { sql, err := engine.Interpolate(query, timeRange, "select $__timeTo(time_column)") So(err, ShouldBeNil) diff --git a/pkg/tsdb/postgres/postgres_test.go b/pkg/tsdb/postgres/postgres_test.go index 9e363529df1..27888b318a9 100644 --- a/pkg/tsdb/postgres/postgres_test.go +++ b/pkg/tsdb/postgres/postgres_test.go @@ -27,7 +27,7 @@ import ( // use to verify that the generated data are vizualized as expected, see // devenv/README.md for setup instructions. func TestPostgres(t *testing.T) { - // change to true to run the MySQL tests + // change to true to run the PostgreSQL tests runPostgresTests := false // runPostgresTests := true @@ -102,6 +102,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": "SELECT * FROM postgres_types", "format": "table", @@ -182,6 +183,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": "SELECT $__timeGroup(time, '5m') AS time, avg(value) as value FROM metric GROUP BY 1 ORDER BY 1", "format": "time_series", @@ -226,6 +228,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": "SELECT $__timeGroup(time, '5m', NULL) AS time, avg(value) as value FROM metric GROUP BY 1 ORDER BY 1", "format": "time_series", @@ -280,6 +283,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": "SELECT $__timeGroup(time, '5m', 1.5) AS time, avg(value) as value FROM metric GROUP BY 1 ORDER BY 1", "format": "time_series", @@ -401,6 +405,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT "timeInt64" as time, "timeInt64" FROM metric_values ORDER BY time LIMIT 1`, "format": "time_series", @@ -423,6 +428,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT "timeInt64Nullable" as time, "timeInt64Nullable" FROM metric_values ORDER BY time LIMIT 1`, "format": "time_series", @@ -445,6 +451,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT "timeFloat64" as time, "timeFloat64" FROM metric_values ORDER BY time LIMIT 1`, "format": "time_series", @@ -467,6 +474,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT "timeFloat64Nullable" as time, "timeFloat64Nullable" FROM metric_values ORDER BY time LIMIT 1`, "format": "time_series", @@ -511,6 +519,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT "timeInt32Nullable" as time, "timeInt32Nullable" FROM metric_values ORDER BY time LIMIT 1`, "format": "time_series", @@ -533,6 +542,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT "timeFloat32" as time, "timeFloat32" FROM metric_values ORDER BY time LIMIT 1`, "format": "time_series", @@ -555,6 +565,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT "timeFloat32Nullable" as time, "timeFloat32Nullable" FROM metric_values ORDER BY time LIMIT 1`, "format": "time_series", @@ -577,6 +588,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT $__timeEpoch(time), measurement || ' - value one' as metric, "valueOne" FROM metric_values ORDER BY 1`, "format": "time_series", @@ -625,6 +637,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT $__timeEpoch(time), "valueOne", "valueTwo" FROM metric_values ORDER BY 1`, "format": "time_series", @@ -682,6 +695,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT "time_sec" as time, description as text, tags FROM event WHERE $__unixEpochFilter(time_sec) AND tags='deploy' ORDER BY 1 ASC`, "format": "table", @@ -705,6 +719,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT "time_sec" as time, description as text, tags FROM event WHERE $__unixEpochFilter(time_sec) AND tags='ticket' ORDER BY 1 ASC`, "format": "table", @@ -731,6 +746,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": fmt.Sprintf(`SELECT CAST('%s' AS TIMESTAMP) as time, @@ -761,6 +777,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": fmt.Sprintf(`SELECT %d as time, @@ -791,6 +808,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": fmt.Sprintf(`SELECT cast(%d as bigint) as time, @@ -821,6 +839,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": fmt.Sprintf(`SELECT %d as time, @@ -849,6 +868,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT cast(null as bigint) as time, @@ -877,6 +897,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT cast(null as timestamp) as time, diff --git a/public/app/features/plugins/ds_edit_ctrl.ts b/public/app/features/plugins/ds_edit_ctrl.ts index 542e9cc3648..6e05ddc36be 100644 --- a/public/app/features/plugins/ds_edit_ctrl.ts +++ b/public/app/features/plugins/ds_edit_ctrl.ts @@ -132,7 +132,7 @@ export class DataSourceEditCtrl { this.backendSrv .withNoBackendCache(() => { return datasource - .testDatasource() + .testDatasource(this) .then(result => { this.testing.message = result.message; this.testing.status = result.status; diff --git a/public/app/plugins/datasource/postgres/datasource.ts b/public/app/plugins/datasource/postgres/datasource.ts index 644c9e48b9b..88c928e425a 100644 --- a/public/app/plugins/datasource/postgres/datasource.ts +++ b/public/app/plugins/datasource/postgres/datasource.ts @@ -123,27 +123,27 @@ export class PostgresDatasource { .then(data => this.responseParser.parseMetricFindQueryResult(refId, data)); } - testDatasource() { - return this.backendSrv - .datasourceRequest({ - url: '/api/tsdb/query', - method: 'POST', - data: { - from: '5m', - to: 'now', - queries: [ - { - refId: 'A', - intervalMs: 1, - maxDataPoints: 1, - datasourceId: this.id, - rawSql: 'SELECT 1', - format: 'table', - }, - ], - }, - }) + testDatasource(control) { + return this.metricFindQuery('SELECT 1', {}) .then(res => { + if (control.current.jsonData.timescaledb === 'auto') { + return this.metricFindQuery("SELECT 1 FROM pg_extension WHERE extname='timescaledb'", {}) + .then(res => { + if (res.length === 1) { + control.current.jsonData.timescaledb = 'enabled'; + return this.backendSrv.put('/api/datasources/' + this.id, control.current).then(settings => { + control.current = settings.datasource; + control.updateFrontendSettings(); + return { status: 'success', message: 'Database Connection OK, TimescaleDB found' }; + }); + } + throw new Error('timescaledb not found'); + }) + .catch(err => { + // query errored out or empty so timescaledb is not available + return { status: 'success', message: 'Database Connection OK' }; + }); + } return { status: 'success', message: 'Database Connection OK' }; }) .catch(err => { diff --git a/public/app/plugins/datasource/postgres/partials/config.html b/public/app/plugins/datasource/postgres/partials/config.html index 77f0dcfa4a5..07568fdc459 100644 --- a/public/app/plugins/datasource/postgres/partials/config.html +++ b/public/app/plugins/datasource/postgres/partials/config.html @@ -38,6 +38,20 @@ +

PostgreSQL details

+ +
+
+ +
+ + + This option determines whether TimescaleDB features will be used. + +
+
+
+
User Permission
From c3aad100472063957ecf869115cde521c7d5ccf9 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Thu, 9 Aug 2018 09:19:16 +0200 Subject: [PATCH 02/11] change timescaledb to checkbox instead of select --- pkg/tsdb/postgres/macros.go | 2 +- pkg/tsdb/postgres/macros_test.go | 2 +- .../plugins/datasource/postgres/datasource.ts | 20 +------------------ .../datasource/postgres/partials/config.html | 8 +------- 4 files changed, 4 insertions(+), 28 deletions(-) diff --git a/pkg/tsdb/postgres/macros.go b/pkg/tsdb/postgres/macros.go index 4f1d3f72558..69aa04f45f5 100644 --- a/pkg/tsdb/postgres/macros.go +++ b/pkg/tsdb/postgres/macros.go @@ -131,7 +131,7 @@ func (m *postgresMacroEngine) evaluateMacro(name string, args []string) (string, } } - if m.query.DataSource.JsonData.Get("timescaledb").MustString("auto") == "enabled" { + if m.query.DataSource.JsonData.Get("timescaledb").MustBool() { return fmt.Sprintf("time_bucket('%vs',%s) AS time", interval.Seconds(), args[0]), nil } else { return fmt.Sprintf("floor(extract(epoch from %s)/%v)*%v AS time", args[0], interval.Seconds(), interval.Seconds()), nil diff --git a/pkg/tsdb/postgres/macros_test.go b/pkg/tsdb/postgres/macros_test.go index 6c4ba8305b1..8b2fd7a32f8 100644 --- a/pkg/tsdb/postgres/macros_test.go +++ b/pkg/tsdb/postgres/macros_test.go @@ -17,7 +17,7 @@ func TestMacroEngine(t *testing.T) { engine := newPostgresMacroEngine() query := &tsdb.Query{DataSource: &models.DataSource{JsonData: simplejson.New()}} queryTS := &tsdb.Query{DataSource: &models.DataSource{JsonData: simplejson.New()}} - queryTS.DataSource.JsonData.Set("timescaledb", "enabled") + queryTS.DataSource.JsonData.Set("timescaledb", true) Convey("Given a time range between 2018-04-12 00:00 and 2018-04-12 00:05", func() { from := time.Date(2018, 4, 12, 18, 0, 0, 0, time.UTC) diff --git a/public/app/plugins/datasource/postgres/datasource.ts b/public/app/plugins/datasource/postgres/datasource.ts index 88c928e425a..3d48dce45b2 100644 --- a/public/app/plugins/datasource/postgres/datasource.ts +++ b/public/app/plugins/datasource/postgres/datasource.ts @@ -123,27 +123,9 @@ export class PostgresDatasource { .then(data => this.responseParser.parseMetricFindQueryResult(refId, data)); } - testDatasource(control) { + testDatasource() { return this.metricFindQuery('SELECT 1', {}) .then(res => { - if (control.current.jsonData.timescaledb === 'auto') { - return this.metricFindQuery("SELECT 1 FROM pg_extension WHERE extname='timescaledb'", {}) - .then(res => { - if (res.length === 1) { - control.current.jsonData.timescaledb = 'enabled'; - return this.backendSrv.put('/api/datasources/' + this.id, control.current).then(settings => { - control.current = settings.datasource; - control.updateFrontendSettings(); - return { status: 'success', message: 'Database Connection OK, TimescaleDB found' }; - }); - } - throw new Error('timescaledb not found'); - }) - .catch(err => { - // query errored out or empty so timescaledb is not available - return { status: 'success', message: 'Database Connection OK' }; - }); - } return { status: 'success', message: 'Database Connection OK' }; }) .catch(err => { diff --git a/public/app/plugins/datasource/postgres/partials/config.html b/public/app/plugins/datasource/postgres/partials/config.html index 07568fdc459..14b0b03ddb5 100644 --- a/public/app/plugins/datasource/postgres/partials/config.html +++ b/public/app/plugins/datasource/postgres/partials/config.html @@ -42,13 +42,7 @@
- -
- - - This option determines whether TimescaleDB features will be used. - -
+
From acd1acba2d426270ddb54a6e9b233562ec5f1ebd Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Thu, 9 Aug 2018 09:22:02 +0200 Subject: [PATCH 03/11] revert passing ctrl to testDatasource --- public/app/features/plugins/ds_edit_ctrl.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/app/features/plugins/ds_edit_ctrl.ts b/public/app/features/plugins/ds_edit_ctrl.ts index 6e05ddc36be..542e9cc3648 100644 --- a/public/app/features/plugins/ds_edit_ctrl.ts +++ b/public/app/features/plugins/ds_edit_ctrl.ts @@ -132,7 +132,7 @@ export class DataSourceEditCtrl { this.backendSrv .withNoBackendCache(() => { return datasource - .testDatasource(this) + .testDatasource() .then(result => { this.testing.message = result.message; this.testing.status = result.status; From d2984f3b0f578423a56444516682f475842fa6e7 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Thu, 9 Aug 2018 10:14:14 +0200 Subject: [PATCH 04/11] fix rebase error --- pkg/tsdb/postgres/macros.go | 4 ++-- pkg/tsdb/postgres/macros_test.go | 4 ++-- pkg/tsdb/postgres/postgres_test.go | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/tsdb/postgres/macros.go b/pkg/tsdb/postgres/macros.go index 69aa04f45f5..d9f97e9262c 100644 --- a/pkg/tsdb/postgres/macros.go +++ b/pkg/tsdb/postgres/macros.go @@ -132,9 +132,9 @@ func (m *postgresMacroEngine) evaluateMacro(name string, args []string) (string, } if m.query.DataSource.JsonData.Get("timescaledb").MustBool() { - return fmt.Sprintf("time_bucket('%vs',%s) AS time", interval.Seconds(), args[0]), nil + return fmt.Sprintf("time_bucket('%vs',%s)", interval.Seconds(), args[0]), nil } else { - return fmt.Sprintf("floor(extract(epoch from %s)/%v)*%v AS time", args[0], interval.Seconds(), interval.Seconds()), nil + return fmt.Sprintf("floor(extract(epoch from %s)/%v)*%v", args[0], interval.Seconds(), interval.Seconds()), nil } case "__timeGroupAlias": tg, err := m.evaluateMacro("__timeGroup", args) diff --git a/pkg/tsdb/postgres/macros_test.go b/pkg/tsdb/postgres/macros_test.go index 8b2fd7a32f8..449331224c2 100644 --- a/pkg/tsdb/postgres/macros_test.go +++ b/pkg/tsdb/postgres/macros_test.go @@ -92,7 +92,7 @@ func TestMacroEngine(t *testing.T) { sql, err := engine.Interpolate(queryTS, timeRange, "GROUP BY $__timeGroup(time_column,'5m')") So(err, ShouldBeNil) - So(sql, ShouldEqual, "GROUP BY time_bucket('300s',time_column) AS time") + So(sql, ShouldEqual, "GROUP BY time_bucket('300s',time_column)") }) Convey("interpolate __timeGroup function with spaces between args and TimescaleDB enabled", func() { @@ -100,7 +100,7 @@ func TestMacroEngine(t *testing.T) { sql, err := engine.Interpolate(queryTS, timeRange, "GROUP BY $__timeGroup(time_column , '5m')") So(err, ShouldBeNil) - So(sql, ShouldEqual, "GROUP BY time_bucket('300s',time_column) AS time") + So(sql, ShouldEqual, "GROUP BY time_bucket('300s',time_column)") }) Convey("interpolate __timeTo function", func() { diff --git a/pkg/tsdb/postgres/postgres_test.go b/pkg/tsdb/postgres/postgres_test.go index 27888b318a9..87b7f916ca9 100644 --- a/pkg/tsdb/postgres/postgres_test.go +++ b/pkg/tsdb/postgres/postgres_test.go @@ -311,6 +311,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": "SELECT $__timeGroup(time, '5m', previous), avg(value) as value FROM metric GROUP BY 1 ORDER BY 1", "format": "time_series", From a4a33d80dbe1ee0dfe4a3a53a434c90919842e76 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Thu, 9 Aug 2018 17:30:46 +0200 Subject: [PATCH 05/11] mention time_bucket in timescaledb tooltip --- public/app/plugins/datasource/postgres/partials/config.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/partials/config.html b/public/app/plugins/datasource/postgres/partials/config.html index 14b0b03ddb5..a1783c09dc4 100644 --- a/public/app/plugins/datasource/postgres/partials/config.html +++ b/public/app/plugins/datasource/postgres/partials/config.html @@ -42,7 +42,7 @@
- +
From 3552a4cb86151c91ecbf0b2d3265761b276dbaa6 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 14 Aug 2018 08:34:20 +0200 Subject: [PATCH 06/11] refactor timescaledb handling in MacroEngine --- pkg/tsdb/postgres/macros.go | 15 +++++++++------ pkg/tsdb/postgres/macros_test.go | 14 ++++++++------ pkg/tsdb/postgres/postgres.go | 2 +- pkg/tsdb/postgres/postgres_test.go | 22 ---------------------- 4 files changed, 18 insertions(+), 35 deletions(-) diff --git a/pkg/tsdb/postgres/macros.go b/pkg/tsdb/postgres/macros.go index d9f97e9262c..81b0da9fbce 100644 --- a/pkg/tsdb/postgres/macros.go +++ b/pkg/tsdb/postgres/macros.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/tsdb" ) @@ -15,12 +16,15 @@ const rsIdentifier = `([_a-zA-Z0-9]+)` const sExpr = `\$` + rsIdentifier + `\(([^\)]*)\)` type postgresMacroEngine struct { - timeRange *tsdb.TimeRange - query *tsdb.Query + timeRange *tsdb.TimeRange + query *tsdb.Query + timescaledb bool } -func newPostgresMacroEngine() tsdb.SqlMacroEngine { - return &postgresMacroEngine{} +func newPostgresMacroEngine(datasource *models.DataSource) tsdb.SqlMacroEngine { + engine := &postgresMacroEngine{} + engine.timescaledb = datasource.JsonData.Get("timescaledb").MustBool(false) + return engine } func (m *postgresMacroEngine) Interpolate(query *tsdb.Query, timeRange *tsdb.TimeRange, sql string) (string, error) { @@ -131,7 +135,7 @@ func (m *postgresMacroEngine) evaluateMacro(name string, args []string) (string, } } - if m.query.DataSource.JsonData.Get("timescaledb").MustBool() { + if m.timescaledb { return fmt.Sprintf("time_bucket('%vs',%s)", interval.Seconds(), args[0]), nil } else { return fmt.Sprintf("floor(extract(epoch from %s)/%v)*%v", args[0], interval.Seconds(), interval.Seconds()), nil @@ -142,7 +146,6 @@ func (m *postgresMacroEngine) evaluateMacro(name string, args []string) (string, return tg + " AS \"time\"", err } return "", err - case "__unixEpochFilter": if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) diff --git a/pkg/tsdb/postgres/macros_test.go b/pkg/tsdb/postgres/macros_test.go index 449331224c2..fe95535fe0c 100644 --- a/pkg/tsdb/postgres/macros_test.go +++ b/pkg/tsdb/postgres/macros_test.go @@ -14,10 +14,12 @@ import ( func TestMacroEngine(t *testing.T) { Convey("MacroEngine", t, func() { - engine := newPostgresMacroEngine() - query := &tsdb.Query{DataSource: &models.DataSource{JsonData: simplejson.New()}} - queryTS := &tsdb.Query{DataSource: &models.DataSource{JsonData: simplejson.New()}} - queryTS.DataSource.JsonData.Set("timescaledb", true) + datasource := &models.DataSource{JsonData: simplejson.New()} + engine := newPostgresMacroEngine(datasource) + datasourceTS := &models.DataSource{JsonData: simplejson.New()} + datasourceTS.JsonData.Set("timescaledb", true) + engineTS := newPostgresMacroEngine(datasourceTS) + query := &tsdb.Query{} Convey("Given a time range between 2018-04-12 00:00 and 2018-04-12 00:05", func() { from := time.Date(2018, 4, 12, 18, 0, 0, 0, time.UTC) @@ -89,7 +91,7 @@ func TestMacroEngine(t *testing.T) { Convey("interpolate __timeGroup function with TimescaleDB enabled", func() { - sql, err := engine.Interpolate(queryTS, timeRange, "GROUP BY $__timeGroup(time_column,'5m')") + sql, err := engineTS.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column,'5m')") So(err, ShouldBeNil) So(sql, ShouldEqual, "GROUP BY time_bucket('300s',time_column)") @@ -97,7 +99,7 @@ func TestMacroEngine(t *testing.T) { Convey("interpolate __timeGroup function with spaces between args and TimescaleDB enabled", func() { - sql, err := engine.Interpolate(queryTS, timeRange, "GROUP BY $__timeGroup(time_column , '5m')") + sql, err := engineTS.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column , '5m')") So(err, ShouldBeNil) So(sql, ShouldEqual, "GROUP BY time_bucket('300s',time_column)") diff --git a/pkg/tsdb/postgres/postgres.go b/pkg/tsdb/postgres/postgres.go index b9f333db127..46d766f9a11 100644 --- a/pkg/tsdb/postgres/postgres.go +++ b/pkg/tsdb/postgres/postgres.go @@ -32,7 +32,7 @@ func newPostgresQueryEndpoint(datasource *models.DataSource) (tsdb.TsdbQueryEndp log: logger, } - return tsdb.NewSqlQueryEndpoint(&config, &rowTransformer, newPostgresMacroEngine(), logger) + return tsdb.NewSqlQueryEndpoint(&config, &rowTransformer, newPostgresMacroEngine(datasource), logger) } func generateConnectionString(datasource *models.DataSource) string { diff --git a/pkg/tsdb/postgres/postgres_test.go b/pkg/tsdb/postgres/postgres_test.go index 87b7f916ca9..4e05f676682 100644 --- a/pkg/tsdb/postgres/postgres_test.go +++ b/pkg/tsdb/postgres/postgres_test.go @@ -102,7 +102,6 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { - DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": "SELECT * FROM postgres_types", "format": "table", @@ -183,7 +182,6 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { - DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": "SELECT $__timeGroup(time, '5m') AS time, avg(value) as value FROM metric GROUP BY 1 ORDER BY 1", "format": "time_series", @@ -228,7 +226,6 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { - DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": "SELECT $__timeGroup(time, '5m', NULL) AS time, avg(value) as value FROM metric GROUP BY 1 ORDER BY 1", "format": "time_series", @@ -283,7 +280,6 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { - DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": "SELECT $__timeGroup(time, '5m', 1.5) AS time, avg(value) as value FROM metric GROUP BY 1 ORDER BY 1", "format": "time_series", @@ -311,7 +307,6 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { - DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": "SELECT $__timeGroup(time, '5m', previous), avg(value) as value FROM metric GROUP BY 1 ORDER BY 1", "format": "time_series", @@ -406,7 +401,6 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { - DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT "timeInt64" as time, "timeInt64" FROM metric_values ORDER BY time LIMIT 1`, "format": "time_series", @@ -429,7 +423,6 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { - DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT "timeInt64Nullable" as time, "timeInt64Nullable" FROM metric_values ORDER BY time LIMIT 1`, "format": "time_series", @@ -452,7 +445,6 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { - DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT "timeFloat64" as time, "timeFloat64" FROM metric_values ORDER BY time LIMIT 1`, "format": "time_series", @@ -475,7 +467,6 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { - DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT "timeFloat64Nullable" as time, "timeFloat64Nullable" FROM metric_values ORDER BY time LIMIT 1`, "format": "time_series", @@ -520,7 +511,6 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { - DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT "timeInt32Nullable" as time, "timeInt32Nullable" FROM metric_values ORDER BY time LIMIT 1`, "format": "time_series", @@ -543,7 +533,6 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { - DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT "timeFloat32" as time, "timeFloat32" FROM metric_values ORDER BY time LIMIT 1`, "format": "time_series", @@ -566,7 +555,6 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { - DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT "timeFloat32Nullable" as time, "timeFloat32Nullable" FROM metric_values ORDER BY time LIMIT 1`, "format": "time_series", @@ -589,7 +577,6 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { - DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT $__timeEpoch(time), measurement || ' - value one' as metric, "valueOne" FROM metric_values ORDER BY 1`, "format": "time_series", @@ -638,7 +625,6 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { - DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT $__timeEpoch(time), "valueOne", "valueTwo" FROM metric_values ORDER BY 1`, "format": "time_series", @@ -696,7 +682,6 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { - DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT "time_sec" as time, description as text, tags FROM event WHERE $__unixEpochFilter(time_sec) AND tags='deploy' ORDER BY 1 ASC`, "format": "table", @@ -720,7 +705,6 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { - DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT "time_sec" as time, description as text, tags FROM event WHERE $__unixEpochFilter(time_sec) AND tags='ticket' ORDER BY 1 ASC`, "format": "table", @@ -747,7 +731,6 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { - DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": fmt.Sprintf(`SELECT CAST('%s' AS TIMESTAMP) as time, @@ -778,7 +761,6 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { - DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": fmt.Sprintf(`SELECT %d as time, @@ -809,7 +791,6 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { - DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": fmt.Sprintf(`SELECT cast(%d as bigint) as time, @@ -840,7 +821,6 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { - DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": fmt.Sprintf(`SELECT %d as time, @@ -869,7 +849,6 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { - DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT cast(null as bigint) as time, @@ -898,7 +877,6 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { - DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT cast(null as timestamp) as time, From 3955133f7e143002bd7b141808a1323ade444694 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 14 Aug 2018 09:15:24 +0200 Subject: [PATCH 07/11] Don't pass datasource to newPostgresMacroEngine --- pkg/tsdb/postgres/macros.go | 7 ++----- pkg/tsdb/postgres/macros_test.go | 9 ++------- pkg/tsdb/postgres/postgres.go | 4 +++- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/pkg/tsdb/postgres/macros.go b/pkg/tsdb/postgres/macros.go index 81b0da9fbce..0a9162a2d4c 100644 --- a/pkg/tsdb/postgres/macros.go +++ b/pkg/tsdb/postgres/macros.go @@ -7,7 +7,6 @@ import ( "strings" "time" - "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/tsdb" ) @@ -21,10 +20,8 @@ type postgresMacroEngine struct { timescaledb bool } -func newPostgresMacroEngine(datasource *models.DataSource) tsdb.SqlMacroEngine { - engine := &postgresMacroEngine{} - engine.timescaledb = datasource.JsonData.Get("timescaledb").MustBool(false) - return engine +func newPostgresMacroEngine(timescaledb bool) tsdb.SqlMacroEngine { + return &postgresMacroEngine{timescaledb: timescaledb} } func (m *postgresMacroEngine) Interpolate(query *tsdb.Query, timeRange *tsdb.TimeRange, sql string) (string, error) { diff --git a/pkg/tsdb/postgres/macros_test.go b/pkg/tsdb/postgres/macros_test.go index fe95535fe0c..30a57a7095f 100644 --- a/pkg/tsdb/postgres/macros_test.go +++ b/pkg/tsdb/postgres/macros_test.go @@ -6,19 +6,14 @@ import ( "testing" "time" - "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/tsdb" . "github.com/smartystreets/goconvey/convey" ) func TestMacroEngine(t *testing.T) { Convey("MacroEngine", t, func() { - datasource := &models.DataSource{JsonData: simplejson.New()} - engine := newPostgresMacroEngine(datasource) - datasourceTS := &models.DataSource{JsonData: simplejson.New()} - datasourceTS.JsonData.Set("timescaledb", true) - engineTS := newPostgresMacroEngine(datasourceTS) + engine := newPostgresMacroEngine(false) + engineTS := newPostgresMacroEngine(true) query := &tsdb.Query{} Convey("Given a time range between 2018-04-12 00:00 and 2018-04-12 00:05", func() { diff --git a/pkg/tsdb/postgres/postgres.go b/pkg/tsdb/postgres/postgres.go index 46d766f9a11..4bcf06638f4 100644 --- a/pkg/tsdb/postgres/postgres.go +++ b/pkg/tsdb/postgres/postgres.go @@ -32,7 +32,9 @@ func newPostgresQueryEndpoint(datasource *models.DataSource) (tsdb.TsdbQueryEndp log: logger, } - return tsdb.NewSqlQueryEndpoint(&config, &rowTransformer, newPostgresMacroEngine(datasource), logger) + timescaledb := datasource.JsonData.Get("timescaledb").MustBool(false) + + return tsdb.NewSqlQueryEndpoint(&config, &rowTransformer, newPostgresMacroEngine(timescaledb), logger) } func generateConnectionString(datasource *models.DataSource) string { From 837388d13e0a0a84c4829edf4eca285321079f5e Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 14 Aug 2018 09:44:58 +0200 Subject: [PATCH 08/11] Use variable in newPostgresMacroEngine --- pkg/tsdb/postgres/macros_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/tsdb/postgres/macros_test.go b/pkg/tsdb/postgres/macros_test.go index 30a57a7095f..f0c8832dd05 100644 --- a/pkg/tsdb/postgres/macros_test.go +++ b/pkg/tsdb/postgres/macros_test.go @@ -12,8 +12,10 @@ import ( func TestMacroEngine(t *testing.T) { Convey("MacroEngine", t, func() { - engine := newPostgresMacroEngine(false) - engineTS := newPostgresMacroEngine(true) + timescaledbEnabled := false + engine := newPostgresMacroEngine(timescaledbEnabled) + timescaledbEnabled = true + engineTS := newPostgresMacroEngine(timescaledbEnabled) query := &tsdb.Query{} Convey("Given a time range between 2018-04-12 00:00 and 2018-04-12 00:05", func() { From d33019ca6740e55d89119bbf6fce32056cdded3f Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 14 Aug 2018 10:22:57 +0200 Subject: [PATCH 09/11] document TimescaleDB datasource option --- docs/sources/features/datasources/postgres.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/sources/features/datasources/postgres.md b/docs/sources/features/datasources/postgres.md index 2be2db0837b..e8ed742f64f 100644 --- a/docs/sources/features/datasources/postgres.md +++ b/docs/sources/features/datasources/postgres.md @@ -31,6 +31,7 @@ Name | Description *User* | Database user's login/username *Password* | Database user's password *SSL Mode* | This option determines whether or with what priority a secure SSL TCP/IP connection will be negotiated with the server. +*TimescaleDB* | With this option enabled Grafana will use TimescaleDB features, e.g. use ```time_bucket``` for grouping by time. ### Database User Permissions (Important!) From a96d97e347ad8a8725ca7f8fbb80271812d7e64c Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 14 Aug 2018 10:26:08 +0200 Subject: [PATCH 10/11] add version disclaimer for TimescaleDB --- docs/sources/features/datasources/postgres.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/features/datasources/postgres.md b/docs/sources/features/datasources/postgres.md index e8ed742f64f..e2dcf888025 100644 --- a/docs/sources/features/datasources/postgres.md +++ b/docs/sources/features/datasources/postgres.md @@ -31,7 +31,7 @@ Name | Description *User* | Database user's login/username *Password* | Database user's password *SSL Mode* | This option determines whether or with what priority a secure SSL TCP/IP connection will be negotiated with the server. -*TimescaleDB* | With this option enabled Grafana will use TimescaleDB features, e.g. use ```time_bucket``` for grouping by time. +*TimescaleDB* | With this option enabled Grafana will use TimescaleDB features, e.g. use ```time_bucket``` for grouping by time (only available in Grafana 5.3+). ### Database User Permissions (Important!) From 6225efa50ccdcd1a80fe4deeac2570deffcbac06 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Tue, 14 Aug 2018 11:24:08 +0200 Subject: [PATCH 11/11] docs: update postgres provisioning --- docs/sources/features/datasources/postgres.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/sources/features/datasources/postgres.md b/docs/sources/features/datasources/postgres.md index e2dcf888025..4afde5cc6cb 100644 --- a/docs/sources/features/datasources/postgres.md +++ b/docs/sources/features/datasources/postgres.md @@ -290,4 +290,5 @@ datasources: password: "Password!" jsonData: sslmode: "disable" # disable/require/verify-ca/verify-full + timescaledb: false ```