public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

PostgreSQL: Remove feature toggle postgresDSUsePGX (#113675)

Browse Source 
* PostgreSQL: Remove feature toggle `postgresDSUsePGX`

* Fix tests and linting

* Address review comments
This commit is contained in:
Zoltán Bedi
2025-11-24 10:26:41 +01:00
committed by GitHub
parent 39dc659ad8
commit 8d75d79313
34 changed files with 1176 additions and 5165 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/grafana-data/src/types/featureToggles.gen.ts
Generated
-4
View File
@@ -922,10 +922,6 @@ export interface FeatureToggles {
*/
grafanaAssistantInProfilesDrilldown?: boolean;
/**
* Enables using PGX instead of libpq for PostgreSQL datasource
*/
postgresDSUsePGX?: boolean;
/**
* Enables creating alerts from Tempo data source
*/
tempoAlerting?: boolean;
pkg/plugins/backendplugin/coreplugin/registry.go
+1 -1
View File
@@ -237,7 +237,7 @@ func NewPlugin(pluginID string, cfg *setting.Cfg, httpClientProvider *httpclient
case Tempo:
svc = tempo.ProvideService(httpClientProvider, tracer)
case PostgreSQL:
svc = postgres.ProvideService(cfg)
svc = postgres.ProvideService()
case MySQL:
svc = mysql.ProvideService()
case MSSQL:
pkg/server/wire_gen.go
Generated
+2 -2
View File
@@ -400,7 +400,7 @@ func Initialize(ctx context.Context, cfg *setting.Cfg, opts Options, apiOpts api
prometheusService := prometheus.ProvideService(httpclientProvider)
tempoService := tempo.ProvideService(httpclientProvider, tracer)
testdatasourceService := testdatasource.ProvideService()
postgresService := postgres.ProvideService(cfg)
postgresService := postgres.ProvideService()
mysqlService := mysql.ProvideService()
mssqlService := mssql.ProvideService(cfg)
entityEventsService := store.ProvideEntityEventsService(cfg, sqlStore, featureToggles)
@@ -1047,7 +1047,7 @@ func InitializeForTest(ctx context.Context, t sqlutil.ITestDB, testingT interfac
prometheusService := prometheus.ProvideService(httpclientProvider)
tempoService := tempo.ProvideService(httpclientProvider, tracer)
testdatasourceService := testdatasource.ProvideService()
postgresService := postgres.ProvideService(cfg)
postgresService := postgres.ProvideService()
mysqlService := mysql.ProvideService()
mssqlService := mssql.ProvideService(cfg)
entityEventsService := store.ProvideEntityEventsService(cfg, sqlStore, featureToggles)
pkg/services/featuremgmt/registry.go
-6
View File
@@ -1518,12 +1518,6 @@ var (
FrontendOnly: true,
Expression: "true",
},
{
Name: "postgresDSUsePGX",
Description: "Enables using PGX instead of libpq for PostgreSQL datasource",
Stage: FeatureStageExperimental,
Owner: grafanaOSSBigTent,
},
{
Name: "tempoAlerting",
Description: "Enables creating alerts from Tempo data source",
pkg/services/featuremgmt/toggles_gen.csv
Generated
-1
View File
@@ -208,7 +208,6 @@ unifiedNavbars,GA,@grafana/plugins-platform-backend,false,false,true
logsPanelControls,preview,@grafana/observability-logs,false,false,true
metricsFromProfiles,experimental,@grafana/observability-traces-and-profiling,false,false,true
grafanaAssistantInProfilesDrilldown,GA,@grafana/observability-traces-and-profiling,false,false,true
postgresDSUsePGX,experimental,@grafana/oss-big-tent,false,false,false
tempoAlerting,experimental,@grafana/observability-traces-and-profiling,false,false,false
pluginsAutoUpdate,experimental,@grafana/plugins-platform-backend,false,false,false
alertingListViewV2PreviewToggle,privatePreview,@grafana/alerting-squad,false,false,true
1 Name Stage Owner requiresDevMode RequiresRestart FrontendOnly
208 logsPanelControls preview @grafana/observability-logs false false true
209 metricsFromProfiles experimental @grafana/observability-traces-and-profiling false false true
210 grafanaAssistantInProfilesDrilldown GA @grafana/observability-traces-and-profiling false false true
postgresDSUsePGX experimental @grafana/oss-big-tent false false false
211 tempoAlerting experimental @grafana/observability-traces-and-profiling false false false
212 pluginsAutoUpdate experimental @grafana/plugins-platform-backend false false false
213 alertingListViewV2PreviewToggle privatePreview @grafana/alerting-squad false false true
pkg/services/featuremgmt/toggles_gen.go
Generated
-4
View File
@@ -606,10 +606,6 @@ const (
// use multi-tenant path for awsTempCredentials
FlagMultiTenantTempCredentials = "multiTenantTempCredentials"
// FlagPostgresDSUsePGX
// Enables using PGX instead of libpq for PostgreSQL datasource
FlagPostgresDSUsePGX = "postgresDSUsePGX"
// FlagTempoAlerting
// Enables creating alerts from Tempo data source
FlagTempoAlerting = "tempoAlerting"
pkg/services/pluginsintegration/plugintest/plugins_test.go
+1 -1
View File
@@ -161,7 +161,7 @@ func TestIntegrationPluginManager(t *testing.T) {
pr := prometheus.ProvideService(hcp)
tmpo := tempo.ProvideService(hcp, tracer)
td := testdatasource.ProvideService()
pg := postgres.ProvideService(cfg)
pg := postgres.ProvideService()
my := mysql.ProvideService()
ms := mssql.ProvideService(cfg)
db := db.InitTestDB(t, sqlstore.InitTestDBOpt{Cfg: cfg})
pkg/tsdb/grafana-postgresql-datasource/locker.go
-85
View File
@@ -1,85 +0,0 @@
package postgres
import (
"fmt"
"sync"
)
// locker is a named reader/writer mutual exclusion lock.
// The lock for each particular key can be held by an arbitrary number of readers or a single writer.
type locker struct {
locks map[any]*sync.RWMutex
locksRW *sync.RWMutex
}
func newLocker() *locker {
return &locker{
locks: make(map[any]*sync.RWMutex),
locksRW: new(sync.RWMutex),
}
}
// Lock locks named rw mutex with specified key for writing.
// If the lock with the same key is already locked for reading or writing,
// Lock blocks until the lock is available.
func (lkr *locker) Lock(key any) {
lk, ok := lkr.getLock(key)
if !ok {
lk = lkr.newLock(key)
}
lk.Lock()
}
// Unlock unlocks named rw mutex with specified key for writing. It is a run-time error if rw is
// not locked for writing on entry to Unlock.
func (lkr *locker) Unlock(key any) {
lk, ok := lkr.getLock(key)
if !ok {
panic(fmt.Errorf("lock for key '%s' not initialized", key))
}
lk.Unlock()
}
// RLock locks named rw mutex with specified key for reading.
//
// It should not be used for recursive read locking for the same key; a blocked Lock
// call excludes new readers from acquiring the lock. See the
// documentation on the golang RWMutex type.
func (lkr *locker) RLock(key any) {
lk, ok := lkr.getLock(key)
if !ok {
lk = lkr.newLock(key)
}
lk.RLock()
}
// RUnlock undoes a single RLock call for specified key;
// it does not affect other simultaneous readers of locker for specified key.
// It is a run-time error if locker for specified key is not locked for reading
func (lkr *locker) RUnlock(key any) {
lk, ok := lkr.getLock(key)
if !ok {
panic(fmt.Errorf("lock for key '%s' not initialized", key))
}
lk.RUnlock()
}
func (lkr *locker) newLock(key any) *sync.RWMutex {
lkr.locksRW.Lock()
defer lkr.locksRW.Unlock()
if lk, ok := lkr.locks[key]; ok {
return lk
}
lk := new(sync.RWMutex)
lkr.locks[key] = lk
return lk
}
func (lkr *locker) getLock(key any) (*sync.RWMutex, bool) {
lkr.locksRW.RLock()
defer lkr.locksRW.RUnlock()
lock, ok := lkr.locks[key]
return lock, ok
}
pkg/tsdb/grafana-postgresql-datasource/locker_test.go
-63
View File
@@ -1,63 +0,0 @@
package postgres
import (
"sync"
"testing"
"time"
"github.com/grafana/grafana/pkg/util/testutil"
"github.com/stretchr/testify/require"
)
func TestIntegrationLocker(t *testing.T) {
testutil.SkipIntegrationTestInShortMode(t)
const notUpdated = "not_updated"
const atThread1 = "at_thread_1"
const atThread2 = "at_thread_2"
t.Run("Should lock for same keys", func(t *testing.T) {
updated := notUpdated
locker := newLocker()
locker.Lock(1)
var wg sync.WaitGroup
wg.Add(1)
defer func() {
locker.Unlock(1)
wg.Wait()
}()
go func() {
locker.RLock(1)
defer func() {
locker.RUnlock(1)
wg.Done()
}()
require.Equal(t, atThread1, updated, "Value should be updated in different thread")
updated = atThread2
}()
time.Sleep(time.Millisecond * 10)
require.Equal(t, notUpdated, updated, "Value should not be updated in different thread")
updated = atThread1
})
t.Run("Should not lock for different keys", func(t *testing.T) {
updated := notUpdated
locker := newLocker()
locker.Lock(1)
defer locker.Unlock(1)
var wg sync.WaitGroup
wg.Add(1)
go func() {
locker.RLock(2)
defer func() {
locker.RUnlock(2)
wg.Done()
}()
require.Equal(t, notUpdated, updated, "Value should not be updated in different thread")
updated = atThread2
}()
wg.Wait()
require.Equal(t, atThread2, updated, "Value should be updated in different thread")
updated = atThread1
})
}
pkg/tsdb/grafana-postgresql-datasource/pgx/handler_checkhealth.go
-117
View File
@@ -1,117 +0,0 @@
package pgx
import (
"context"
"encoding/json"
"errors"
"fmt"
"net"
"strings"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/backend/log"
"github.com/grafana/grafana/pkg/tsdb/grafana-postgresql-datasource/sqleng"
)
func (e *DataSourceHandler) CheckHealth(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) {
err := e.Ping(ctx)
if err != nil {
logCheckHealthError(ctx, e.dsInfo, err)
if strings.EqualFold(req.PluginContext.User.Role, "Admin") {
return ErrToHealthCheckResult(err)
}
errResponse := &backend.CheckHealthResult{
Status: backend.HealthStatusError,
Message: e.TransformQueryError(e.log, err).Error(),
}
return errResponse, nil
}
return &backend.CheckHealthResult{Status: backend.HealthStatusOk, Message: "Database Connection OK"}, nil
}
// ErrToHealthCheckResult converts error into user friendly health check message
// This should be called with non nil error. If the err parameter is empty, we will send Internal Server Error
func ErrToHealthCheckResult(err error) (*backend.CheckHealthResult, error) {
if err == nil {
return &backend.CheckHealthResult{Status: backend.HealthStatusError, Message: "Internal Server Error"}, nil
}
res := &backend.CheckHealthResult{Status: backend.HealthStatusError, Message: err.Error()}
details := map[string]string{
"verboseMessage": err.Error(),
"errorDetailsLink": "https://grafana.com/docs/grafana/latest/datasources/postgres",
}
var opErr *net.OpError
if errors.As(err, &opErr) {
res.Message = "Network error: Failed to connect to the server"
if opErr != nil && opErr.Err != nil {
errMessage := opErr.Err.Error()
if strings.HasSuffix(opErr.Err.Error(), "no such host") {
errMessage = "no such host"
}
if strings.HasSuffix(opErr.Err.Error(), "unknown port") {
errMessage = "unknown port"
}
if strings.HasSuffix(opErr.Err.Error(), "invalid port") {
errMessage = "invalid port"
}
if strings.HasSuffix(opErr.Err.Error(), "missing port in address") {
errMessage = "missing port in address"
}
if strings.HasSuffix(opErr.Err.Error(), "invalid syntax") {
errMessage = "invalid syntax found in the address"
}
res.Message += fmt.Sprintf(". Error message: %s", errMessage)
}
}
if errors.Is(err, sqleng.ErrParsingPostgresURL) {
res.Message = fmt.Sprintf("Connection string error: %s", sqleng.ErrParsingPostgresURL.Error())
if unwrappedErr := errors.Unwrap(err); unwrappedErr != nil {
details["verboseMessage"] = unwrappedErr.Error()
}
}
detailBytes, marshalErr := json.Marshal(details)
if marshalErr != nil {
return res, nil
}
res.JSONDetails = detailBytes
return res, nil
}
func logCheckHealthError(ctx context.Context, dsInfo sqleng.DataSourceInfo, err error) {
logger := log.DefaultLogger.FromContext(ctx)
configSummary := map[string]any{
"config_url_length": len(dsInfo.URL),
"config_user_length": len(dsInfo.User),
"config_database_length": len(dsInfo.Database),
"config_json_data_database_length": len(dsInfo.JsonData.Database),
"config_max_open_conns": dsInfo.JsonData.MaxOpenConns,
"config_max_idle_conns": dsInfo.JsonData.MaxIdleConns,
"config_conn_max_life_time": dsInfo.JsonData.ConnMaxLifetime,
"config_conn_timeout": dsInfo.JsonData.ConnectionTimeout,
"config_timescaledb": dsInfo.JsonData.Timescaledb,
"config_ssl_mode": dsInfo.JsonData.Mode,
"config_tls_configuration_method": dsInfo.JsonData.ConfigurationMethod,
"config_tls_skip_verify": dsInfo.JsonData.TlsSkipVerify,
"config_timezone": dsInfo.JsonData.Timezone,
"config_time_interval": dsInfo.JsonData.TimeInterval,
"config_enable_secure_proxy": dsInfo.JsonData.SecureDSProxy,
"config_allow_clear_text_passwords": dsInfo.JsonData.AllowCleartextPasswords,
"config_authentication_type": dsInfo.JsonData.AuthenticationType,
"config_ssl_root_cert_file_length": len(dsInfo.JsonData.RootCertFile),
"config_ssl_cert_file_length": len(dsInfo.JsonData.CertFile),
"config_ssl_key_file_length": len(dsInfo.JsonData.CertKeyFile),
"config_encrypt_length": len(dsInfo.JsonData.Encrypt),
"config_server_name_length": len(dsInfo.JsonData.Servername),
"config_password_length": len(dsInfo.DecryptedSecureJSONData["password"]),
"config_tls_ca_cert_length": len(dsInfo.DecryptedSecureJSONData["tlsCACert"]),
"config_tls_client_cert_length": len(dsInfo.DecryptedSecureJSONData["tlsClientCert"]),
"config_tls_client_key_length": len(dsInfo.DecryptedSecureJSONData["tlsClientKey"]),
}
configSummaryJSON, marshalError := json.Marshal(configSummary)
if marshalError != nil {
logger.Error("Check health failed", "error", err, "message_type", "ds_config_health_check_error")
return
}
logger.Error("Check health failed", "error", err, "message_type", "ds_config_health_check_error_detailed", "details", string(configSummaryJSON))
}
pkg/tsdb/grafana-postgresql-datasource/pgx/handler_checkhealth_test.go
-61
View File
@@ -1,61 +0,0 @@
package pgx
import (
"errors"
"fmt"
"net"
"testing"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/tsdb/grafana-postgresql-datasource/sqleng"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestErrToHealthCheckResult(t *testing.T) {
tests := []struct {
name string
err error
want *backend.CheckHealthResult
}{
{
name: "without error",
want: &backend.CheckHealthResult{Status: backend.HealthStatusError, Message: "Internal Server Error"},
},
{
name: "network error",
err: errors.Join(errors.New("foo"), &net.OpError{Op: "read", Net: "tcp", Err: errors.New("some op")}),
want: &backend.CheckHealthResult{
Status: backend.HealthStatusError,
Message: "Network error: Failed to connect to the server. Error message: some op",
JSONDetails: []byte(`{"errorDetailsLink":"https://grafana.com/docs/grafana/latest/datasources/postgres","verboseMessage":"foo\nread tcp: some op"}`),
},
},
{
name: "regular error",
err: errors.New("internal server error"),
want: &backend.CheckHealthResult{
Status: backend.HealthStatusError,
Message: "internal server error",
JSONDetails: []byte(`{"errorDetailsLink":"https://grafana.com/docs/grafana/latest/datasources/postgres","verboseMessage":"internal server error"}`),
},
},
{
name: "invalid port specifier error",
err: fmt.Errorf("%w %q: %w", sqleng.ErrParsingPostgresURL, `"foo.bar.co"`, errors.New(`strconv.Atoi: parsing "foo.bar.co": invalid syntax`)),
want: &backend.CheckHealthResult{
Status: backend.HealthStatusError,
Message: "Connection string error: error parsing postgres url",
JSONDetails: []byte(`{"errorDetailsLink":"https://grafana.com/docs/grafana/latest/datasources/postgres","verboseMessage":"error parsing postgres url \"\\\"foo.bar.co\\\"\": strconv.Atoi: parsing \"foo.bar.co\": invalid syntax"}`),
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := ErrToHealthCheckResult(tt.err)
require.Nil(t, err)
assert.Equal(t, string(tt.want.JSONDetails), string(got.JSONDetails))
require.Equal(t, tt.want, got)
})
}
}
pkg/tsdb/grafana-postgresql-datasource/pgx/sql_engine.go
-836
View File
@@ -1,836 +0,0 @@
package pgx
import (
"context"
"encoding/json"
"errors"
"fmt"
"net"
"runtime/debug"
"strconv"
"strings"
"sync"
"time"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/backend/gtime"
"github.com/grafana/grafana-plugin-sdk-go/backend/log"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana-plugin-sdk-go/data/sqlutil"
"github.com/grafana/grafana/pkg/tsdb/grafana-postgresql-datasource/sqleng"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgtype"
"github.com/jackc/pgx/v5/pgxpool"
)
// MetaKeyExecutedQueryString is the key where the executed query should get stored
const MetaKeyExecutedQueryString = "executedQueryString"
// SQLMacroEngine interpolates macros into sql. It takes in the Query to have access to query context and
// timeRange to be able to generate queries that use from and to.
type SQLMacroEngine interface {
Interpolate(query *backend.DataQuery, timeRange backend.TimeRange, sql string) (string, error)
}
// SqlQueryResultTransformer transforms a query result row to RowValues with proper types.
type SqlQueryResultTransformer interface {
// TransformQueryError transforms a query error.
TransformQueryError(logger log.Logger, err error) error
GetConverterList() []sqlutil.StringConverter
}
type JsonData struct {
MaxOpenConns int `json:"maxOpenConns"`
MaxIdleConns int `json:"maxIdleConns"`
ConnMaxLifetime int `json:"connMaxLifetime"`
ConnectionTimeout int `json:"connectionTimeout"`
Timescaledb bool `json:"timescaledb"`
Mode string `json:"sslmode"`
ConfigurationMethod string `json:"tlsConfigurationMethod"`
TlsSkipVerify bool `json:"tlsSkipVerify"`
RootCertFile string `json:"sslRootCertFile"`
CertFile string `json:"sslCertFile"`
CertKeyFile string `json:"sslKeyFile"`
Timezone string `json:"timezone"`
Encrypt string `json:"encrypt"`
Servername string `json:"servername"`
TimeInterval string `json:"timeInterval"`
Database string `json:"database"`
SecureDSProxy bool `json:"enableSecureSocksProxy"`
SecureDSProxyUsername string `json:"secureSocksProxyUsername"`
AllowCleartextPasswords bool `json:"allowCleartextPasswords"`
AuthenticationType string `json:"authenticationType"`
}
type DataPluginConfiguration struct {
DSInfo sqleng.DataSourceInfo
TimeColumnNames []string
MetricColumnTypes []string
RowLimit int64
}
type DataSourceHandler struct {
macroEngine SQLMacroEngine
queryResultTransformer SqlQueryResultTransformer
timeColumnNames []string
metricColumnTypes []string
log log.Logger
dsInfo sqleng.DataSourceInfo
rowLimit int64
userError string
pool *pgxpool.Pool
}
type QueryJson struct {
RawSql string `json:"rawSql"`
Fill bool `json:"fill"`
FillInterval float64 `json:"fillInterval"`
FillMode string `json:"fillMode"`
FillValue float64 `json:"fillValue"`
Format string `json:"format"`
}
func (e *DataSourceHandler) TransformQueryError(logger log.Logger, err error) error {
// OpError is the error type usually returned by functions in the net
// package. It describes the operation, network type, and address of
// an error. We log this error rather than return it to the client
// for security purposes.
var opErr *net.OpError
if errors.As(err, &opErr) {
return fmt.Errorf("failed to connect to server - %s", e.userError)
}
return e.queryResultTransformer.TransformQueryError(logger, err)
}
func NewQueryDataHandler(userFacingDefaultError string, p *pgxpool.Pool, config DataPluginConfiguration, queryResultTransformer SqlQueryResultTransformer,
macroEngine SQLMacroEngine, log log.Logger) (*DataSourceHandler, error) {
queryDataHandler := DataSourceHandler{
queryResultTransformer: queryResultTransformer,
macroEngine: macroEngine,
timeColumnNames: []string{"time"},
log: log,
dsInfo: config.DSInfo,
rowLimit: config.RowLimit,
userError: userFacingDefaultError,
}
if len(config.TimeColumnNames) > 0 {
queryDataHandler.timeColumnNames = config.TimeColumnNames
}
if len(config.MetricColumnTypes) > 0 {
queryDataHandler.metricColumnTypes = config.MetricColumnTypes
}
queryDataHandler.pool = p
return &queryDataHandler, nil
}
type DBDataResponse struct {
dataResponse backend.DataResponse
refID string
}
func (e *DataSourceHandler) Dispose() {
e.log.Debug("Disposing DB...")
if e.pool != nil {
e.pool.Close()
}
e.log.Debug("DB disposed")
}
func (e *DataSourceHandler) Ping(ctx context.Context) error {
return e.pool.Ping(ctx)
}
func (e *DataSourceHandler) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) {
result := backend.NewQueryDataResponse()
ch := make(chan DBDataResponse, len(req.Queries))
var wg sync.WaitGroup
// Execute each query in a goroutine and wait for them to finish afterwards
for _, query := range req.Queries {
queryjson := QueryJson{
Fill: false,
Format: "time_series",
}
err := json.Unmarshal(query.JSON, &queryjson)
if err != nil {
return nil, backend.DownstreamErrorf("error unmarshal query json: %s", err.Error())
}
// the fill-params are only stored inside this function, during query-interpolation. we do not support
// sending them in "from the outside"
if queryjson.Fill || queryjson.FillInterval != 0.0 || queryjson.FillMode != "" || queryjson.FillValue != 0.0 {
return nil, backend.DownstreamErrorf("query fill-parameters not supported")
}
if queryjson.RawSql == "" {
continue
}
wg.Add(1)
go e.executeQuery(ctx, query, &wg, ch, queryjson)
}
wg.Wait()
// Read results from channels
close(ch)
result.Responses = make(map[string]backend.DataResponse)
for queryResult := range ch {
result.Responses[queryResult.refID] = queryResult.dataResponse
}
return result, nil
}
func (e *DataSourceHandler) handleQueryError(frameErr string, err error, query string, source backend.ErrorSource, ch chan DBDataResponse, queryResult DBDataResponse) {
var emptyFrame data.Frame
emptyFrame.SetMeta(&data.FrameMeta{ExecutedQueryString: query})
if isDownstreamError(err) {
source = backend.ErrorSourceDownstream
}
queryResult.dataResponse.Error = fmt.Errorf("%s: %w", frameErr, err)
queryResult.dataResponse.ErrorSource = source
queryResult.dataResponse.Frames = data.Frames{&emptyFrame}
ch <- queryResult
}
func (e *DataSourceHandler) handlePanic(logger log.Logger, queryResult *DBDataResponse, ch chan DBDataResponse) {
if r := recover(); r != nil {
logger.Error("ExecuteQuery panic", "error", r, "stack", string(debug.Stack()))
if theErr, ok := r.(error); ok {
queryResult.dataResponse.Error = theErr
queryResult.dataResponse.ErrorSource = backend.ErrorSourcePlugin
} else if theErrString, ok := r.(string); ok {
queryResult.dataResponse.Error = errors.New(theErrString)
queryResult.dataResponse.ErrorSource = backend.ErrorSourcePlugin
} else {
queryResult.dataResponse.Error = fmt.Errorf("unexpected error - %s", e.userError)
queryResult.dataResponse.ErrorSource = backend.ErrorSourceDownstream
}
ch <- *queryResult
}
}
// Interpolate provides global macros/substitutions for all sql datasources.
var Interpolate = func(query backend.DataQuery, timeRange backend.TimeRange, timeInterval string, sql string) string {
interval := query.Interval
sql = strings.ReplaceAll(sql, "$__interval_ms", strconv.FormatInt(interval.Milliseconds(), 10))
sql = strings.ReplaceAll(sql, "$__interval", gtime.FormatInterval(interval))
sql = strings.ReplaceAll(sql, "$__unixEpochFrom()", fmt.Sprintf("%d", timeRange.From.UTC().Unix()))
sql = strings.ReplaceAll(sql, "$__unixEpochTo()", fmt.Sprintf("%d", timeRange.To.UTC().Unix()))
return sql
}
func (e *DataSourceHandler) execQuery(ctx context.Context, query string) ([]*pgconn.Result, error) {
c, err := e.pool.Acquire(ctx)
if err != nil {
return nil, backend.DownstreamErrorf("failed to acquire connection: %w", err)
}
defer c.Release()
mrr := c.Conn().PgConn().Exec(ctx, query)
// Close returns the first error that occurred during the MultiResultReader's use. We will log that later.
defer mrr.Close() //nolint:errcheck
return mrr.ReadAll()
}
func (e *DataSourceHandler) executeQuery(queryContext context.Context, query backend.DataQuery, wg *sync.WaitGroup,
ch chan DBDataResponse, queryJSON QueryJson) {
defer wg.Done()
queryResult := DBDataResponse{
dataResponse: backend.DataResponse{},
refID: query.RefID,
}
logger := e.log.FromContext(queryContext)
defer e.handlePanic(logger, &queryResult, ch)
if queryJSON.RawSql == "" {
panic("Query model property rawSql should not be empty at this point")
}
// global substitutions
interpolatedQuery := Interpolate(query, query.TimeRange, e.dsInfo.JsonData.TimeInterval, queryJSON.RawSql)
// data source specific substitutions
interpolatedQuery, err := e.macroEngine.Interpolate(&query, query.TimeRange, interpolatedQuery)
if err != nil {
e.handleQueryError("interpolation failed", e.TransformQueryError(logger, err), interpolatedQuery, backend.ErrorSourceDownstream, ch, queryResult)
return
}
results, err := e.execQuery(queryContext, interpolatedQuery)
if err != nil {
e.handleQueryError("db query error", e.TransformQueryError(logger, err), interpolatedQuery, backend.ErrorSourceDownstream, ch, queryResult)
return
}
qm, err := e.newProcessCfg(queryContext, query, results, interpolatedQuery)
if err != nil {
e.handleQueryError("failed to get configurations", err, interpolatedQuery, backend.ErrorSourceDownstream, ch, queryResult)
return
}
frame, err := convertResultsToFrame(results, e.rowLimit)
if err != nil {
e.handleQueryError("convert frame from rows error", err, interpolatedQuery, backend.ErrorSourceDownstream, ch, queryResult)
return
}
e.processFrame(frame, qm, queryResult, ch, logger)
}
// dataQueryFormat is the type of query.
type dataQueryFormat string
const (
// dataQueryFormatTable identifies a table query (default).
dataQueryFormatTable dataQueryFormat = "table"
// dataQueryFormatSeries identifies a time series query.
dataQueryFormatSeries dataQueryFormat = "time_series"
)
type dataQueryModel struct {
InterpolatedQuery string // property not set until after Interpolate()
Format dataQueryFormat
TimeRange backend.TimeRange
FillMissing *data.FillMissing // property not set until after Interpolate()
Interval time.Duration
columnNames []string
columnTypes []string
timeIndex int
timeEndIndex int
metricIndex int
metricPrefix bool
queryContext context.Context
}
func convertSQLTimeColumnsToEpochMS(frame *data.Frame, qm *dataQueryModel) error {
if qm.timeIndex != -1 {
if err := convertSQLTimeColumnToEpochMS(frame, qm.timeIndex); err != nil {
return fmt.Errorf("%v: %w", "failed to convert time column", err)
}
}
if qm.timeEndIndex != -1 {
if err := convertSQLTimeColumnToEpochMS(frame, qm.timeEndIndex); err != nil {
return fmt.Errorf("%v: %w", "failed to convert timeend column", err)
}
}
return nil
}
func (e *DataSourceHandler) processFrame(frame *data.Frame, qm *dataQueryModel, queryResult DBDataResponse, ch chan DBDataResponse, logger log.Logger) {
if frame.Meta == nil {
frame.Meta = &data.FrameMeta{}
}
frame.Meta.ExecutedQueryString = qm.InterpolatedQuery
// If no rows were returned, clear any previously set `Fields` with a single empty `data.Field` slice.
// Then assign `queryResult.dataResponse.Frames` the current single frame with that single empty Field.
// This assures 1) our visualization doesn't display unwanted empty fields, and also that 2)
// additionally-needed frame data stays intact and is correctly passed to our visulization.
if frame.Rows() == 0 {
frame.Fields = []*data.Field{}
queryResult.dataResponse.Frames = data.Frames{frame}
ch <- queryResult
return
}
if err := convertSQLTimeColumnsToEpochMS(frame, qm); err != nil {
e.handleQueryError("converting time columns failed", err, qm.InterpolatedQuery, backend.ErrorSourceDownstream, ch, queryResult)
return
}
if qm.Format == dataQueryFormatSeries {
// time series has to have time column
if qm.timeIndex == -1 {
e.handleQueryError("db has no time column", errors.New("time column is missing; make sure your data includes a time column for time series format or switch to a table format that doesn't require it"), qm.InterpolatedQuery, backend.ErrorSourceDownstream, ch, queryResult)
return
}
// Make sure to name the time field 'Time' to be backward compatible with Grafana pre-v8.
frame.Fields[qm.timeIndex].Name = data.TimeSeriesTimeFieldName
for i := range qm.columnNames {
if i == qm.timeIndex || i == qm.metricIndex {
continue
}
if t := frame.Fields[i].Type(); t == data.FieldTypeString || t == data.FieldTypeNullableString {
continue
}
var err error
if frame, err = convertSQLValueColumnToFloat(frame, i); err != nil {
e.handleQueryError("convert value to float failed", err, qm.InterpolatedQuery, backend.ErrorSourceDownstream, ch, queryResult)
return
}
}
tsSchema := frame.TimeSeriesSchema()
if tsSchema.Type == data.TimeSeriesTypeLong {
var err error
originalData := frame
frame, err = data.LongToWide(frame, qm.FillMissing)
if err != nil {
e.handleQueryError("failed to convert long to wide series when converting from dataframe", err, qm.InterpolatedQuery, backend.ErrorSourceDownstream, ch, queryResult)
return
}
// Before 8x, a special metric column was used to name time series. The LongToWide transforms that into a metric label on the value field.
// But that makes series name have both the value column name AND the metric name. So here we are removing the metric label here and moving it to the
// field name to get the same naming for the series as pre v8
if len(originalData.Fields) == 3 {
for _, field := range frame.Fields {
if len(field.Labels) == 1 { // 7x only supported one label
name, ok := field.Labels["metric"]
if ok {
field.Name = name
field.Labels = nil
}
}
}
}
}
if qm.FillMissing != nil {
// we align the start-time
startUnixTime := qm.TimeRange.From.Unix() / int64(qm.Interval.Seconds()) * int64(qm.Interval.Seconds())
alignedTimeRange := backend.TimeRange{
From: time.Unix(startUnixTime, 0),
To: qm.TimeRange.To,
}
var err error
frame, err = sqlutil.ResampleWideFrame(frame, qm.FillMissing, alignedTimeRange, qm.Interval)
if err != nil {
logger.Error("Failed to resample dataframe", "err", err)
frame.AppendNotices(data.Notice{Text: "Failed to resample dataframe", Severity: data.NoticeSeverityWarning})
return
}
}
}
queryResult.dataResponse.Frames = data.Frames{frame}
ch <- queryResult
}
func (e *DataSourceHandler) newProcessCfg(queryContext context.Context, query backend.DataQuery,
results []*pgconn.Result, interpolatedQuery string) (*dataQueryModel, error) {
columnNames := []string{}
columnTypes := []string{}
// The results will contain column information in the metadata
for _, result := range results {
// Get column names from the result metadata
for _, field := range result.FieldDescriptions {
columnNames = append(columnNames, field.Name)
pqtype, ok := pgtype.NewMap().TypeForOID(field.DataTypeOID)
if !ok {
// Handle special cases for field types
switch field.DataTypeOID {
case pgtype.TimetzOID:
columnTypes = append(columnTypes, "timetz")
case 790:
columnTypes = append(columnTypes, "money")
default:
columnTypes = append(columnTypes, "unknown")
}
} else {
columnTypes = append(columnTypes, pqtype.Name)
}
}
}
qm := &dataQueryModel{
columnTypes: columnTypes,
columnNames: columnNames,
timeIndex: -1,
timeEndIndex: -1,
metricIndex: -1,
metricPrefix: false,
queryContext: queryContext,
}
queryJSON := QueryJson{}
err := json.Unmarshal(query.JSON, &queryJSON)
if err != nil {
return nil, err
}
if queryJSON.Fill {
qm.FillMissing = &data.FillMissing{}
qm.Interval = time.Duration(queryJSON.FillInterval * float64(time.Second))
switch strings.ToLower(queryJSON.FillMode) {
case "null":
qm.FillMissing.Mode = data.FillModeNull
case "previous":
qm.FillMissing.Mode = data.FillModePrevious
case "value":
qm.FillMissing.Mode = data.FillModeValue
qm.FillMissing.Value = queryJSON.FillValue
default:
}
}
qm.TimeRange.From = query.TimeRange.From.UTC()
qm.TimeRange.To = query.TimeRange.To.UTC()
// Default to time_series if no format is provided
switch queryJSON.Format {
case "table":
qm.Format = dataQueryFormatTable
case "time_series":
fallthrough
default:
qm.Format = dataQueryFormatSeries
}
for i, col := range qm.columnNames {
for _, tc := range e.timeColumnNames {
if col == tc {
qm.timeIndex = i
break
}
}
if qm.Format == dataQueryFormatTable && strings.EqualFold(col, "timeend") {
qm.timeEndIndex = i
continue
}
switch col {
case "metric":
qm.metricIndex = i
default:
if qm.metricIndex == -1 {
columnType := qm.columnTypes[i]
for _, mct := range e.metricColumnTypes {
if columnType == mct {
qm.metricIndex = i
continue
}
}
}
}
}
qm.InterpolatedQuery = interpolatedQuery
return qm, nil
}
func convertResultsToFrame(results []*pgconn.Result, rowLimit int64) (*data.Frame, error) {
m := pgtype.NewMap()
// Find the first SELECT result to establish the frame structure
var firstSelectResult *pgconn.Result
for _, result := range results {
if result.CommandTag.Select() {
firstSelectResult = result
break
}
}
// If no SELECT results found, return empty frame
if firstSelectResult == nil {
return data.NewFrame(""), nil
}
// Create frame structure based on the first SELECT result
fields := make(data.Fields, len(firstSelectResult.FieldDescriptions))
fieldTypes, err := getFieldTypesFromDescriptions(firstSelectResult.FieldDescriptions, m)
if err != nil {
return nil, err
}
for i, v := range firstSelectResult.FieldDescriptions {
fields[i] = data.NewFieldFromFieldType(fieldTypes[i], 0)
fields[i].Name = v.Name
}
frame := *data.NewFrame("", fields...)
// Process all SELECT results, but validate column compatibility
for _, result := range results {
// Skip non-select statements
if !result.CommandTag.Select() {
continue
}
// Validate that this result has the same structure as the frame
if len(result.FieldDescriptions) != len(frame.Fields) {
return nil, fmt.Errorf("incompatible result structure: expected %d columns, got %d columns",
len(frame.Fields), len(result.FieldDescriptions))
}
// Validate column names and types match
for i, fd := range result.FieldDescriptions {
if fd.Name != frame.Fields[i].Name {
return nil, fmt.Errorf("column name mismatch at position %d: expected %q, got %q",
i, frame.Fields[i].Name, fd.Name)
}
}
fieldDescriptions := result.FieldDescriptions
for rowIdx := range result.Rows {
if rowIdx == int(rowLimit) {
frame.AppendNotices(data.Notice{
Severity: data.NoticeSeverityWarning,
Text: fmt.Sprintf("Results have been limited to %v because the SQL row limit was reached", rowLimit),
})
break
}
row := make([]any, len(fieldDescriptions))
for colIdx, fd := range fieldDescriptions {
rawValue := result.Rows[rowIdx][colIdx]
if rawValue == nil {
row[colIdx] = nil
continue
}
convertedValue, err := convertPostgresValue(rawValue, fd, m)
if err != nil {
return nil, err
}
row[colIdx] = convertedValue
}
// Validate row length matches frame field count before appending
if len(row) != len(frame.Fields) {
return nil, fmt.Errorf("row data length mismatch: expected %d values, got %d values",
len(frame.Fields), len(row))
}
frame.AppendRow(row...)
}
}
return &frame, nil
}
// convertPostgresValue converts a raw PostgreSQL value to the appropriate Go type
func convertPostgresValue(rawValue []byte, fd pgconn.FieldDescription, m *pgtype.Map) (interface{}, error) {
dataTypeOID := fd.DataTypeOID
format := fd.Format
// Convert based on type
switch fd.DataTypeOID {
case pgtype.Int2OID:
var d *int16
scanPlan := m.PlanScan(dataTypeOID, format, &d)
err := scanPlan.Scan(rawValue, &d)
if err != nil {
return nil, err
}
return d, nil
case pgtype.Int4OID:
var d *int32
scanPlan := m.PlanScan(dataTypeOID, format, &d)
err := scanPlan.Scan(rawValue, &d)
if err != nil {
return nil, err
}
return d, nil
case pgtype.Int8OID:
var d *int64
scanPlan := m.PlanScan(dataTypeOID, format, &d)
err := scanPlan.Scan(rawValue, &d)
if err != nil {
return nil, err
}
return d, nil
case pgtype.NumericOID, pgtype.Float8OID, pgtype.Float4OID:
var d *float64
scanPlan := m.PlanScan(dataTypeOID, format, &d)
err := scanPlan.Scan(rawValue, &d)
if err != nil {
return nil, err
}
return d, nil
case pgtype.BoolOID:
var d *bool
scanPlan := m.PlanScan(dataTypeOID, format, &d)
err := scanPlan.Scan(rawValue, &d)
if err != nil {
return nil, err
}
return d, nil
case pgtype.ByteaOID:
d, err := pgtype.ByteaCodec.DecodeValue(pgtype.ByteaCodec{}, m, dataTypeOID, format, rawValue)
if err != nil {
return nil, err
}
str := string(d.([]byte))
return &str, nil
case pgtype.TimestampOID, pgtype.TimestamptzOID, pgtype.DateOID:
var d *time.Time
scanPlan := m.PlanScan(dataTypeOID, format, &d)
err := scanPlan.Scan(rawValue, &d)
if err != nil {
return nil, err
}
return d, nil
case pgtype.TimeOID, pgtype.TimetzOID:
var d *string
scanPlan := m.PlanScan(dataTypeOID, format, &d)
err := scanPlan.Scan(rawValue, &d)
if err != nil {
return nil, err
}
return d, nil
case pgtype.JSONOID, pgtype.JSONBOID:
var d *string
scanPlan := m.PlanScan(dataTypeOID, format, &d)
err := scanPlan.Scan(rawValue, &d)
if err != nil {
return nil, err
}
// Handle null JSON values
if d == nil {
return nil, nil
}
j := json.RawMessage(*d)
return &j, nil
default:
var d *string
scanPlan := m.PlanScan(dataTypeOID, format, &d)
err := scanPlan.Scan(rawValue, &d)
if err != nil {
return nil, err
}
return d, nil
}
}
func getFieldTypesFromDescriptions(fieldDescriptions []pgconn.FieldDescription, m *pgtype.Map) ([]data.FieldType, error) {
fieldTypes := make([]data.FieldType, len(fieldDescriptions))
for i, v := range fieldDescriptions {
typeName, ok := m.TypeForOID(v.DataTypeOID)
if !ok {
fieldTypes[i] = data.FieldTypeNullableString
} else {
switch typeName.Name {
case "int2":
fieldTypes[i] = data.FieldTypeNullableInt16
case "int4":
fieldTypes[i] = data.FieldTypeNullableInt32
case "int8":
fieldTypes[i] = data.FieldTypeNullableInt64
case "float4", "float8", "numeric":
fieldTypes[i] = data.FieldTypeNullableFloat64
case "bool":
fieldTypes[i] = data.FieldTypeNullableBool
case "timestamptz", "timestamp", "date":
fieldTypes[i] = data.FieldTypeNullableTime
case "json", "jsonb":
fieldTypes[i] = data.FieldTypeNullableJSON
default:
fieldTypes[i] = data.FieldTypeNullableString
}
}
}
return fieldTypes, nil
}
// convertSQLTimeColumnToEpochMS converts column named time to unix timestamp in milliseconds
// to make native datetime types and epoch dates work in annotation and table queries.
func convertSQLTimeColumnToEpochMS(frame *data.Frame, timeIndex int) error {
if timeIndex < 0 || timeIndex >= len(frame.Fields) {
return fmt.Errorf("timeIndex %d is out of range", timeIndex)
}
origin := frame.Fields[timeIndex]
valueType := origin.Type()
if valueType == data.FieldTypeTime || valueType == data.FieldTypeNullableTime {
return nil
}
newField := data.NewFieldFromFieldType(data.FieldTypeNullableTime, 0)
newField.Name = origin.Name
newField.Labels = origin.Labels
valueLength := origin.Len()
for i := 0; i < valueLength; i++ {
v, err := origin.NullableFloatAt(i)
if err != nil {
return fmt.Errorf("unable to convert data to a time field")
}
if v == nil {
newField.Append(nil)
} else {
timestamp := time.Unix(0, int64(epochPrecisionToMS(*v))*int64(time.Millisecond))
newField.Append(&timestamp)
}
}
frame.Fields[timeIndex] = newField
return nil
}
// convertSQLValueColumnToFloat converts timeseries value column to float.
func convertSQLValueColumnToFloat(frame *data.Frame, Index int) (*data.Frame, error) {
if Index < 0 || Index >= len(frame.Fields) {
return frame, fmt.Errorf("metricIndex %d is out of range", Index)
}
origin := frame.Fields[Index]
valueType := origin.Type()
if valueType == data.FieldTypeFloat64 || valueType == data.FieldTypeNullableFloat64 {
return frame, nil
}
newField := data.NewFieldFromFieldType(data.FieldTypeNullableFloat64, origin.Len())
newField.Name = origin.Name
newField.Labels = origin.Labels
for i := 0; i < origin.Len(); i++ {
v, err := origin.NullableFloatAt(i)
if err != nil {
return frame, err
}
newField.Set(i, v)
}
frame.Fields[Index] = newField
return frame, nil
}
// epochPrecisionToMS converts epoch precision to millisecond, if needed.
// Only seconds to milliseconds supported right now
func epochPrecisionToMS(value float64) float64 {
s := strconv.FormatFloat(value, 'e', -1, 64)
if strings.HasSuffix(s, "e+09") {
return value * float64(1e3)
}
if strings.HasSuffix(s, "e+18") {
return value / float64(time.Millisecond)
}
return value
}
func isDownstreamError(err error) bool {
if backend.IsDownstreamError(err) {
return true
}
resultProcessingDownstreamErrors := []error{
data.ErrorInputFieldsWithoutRows,
data.ErrorSeriesUnsorted,
data.ErrorNullTimeValues,
}
for _, e := range resultProcessingDownstreamErrors {
if errors.Is(err, e) {
return true
}
}
return false
}
pkg/tsdb/grafana-postgresql-datasource/pgx/sql_engine_test.go
-681
View File
@@ -1,681 +0,0 @@
package pgx
import (
"fmt"
"net"
"testing"
"time"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana-plugin-sdk-go/data/sqlutil"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgtype"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana-plugin-sdk-go/backend/log"
)
func Pointer[T any](v T) *T { return &v }
func TestSQLEngine(t *testing.T) {
dt := time.Date(2018, 3, 14, 21, 20, 6, int(527345*time.Microsecond), time.UTC)
t.Run("Handle interpolating $__interval and $__interval_ms", func(t *testing.T) {
from := time.Date(2018, 4, 12, 18, 0, 0, 0, time.UTC)
to := from.Add(5 * time.Minute)
timeRange := backend.TimeRange{From: from, To: to}
text := "$__interval $__timeGroupAlias(time,$__interval) $__interval_ms"
t.Run("interpolate 10 minutes $__interval", func(t *testing.T) {
query := backend.DataQuery{JSON: []byte("{}"), MaxDataPoints: 1500, Interval: time.Minute * 10}
sql := Interpolate(query, timeRange, "", text)
require.Equal(t, "10m $__timeGroupAlias(time,10m) 600000", sql)
})
t.Run("interpolate 4seconds $__interval", func(t *testing.T) {
query := backend.DataQuery{JSON: []byte("{}"), MaxDataPoints: 1500, Interval: time.Second * 4}
sql := Interpolate(query, timeRange, "", text)
require.Equal(t, "4s $__timeGroupAlias(time,4s) 4000", sql)
})
t.Run("interpolate 200 milliseconds $__interval", func(t *testing.T) {
query := backend.DataQuery{JSON: []byte("{}"), MaxDataPoints: 1500, Interval: time.Millisecond * 200}
sql := Interpolate(query, timeRange, "", text)
require.Equal(t, "200ms $__timeGroupAlias(time,200ms) 200", sql)
})
})
t.Run("Given a time range between 2018-04-12 00:00 and 2018-04-12 00:05", func(t *testing.T) {
from := time.Date(2018, 4, 12, 18, 0, 0, 0, time.UTC)
to := from.Add(5 * time.Minute)
timeRange := backend.TimeRange{From: from, To: to}
query := backend.DataQuery{JSON: []byte("{}"), MaxDataPoints: 1500, Interval: time.Second * 60}
t.Run("interpolate __unixEpochFrom function", func(t *testing.T) {
sql := Interpolate(query, timeRange, "", "select $__unixEpochFrom()")
require.Equal(t, fmt.Sprintf("select %d", from.Unix()), sql)
})
t.Run("interpolate __unixEpochTo function", func(t *testing.T) {
sql := Interpolate(query, timeRange, "", "select $__unixEpochTo()")
require.Equal(t, fmt.Sprintf("select %d", to.Unix()), sql)
})
})
t.Run("Given row values with int64 as time columns", func(t *testing.T) {
tSeconds := dt.Unix()
tMilliseconds := dt.UnixNano() / 1e6
tNanoSeconds := dt.UnixNano()
var nilPointer *int64
originFrame := data.NewFrame("",
data.NewField("time1", nil, []int64{
tSeconds,
}),
data.NewField("time2", nil, []*int64{
Pointer(tSeconds),
}),
data.NewField("time3", nil, []int64{
tMilliseconds,
}),
data.NewField("time4", nil, []*int64{
Pointer(tMilliseconds),
}),
data.NewField("time5", nil, []int64{
tNanoSeconds,
}),
data.NewField("time6", nil, []*int64{
Pointer(tNanoSeconds),
}),
data.NewField("time7", nil, []*int64{
nilPointer,
}),
)
for i := 0; i < len(originFrame.Fields); i++ {
err := convertSQLTimeColumnToEpochMS(originFrame, i)
require.NoError(t, err)
}
require.Equal(t, dt.Unix(), (*originFrame.Fields[0].At(0).(*time.Time)).Unix())
require.Equal(t, dt.Unix(), (*originFrame.Fields[1].At(0).(*time.Time)).Unix())
require.Equal(t, dt.Unix(), (*originFrame.Fields[2].At(0).(*time.Time)).Unix())
require.Equal(t, dt.Unix(), (*originFrame.Fields[3].At(0).(*time.Time)).Unix())
require.Equal(t, dt.Unix(), (*originFrame.Fields[4].At(0).(*time.Time)).Unix())
require.Equal(t, dt.Unix(), (*originFrame.Fields[5].At(0).(*time.Time)).Unix())
require.Nil(t, originFrame.Fields[6].At(0))
})
t.Run("Given row values with uint64 as time columns", func(t *testing.T) {
tSeconds := uint64(dt.Unix())
tMilliseconds := uint64(dt.UnixNano() / 1e6)
tNanoSeconds := uint64(dt.UnixNano())
var nilPointer *uint64
originFrame := data.NewFrame("",
data.NewField("time1", nil, []uint64{
tSeconds,
}),
data.NewField("time2", nil, []*uint64{
Pointer(tSeconds),
}),
data.NewField("time3", nil, []uint64{
tMilliseconds,
}),
data.NewField("time4", nil, []*uint64{
Pointer(tMilliseconds),
}),
data.NewField("time5", nil, []uint64{
tNanoSeconds,
}),
data.NewField("time6", nil, []*uint64{
Pointer(tNanoSeconds),
}),
data.NewField("time7", nil, []*uint64{
nilPointer,
}),
)
for i := 0; i < len(originFrame.Fields); i++ {
err := convertSQLTimeColumnToEpochMS(originFrame, i)
require.NoError(t, err)
}
require.Equal(t, dt.Unix(), (*originFrame.Fields[0].At(0).(*time.Time)).Unix())
require.Equal(t, dt.Unix(), (*originFrame.Fields[1].At(0).(*time.Time)).Unix())
require.Equal(t, dt.Unix(), (*originFrame.Fields[2].At(0).(*time.Time)).Unix())
require.Equal(t, dt.Unix(), (*originFrame.Fields[3].At(0).(*time.Time)).Unix())
require.Equal(t, dt.Unix(), (*originFrame.Fields[4].At(0).(*time.Time)).Unix())
require.Equal(t, dt.Unix(), (*originFrame.Fields[5].At(0).(*time.Time)).Unix())
require.Nil(t, originFrame.Fields[6].At(0))
})
t.Run("Given row values with int32 as time columns", func(t *testing.T) {
tSeconds := int32(dt.Unix())
var nilInt *int32
originFrame := data.NewFrame("",
data.NewField("time1", nil, []int32{
tSeconds,
}),
data.NewField("time2", nil, []*int32{
Pointer(tSeconds),
}),
data.NewField("time7", nil, []*int32{
nilInt,
}),
)
for i := 0; i < 3; i++ {
err := convertSQLTimeColumnToEpochMS(originFrame, i)
require.NoError(t, err)
}
require.Equal(t, dt.Unix(), (*originFrame.Fields[0].At(0).(*time.Time)).Unix())
require.Equal(t, dt.Unix(), (*originFrame.Fields[1].At(0).(*time.Time)).Unix())
require.Nil(t, originFrame.Fields[2].At(0))
})
t.Run("Given row values with uint32 as time columns", func(t *testing.T) {
tSeconds := uint32(dt.Unix())
var nilInt *uint32
originFrame := data.NewFrame("",
data.NewField("time1", nil, []uint32{
tSeconds,
}),
data.NewField("time2", nil, []*uint32{
Pointer(tSeconds),
}),
data.NewField("time7", nil, []*uint32{
nilInt,
}),
)
for i := 0; i < len(originFrame.Fields); i++ {
err := convertSQLTimeColumnToEpochMS(originFrame, i)
require.NoError(t, err)
}
require.Equal(t, dt.Unix(), (*originFrame.Fields[0].At(0).(*time.Time)).Unix())
require.Equal(t, dt.Unix(), (*originFrame.Fields[1].At(0).(*time.Time)).Unix())
require.Nil(t, originFrame.Fields[2].At(0))
})
t.Run("Given row values with float64 as time columns", func(t *testing.T) {
tSeconds := float64(dt.UnixNano()) / float64(time.Second)
tMilliseconds := float64(dt.UnixNano()) / float64(time.Millisecond)
tNanoSeconds := float64(dt.UnixNano())
var nilPointer *float64
originFrame := data.NewFrame("",
data.NewField("time1", nil, []float64{
tSeconds,
}),
data.NewField("time2", nil, []*float64{
Pointer(tSeconds),
}),
data.NewField("time3", nil, []float64{
tMilliseconds,
}),
data.NewField("time4", nil, []*float64{
Pointer(tMilliseconds),
}),
data.NewField("time5", nil, []float64{
tNanoSeconds,
}),
data.NewField("time6", nil, []*float64{
Pointer(tNanoSeconds),
}),
data.NewField("time7", nil, []*float64{
nilPointer,
}),
)
for i := 0; i < len(originFrame.Fields); i++ {
err := convertSQLTimeColumnToEpochMS(originFrame, i)
require.NoError(t, err)
}
require.Equal(t, dt.Unix(), (*originFrame.Fields[0].At(0).(*time.Time)).Unix())
require.Equal(t, dt.Unix(), (*originFrame.Fields[1].At(0).(*time.Time)).Unix())
require.Equal(t, dt.Unix(), (*originFrame.Fields[2].At(0).(*time.Time)).Unix())
require.Equal(t, dt.Unix(), (*originFrame.Fields[3].At(0).(*time.Time)).Unix())
require.Equal(t, dt.Unix(), (*originFrame.Fields[4].At(0).(*time.Time)).Unix())
require.Equal(t, dt.Unix(), (*originFrame.Fields[5].At(0).(*time.Time)).Unix())
require.Nil(t, originFrame.Fields[6].At(0))
})
t.Run("Given row values with float32 as time columns", func(t *testing.T) {
tSeconds := float32(dt.Unix())
var nilInt *float32
originFrame := data.NewFrame("",
data.NewField("time1", nil, []float32{
tSeconds,
}),
data.NewField("time2", nil, []*float32{
Pointer(tSeconds),
}),
data.NewField("time7", nil, []*float32{
nilInt,
}),
)
for i := 0; i < len(originFrame.Fields); i++ {
err := convertSQLTimeColumnToEpochMS(originFrame, i)
require.NoError(t, err)
}
require.Equal(t, int64(tSeconds), (*originFrame.Fields[0].At(0).(*time.Time)).Unix())
require.Equal(t, int64(tSeconds), (*originFrame.Fields[1].At(0).(*time.Time)).Unix())
require.Nil(t, originFrame.Fields[2].At(0))
})
t.Run("Given row with value columns, would be converted to float64", func(t *testing.T) {
originFrame := data.NewFrame("",
data.NewField("value1", nil, []int64{
int64(1),
}),
data.NewField("value2", nil, []*int64{
Pointer(int64(1)),
}),
data.NewField("value3", nil, []int32{
int32(1),
}),
data.NewField("value4", nil, []*int32{
Pointer(int32(1)),
}),
data.NewField("value5", nil, []int16{
int16(1),
}),
data.NewField("value6", nil, []*int16{
Pointer(int16(1)),
}),
data.NewField("value7", nil, []int8{
int8(1),
}),
data.NewField("value8", nil, []*int8{
Pointer(int8(1)),
}),
data.NewField("value9", nil, []float64{
float64(1),
}),
data.NewField("value10", nil, []*float64{
Pointer(1.0),
}),
data.NewField("value11", nil, []float32{
float32(1),
}),
data.NewField("value12", nil, []*float32{
Pointer(float32(1)),
}),
data.NewField("value13", nil, []uint64{
uint64(1),
}),
data.NewField("value14", nil, []*uint64{
Pointer(uint64(1)),
}),
data.NewField("value15", nil, []uint32{
uint32(1),
}),
data.NewField("value16", nil, []*uint32{
Pointer(uint32(1)),
}),
data.NewField("value17", nil, []uint16{
uint16(1),
}),
data.NewField("value18", nil, []*uint16{
Pointer(uint16(1)),
}),
data.NewField("value19", nil, []uint8{
uint8(1),
}),
data.NewField("value20", nil, []*uint8{
Pointer(uint8(1)),
}),
)
for i := 0; i < len(originFrame.Fields); i++ {
_, err := convertSQLValueColumnToFloat(originFrame, i)
require.NoError(t, err)
if i == 8 {
require.Equal(t, float64(1), originFrame.Fields[i].At(0).(float64))
} else {
require.NotNil(t, originFrame.Fields[i].At(0).(*float64))
require.Equal(t, float64(1), *originFrame.Fields[i].At(0).(*float64))
}
}
})
t.Run("Given row with nil value columns", func(t *testing.T) {
var int64NilPointer *int64
var int32NilPointer *int32
var int16NilPointer *int16
var int8NilPointer *int8
var float64NilPointer *float64
var float32NilPointer *float32
var uint64NilPointer *uint64
var uint32NilPointer *uint32
var uint16NilPointer *uint16
var uint8NilPointer *uint8
originFrame := data.NewFrame("",
data.NewField("value1", nil, []*int64{
int64NilPointer,
}),
data.NewField("value2", nil, []*int32{
int32NilPointer,
}),
data.NewField("value3", nil, []*int16{
int16NilPointer,
}),
data.NewField("value4", nil, []*int8{
int8NilPointer,
}),
data.NewField("value5", nil, []*float64{
float64NilPointer,
}),
data.NewField("value6", nil, []*float32{
float32NilPointer,
}),
data.NewField("value7", nil, []*uint64{
uint64NilPointer,
}),
data.NewField("value8", nil, []*uint32{
uint32NilPointer,
}),
data.NewField("value9", nil, []*uint16{
uint16NilPointer,
}),
data.NewField("value10", nil, []*uint8{
uint8NilPointer,
}),
)
for i := 0; i < len(originFrame.Fields); i++ {
t.Run("", func(t *testing.T) {
_, err := convertSQLValueColumnToFloat(originFrame, i)
require.NoError(t, err)
require.Nil(t, originFrame.Fields[i].At(0))
})
}
})
t.Run("Should not return raw connection errors", func(t *testing.T) {
err := net.OpError{Op: "Dial", Err: fmt.Errorf("inner-error")}
transformer := &testQueryResultTransformer{}
dp := DataSourceHandler{
log: backend.NewLoggerWith("logger", "test"),
queryResultTransformer: transformer,
}
resultErr := dp.TransformQueryError(dp.log, &err)
assert.False(t, transformer.transformQueryErrorWasCalled)
errorText := resultErr.Error()
assert.NotEqual(t, err, resultErr)
assert.NotContains(t, errorText, "inner-error")
assert.Contains(t, errorText, "failed to connect to server")
})
t.Run("Should return non-connection errors unmodified", func(t *testing.T) {
err := fmt.Errorf("normal error")
transformer := &testQueryResultTransformer{}
dp := DataSourceHandler{
log: backend.NewLoggerWith("logger", "test"),
queryResultTransformer: transformer,
}
resultErr := dp.TransformQueryError(dp.log, err)
assert.True(t, transformer.transformQueryErrorWasCalled)
assert.Equal(t, err, resultErr)
assert.ErrorIs(t, err, resultErr)
})
}
func TestConvertResultsToFrame(t *testing.T) {
// Import the pgx packages needed for testing
// These imports are included in the main file but need to be accessible for tests
t.Run("convertResultsToFrame with single result", func(t *testing.T) {
// Create mock field descriptions
fieldDescs := []pgconn.FieldDescription{
{Name: "id", DataTypeOID: pgtype.Int4OID},
{Name: "name", DataTypeOID: pgtype.TextOID},
{Name: "value", DataTypeOID: pgtype.Float8OID},
}
// Create mock result data
mockRows := [][][]byte{
{[]byte("1"), []byte("test1"), []byte("10.5")},
{[]byte("2"), []byte("test2"), []byte("20.7")},
}
// Create mock result
result := &pgconn.Result{
FieldDescriptions: fieldDescs,
Rows: mockRows,
}
result.CommandTag = pgconn.NewCommandTag("SELECT 2")
results := []*pgconn.Result{result}
frame, err := convertResultsToFrame(results, 1000)
require.NoError(t, err)
require.NotNil(t, frame)
require.Equal(t, 3, len(frame.Fields))
require.Equal(t, 2, frame.Rows())
// Verify field names
require.Equal(t, "id", frame.Fields[0].Name)
require.Equal(t, "name", frame.Fields[1].Name)
require.Equal(t, "value", frame.Fields[2].Name)
})
t.Run("convertResultsToFrame with multiple compatible results", func(t *testing.T) {
// Create mock field descriptions (same structure for both results)
fieldDescs := []pgconn.FieldDescription{
{Name: "id", DataTypeOID: pgtype.Int4OID},
{Name: "name", DataTypeOID: pgtype.TextOID},
}
// Create first result
mockRows1 := [][][]byte{
{[]byte("1"), []byte("test1")},
{[]byte("2"), []byte("test2")},
}
result1 := &pgconn.Result{
FieldDescriptions: fieldDescs,
Rows: mockRows1,
}
result1.CommandTag = pgconn.NewCommandTag("SELECT 2")
// Create second result with same structure
mockRows2 := [][][]byte{
{[]byte("3"), []byte("test3")},
{[]byte("4"), []byte("test4")},
}
result2 := &pgconn.Result{
FieldDescriptions: fieldDescs,
Rows: mockRows2,
}
result2.CommandTag = pgconn.NewCommandTag("SELECT 2")
results := []*pgconn.Result{result1, result2}
frame, err := convertResultsToFrame(results, 1000)
require.NoError(t, err)
require.NotNil(t, frame)
require.Equal(t, 2, len(frame.Fields))
require.Equal(t, 4, frame.Rows()) // Should have rows from both results
// Verify field names
require.Equal(t, "id", frame.Fields[0].Name)
require.Equal(t, "name", frame.Fields[1].Name)
})
t.Run("convertResultsToFrame with row limit", func(t *testing.T) {
// Create mock field descriptions
fieldDescs := []pgconn.FieldDescription{
{Name: "id", DataTypeOID: pgtype.Int4OID},
}
// Create mock result data with 3 rows
mockRows := [][][]byte{
{[]byte("1")},
{[]byte("2")},
{[]byte("3")},
}
result := &pgconn.Result{
FieldDescriptions: fieldDescs,
Rows: mockRows,
}
result.CommandTag = pgconn.NewCommandTag("SELECT 3")
results := []*pgconn.Result{result}
// Set row limit to 2
frame, err := convertResultsToFrame(results, 2)
require.NoError(t, err)
require.NotNil(t, frame)
require.Equal(t, 1, len(frame.Fields))
require.Equal(t, 2, frame.Rows()) // Should be limited to 2 rows
// Should have a notice about the limit
require.NotNil(t, frame.Meta)
require.Len(t, frame.Meta.Notices, 1)
require.Contains(t, frame.Meta.Notices[0].Text, "Results have been limited to 2")
})
t.Run("convertResultsToFrame with mixed SELECT and non-SELECT results", func(t *testing.T) {
// Create a non-SELECT result (should be skipped)
nonSelectResult := &pgconn.Result{}
nonSelectResult.CommandTag = pgconn.NewCommandTag("UPDATE 1")
// Create a SELECT result
fieldDescs := []pgconn.FieldDescription{
{Name: "id", DataTypeOID: pgtype.Int4OID},
}
mockRows := [][][]byte{
{[]byte("1")},
}
selectResult := &pgconn.Result{
FieldDescriptions: fieldDescs,
Rows: mockRows,
}
selectResult.CommandTag = pgconn.NewCommandTag("SELECT 1")
results := []*pgconn.Result{nonSelectResult, selectResult}
frame, err := convertResultsToFrame(results, 1000)
require.NoError(t, err)
require.NotNil(t, frame)
require.Equal(t, 1, len(frame.Fields))
require.Equal(t, 1, frame.Rows())
})
t.Run("convertResultsToFrame with no SELECT results", func(t *testing.T) {
// Create only non-SELECT results
result1 := &pgconn.Result{}
result1.CommandTag = pgconn.NewCommandTag("UPDATE 1")
result2 := &pgconn.Result{}
result2.CommandTag = pgconn.NewCommandTag("INSERT 1")
results := []*pgconn.Result{result1, result2}
frame, err := convertResultsToFrame(results, 1000)
require.NoError(t, err)
require.NotNil(t, frame)
require.Equal(t, 0, len(frame.Fields))
require.Equal(t, 0, frame.Rows())
})
t.Run("convertResultsToFrame with multiple results and row limit per result", func(t *testing.T) {
// Create mock field descriptions (same structure for both results)
fieldDescs := []pgconn.FieldDescription{
{Name: "id", DataTypeOID: pgtype.Int4OID},
}
// Create first result with 3 rows
mockRows1 := [][][]byte{
{[]byte("1")},
{[]byte("2")},
{[]byte("3")},
}
result1 := &pgconn.Result{
FieldDescriptions: fieldDescs,
Rows: mockRows1,
}
result1.CommandTag = pgconn.NewCommandTag("SELECT 3")
// Create second result with 3 rows
mockRows2 := [][][]byte{
{[]byte("4")},
{[]byte("5")},
{[]byte("6")},
}
result2 := &pgconn.Result{
FieldDescriptions: fieldDescs,
Rows: mockRows2,
}
result2.CommandTag = pgconn.NewCommandTag("SELECT 3")
results := []*pgconn.Result{result1, result2}
// Set row limit to 2 (should limit each result to 2 rows)
frame, err := convertResultsToFrame(results, 2)
require.NoError(t, err)
require.NotNil(t, frame)
require.Equal(t, 1, len(frame.Fields))
require.Equal(t, 4, frame.Rows()) // 2 rows from each result
// Should have notices about the limit from both results
require.NotNil(t, frame.Meta)
require.Len(t, frame.Meta.Notices, 2)
require.Contains(t, frame.Meta.Notices[0].Text, "Results have been limited to 2")
require.Contains(t, frame.Meta.Notices[1].Text, "Results have been limited to 2")
})
t.Run("convertResultsToFrame handles null values correctly", func(t *testing.T) {
// Create mock field descriptions
fieldDescs := []pgconn.FieldDescription{
{Name: "id", DataTypeOID: pgtype.Int4OID},
{Name: "name", DataTypeOID: pgtype.TextOID},
}
// Create mock result data with null values
mockRows := [][][]byte{
{[]byte("1"), nil}, // null name
{nil, []byte("test2")}, // null id
}
result := &pgconn.Result{
FieldDescriptions: fieldDescs,
Rows: mockRows,
}
result.CommandTag = pgconn.NewCommandTag("SELECT 2")
results := []*pgconn.Result{result}
frame, err := convertResultsToFrame(results, 1000)
require.NoError(t, err)
require.NotNil(t, frame)
require.Equal(t, 2, len(frame.Fields))
require.Equal(t, 2, frame.Rows())
// Check that null values are handled correctly
// The exact representation depends on the field type, but should not panic
require.NotPanics(t, func() {
frame.Fields[0].At(1) // null id
frame.Fields[1].At(0) // null name
})
})
}
type testQueryResultTransformer struct {
transformQueryErrorWasCalled bool
}
func (t *testQueryResultTransformer) TransformQueryError(_ log.Logger, err error) error {
t.transformQueryErrorWasCalled = true
return err
}
func (t *testQueryResultTransformer) GetConverterList() []sqlutil.StringConverter {
return nil
}
pkg/tsdb/grafana-postgresql-datasource/pgxtlsmanager.go
-151
View File
@@ -1,151 +0,0 @@
package postgres
import (
"fmt"
"os"
"github.com/grafana/grafana-plugin-sdk-go/backend/log"
"github.com/grafana/grafana/pkg/tsdb/grafana-postgresql-datasource/sqleng"
)
var validateCertFuncPgx = validateCertFilePathsPgx
type pgxTlsManager struct {
logger log.Logger
}
func newPgxTlsManager(logger log.Logger) *pgxTlsManager {
return &pgxTlsManager{
logger: logger,
}
}
// getTLSSettings retrieves TLS settings and handles certificate file creation if needed.
func (m *pgxTlsManager) getTLSSettings(dsInfo sqleng.DataSourceInfo) (tlsSettings, error) {
tlsConfig := tlsSettings{
Mode: dsInfo.JsonData.Mode,
}
if tlsConfig.Mode == "disable" {
m.logger.Debug("Postgres TLS/SSL is disabled")
return tlsConfig, nil
}
tlsConfig.ConfigurationMethod = dsInfo.JsonData.ConfigurationMethod
tlsConfig.RootCertFile = dsInfo.JsonData.RootCertFile
tlsConfig.CertFile = dsInfo.JsonData.CertFile
tlsConfig.CertKeyFile = dsInfo.JsonData.CertKeyFile
if tlsConfig.ConfigurationMethod == "file-content" {
if err := m.createCertFiles(dsInfo, &tlsConfig); err != nil {
return tlsConfig, fmt.Errorf("failed to create TLS certificate files: %w", err)
}
} else {
if err := validateCertFuncPgx(tlsConfig.RootCertFile, tlsConfig.CertFile, tlsConfig.CertKeyFile); err != nil {
return tlsConfig, fmt.Errorf("invalid TLS certificate file paths: %w", err)
}
}
return tlsConfig, nil
}
// createCertFiles writes certificate files to temporary locations.
func (m *pgxTlsManager) createCertFiles(dsInfo sqleng.DataSourceInfo, tlsConfig *tlsSettings) error {
m.logger.Debug("Writing TLS certificate files to temporary locations")
var err error
if tlsConfig.RootCertFile, err = m.writeCertFile("root-*.crt", dsInfo.DecryptedSecureJSONData["tlsCACert"]); err != nil {
return err
}
if tlsConfig.CertFile, err = m.writeCertFile("client-*.crt", dsInfo.DecryptedSecureJSONData["tlsClientCert"]); err != nil {
return err
}
if tlsConfig.CertKeyFile, err = m.writeCertFile("client-*.key", dsInfo.DecryptedSecureJSONData["tlsClientKey"]); err != nil {
return err
}
return nil
}
// writeCertFile writes a single certificate file to a temporary location.
func (m *pgxTlsManager) writeCertFile(pattern, content string) (string, error) {
if content == "" {
return "", nil
}
m.logger.Debug("Writing certificate file", "pattern", pattern)
file, err := os.CreateTemp("", pattern)
if err != nil {
return "", fmt.Errorf("failed to create temporary file: %w", err)
}
defer func() {
if err := file.Close(); err != nil {
m.logger.Error("Failed to close file", "error", err)
}
}()
if _, err := file.WriteString(content); err != nil {
return "", fmt.Errorf("failed to write to temporary file: %w", err)
}
return file.Name(), nil
}
// cleanupCertFiles removes temporary certificate files.
func (m *pgxTlsManager) cleanupCertFiles(tlsConfig tlsSettings) {
// Only clean up if the configuration method is "file-content"
if tlsConfig.ConfigurationMethod != "file-content" {
m.logger.Debug("Skipping cleanup of TLS certificate files")
return
}
m.logger.Debug("Cleaning up TLS certificate files")
files := []struct {
path string
name string
}{
{tlsConfig.RootCertFile, "root certificate"},
{tlsConfig.CertFile, "client certificate"},
{tlsConfig.CertKeyFile, "client key"},
}
for _, file := range files {
if file.path == "" {
continue
}
if err := os.Remove(file.path); err != nil {
m.logger.Error("Failed to remove file", "type", file.name, "path", file.path, "error", err)
} else {
m.logger.Debug("Successfully removed file", "type", file.name, "path", file.path)
}
}
}
// validateCertFilePaths validates the existence of configured certificate file paths.
func validateCertFilePathsPgx(rootCert, clientCert, clientKey string) error {
for _, path := range []string{rootCert, clientCert, clientKey} {
if path == "" {
continue
}
exists, err := fileExistsPgx(path)
if err != nil {
return fmt.Errorf("error checking file existence: %w", err)
}
if !exists {
return sqleng.ErrCertFileNotExist
}
}
return nil
}
// fileExists checks if a file exists at the given path.
func fileExistsPgx(path string) (bool, error) {
_, err := os.Stat(path)
if err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, err
}
return true, nil
}
pkg/tsdb/grafana-postgresql-datasource/pgxtlsmanager_test.go
-245
View File
@@ -1,245 +0,0 @@
package postgres
import (
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/backend/log"
"github.com/grafana/grafana/pkg/tsdb/grafana-postgresql-datasource/sqleng"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// Test getTLSSettings.
func TestPgxGetTLSSettings(t *testing.T) {
mockValidateCertFilePathsPgx()
t.Cleanup(resetValidateCertFilePathsPgx)
updatedTime := time.Now()
testCases := []struct {
desc string
expErr string
jsonData sqleng.JsonData
secureJSONData map[string]string
uid string
tlsSettings tlsSettings
updated time.Time
}{
{
desc: "Custom TLS authentication disabled",
updated: updatedTime,
jsonData: sqleng.JsonData{
Mode: "disable",
RootCertFile: "i/am/coding/ca.crt",
CertFile: "i/am/coding/client.crt",
CertKeyFile: "i/am/coding/client.key",
ConfigurationMethod: "file-path",
},
tlsSettings: tlsSettings{Mode: "disable"},
},
{
desc: "Custom TLS authentication with file path",
updated: updatedTime.Add(time.Minute),
jsonData: sqleng.JsonData{
Mode: "verify-full",
ConfigurationMethod: "file-path",
RootCertFile: "i/am/coding/ca.crt",
CertFile: "i/am/coding/client.crt",
CertKeyFile: "i/am/coding/client.key",
},
tlsSettings: tlsSettings{
Mode: "verify-full",
ConfigurationMethod: "file-path",
RootCertFile: "i/am/coding/ca.crt",
CertFile: "i/am/coding/client.crt",
CertKeyFile: "i/am/coding/client.key",
},
},
}
for _, tt := range testCases {
t.Run(tt.desc, func(t *testing.T) {
var settings tlsSettings
var err error
mng := pgxTlsManager{
logger: backend.NewLoggerWith("logger", "tsdb.postgres"),
}
ds := sqleng.DataSourceInfo{
JsonData: tt.jsonData,
DecryptedSecureJSONData: tt.secureJSONData,
UID: tt.uid,
Updated: tt.updated,
}
settings, err = mng.getTLSSettings(ds)
if tt.expErr == "" {
require.NoError(t, err, tt.desc)
assert.Equal(t, tt.tlsSettings, settings)
} else {
require.Error(t, err, tt.desc)
assert.True(t, strings.HasPrefix(err.Error(), tt.expErr),
fmt.Sprintf("%s: %q doesn't start with %q", tt.desc, err, tt.expErr))
}
})
}
}
func mockValidateCertFilePathsPgx() {
validateCertFuncPgx = func(rootCert, clientCert, clientKey string) error {
return nil
}
}
func resetValidateCertFilePathsPgx() {
validateCertFuncPgx = validateCertFilePathsPgx
}
func TestTLSManager_GetTLSSettings(t *testing.T) {
logger := log.New()
tlsManager := newPgxTlsManager(logger)
dsInfo := sqleng.DataSourceInfo{
JsonData: sqleng.JsonData{
Mode: "require",
ConfigurationMethod: "file-content",
},
DecryptedSecureJSONData: map[string]string{
"tlsCACert": "root-cert-content",
"tlsClientCert": "client-cert-content",
"tlsClientKey": "client-key-content",
},
}
tlsConfig, err := tlsManager.getTLSSettings(dsInfo)
require.NoError(t, err)
assert.Equal(t, "require", tlsConfig.Mode)
assert.NotEmpty(t, tlsConfig.RootCertFile)
assert.NotEmpty(t, tlsConfig.CertFile)
assert.NotEmpty(t, tlsConfig.CertKeyFile)
// Cleanup temporary files
tlsManager.cleanupCertFiles(tlsConfig)
assert.NoFileExists(t, tlsConfig.RootCertFile)
assert.NoFileExists(t, tlsConfig.CertFile)
assert.NoFileExists(t, tlsConfig.CertKeyFile)
}
func TestTLSManager_CleanupCertFiles_FilePath(t *testing.T) {
logger := log.New()
tlsManager := newPgxTlsManager(logger)
// Create temporary files for testing
rootCertFile, err := tlsManager.writeCertFile("root-*.crt", "root-cert-content")
require.NoError(t, err)
clientCertFile, err := tlsManager.writeCertFile("client-*.crt", "client-cert-content")
require.NoError(t, err)
clientKeyFile, err := tlsManager.writeCertFile("client-*.key", "client-key-content")
require.NoError(t, err)
// Simulate a configuration where the method is "file-path"
tlsConfig := tlsSettings{
ConfigurationMethod: "file-path",
RootCertFile: rootCertFile,
CertFile: clientCertFile,
CertKeyFile: clientKeyFile,
}
// Call cleanupCertFiles
tlsManager.cleanupCertFiles(tlsConfig)
// Verify the files are NOT deleted
assert.FileExists(t, rootCertFile, "Root certificate file should not be deleted")
assert.FileExists(t, clientCertFile, "Client certificate file should not be deleted")
assert.FileExists(t, clientKeyFile, "Client key file should not be deleted")
// Cleanup the files manually
err = os.Remove(rootCertFile)
require.NoError(t, err)
err = os.Remove(clientCertFile)
require.NoError(t, err)
err = os.Remove(clientKeyFile)
require.NoError(t, err)
}
func TestTLSManager_CreateCertFiles(t *testing.T) {
logger := log.New()
tlsManager := newPgxTlsManager(logger)
dsInfo := sqleng.DataSourceInfo{
DecryptedSecureJSONData: map[string]string{
"tlsCACert": "root-cert-content",
"tlsClientCert": "client-cert-content",
"tlsClientKey": "client-key-content",
},
}
tlsConfig := tlsSettings{
ConfigurationMethod: "file-content",
}
err := tlsManager.createCertFiles(dsInfo, &tlsConfig)
require.NoError(t, err)
assert.FileExists(t, tlsConfig.RootCertFile)
assert.FileExists(t, tlsConfig.CertFile)
assert.FileExists(t, tlsConfig.CertKeyFile)
// Cleanup temporary files
tlsManager.cleanupCertFiles(tlsConfig)
assert.NoFileExists(t, tlsConfig.RootCertFile)
assert.NoFileExists(t, tlsConfig.CertFile)
assert.NoFileExists(t, tlsConfig.CertKeyFile)
}
func TestTLSManager_WriteCertFile(t *testing.T) {
logger := log.New()
tlsManager := newPgxTlsManager(logger)
// Test writing a valid certificate file
filePath, err := tlsManager.writeCertFile("test-*.crt", "test-cert-content")
require.NoError(t, err)
assert.FileExists(t, filePath)
content, err := os.ReadFile(filepath.Clean(filePath))
require.NoError(t, err)
assert.Equal(t, "test-cert-content", string(content))
// Cleanup the file
err = os.Remove(filePath)
require.NoError(t, err)
assert.NoFileExists(t, filePath)
}
func TestTLSManager_CleanupCertFiles(t *testing.T) {
logger := log.New()
tlsManager := newPgxTlsManager(logger)
// Create temporary files for testing
rootCertFile, err := tlsManager.writeCertFile("root-*.crt", "root-cert-content")
require.NoError(t, err)
clientCertFile, err := tlsManager.writeCertFile("client-*.crt", "client-cert-content")
require.NoError(t, err)
clientKeyFile, err := tlsManager.writeCertFile("client-*.key", "client-key-content")
require.NoError(t, err)
tlsConfig := tlsSettings{
ConfigurationMethod: "file-content",
RootCertFile: rootCertFile,
CertFile: clientCertFile,
CertKeyFile: clientKeyFile,
}
// Cleanup the files
tlsManager.cleanupCertFiles(tlsConfig)
// Verify the files are deleted
assert.NoFileExists(t, rootCertFile)
assert.NoFileExists(t, clientCertFile)
assert.NoFileExists(t, clientKeyFile)
}
pkg/tsdb/grafana-postgresql-datasource/postgres.go
+25 -108
View File
@@ -2,7 +2,6 @@ package postgres
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"reflect"
@@ -16,63 +15,12 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana-plugin-sdk-go/data/sqlutil"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/lib/pq"
"github.com/grafana/grafana-plugin-sdk-go/backend/log"
sqlengpgx "github.com/grafana/grafana/pkg/tsdb/grafana-postgresql-datasource/pgx"
"github.com/grafana/grafana/pkg/tsdb/grafana-postgresql-datasource/sqleng"
)
func newPostgres(ctx context.Context, userFacingDefaultError string, rowLimit int64, dsInfo sqleng.DataSourceInfo, cnnstr string, logger log.Logger, settings backend.DataSourceInstanceSettings) (*sql.DB, *sqleng.DataSourceHandler, error) {
connector, err := pq.NewConnector(cnnstr)
if err != nil {
logger.Error("postgres connector creation failed", "error", err)
return nil, nil, fmt.Errorf("postgres connector creation failed")
}
proxyClient, err := settings.ProxyClient(ctx)
if err != nil {
logger.Error("postgres proxy creation failed", "error", err)
return nil, nil, fmt.Errorf("postgres proxy creation failed")
}
if proxyClient.SecureSocksProxyEnabled() {
dialer, err := proxyClient.NewSecureSocksProxyContextDialer()
if err != nil {
logger.Error("postgres proxy creation failed", "error", err)
return nil, nil, fmt.Errorf("postgres proxy creation failed")
}
postgresDialer := newPostgresProxyDialer(dialer)
// update the postgres dialer with the proxy dialer
connector.Dialer(postgresDialer)
}
config := sqleng.DataPluginConfiguration{
DSInfo: dsInfo,
MetricColumnTypes: []string{"UNKNOWN", "TEXT", "VARCHAR", "CHAR"},
RowLimit: rowLimit,
}
queryResultTransformer := postgresQueryResultTransformer{}
db := sql.OpenDB(connector)
db.SetMaxOpenConns(config.DSInfo.JsonData.MaxOpenConns)
db.SetMaxIdleConns(config.DSInfo.JsonData.MaxIdleConns)
db.SetConnMaxLifetime(time.Duration(config.DSInfo.JsonData.ConnMaxLifetime) * time.Second)
handler, err := sqleng.NewQueryDataHandler(userFacingDefaultError, db, config, &queryResultTransformer, newPostgresMacroEngine(dsInfo.JsonData.Timescaledb),
logger)
if err != nil {
logger.Error("Failed connecting to Postgres", "err", err)
return nil, nil, err
}
logger.Debug("Successfully connected to Postgres")
return db, handler, nil
}
func newPostgresPGX(ctx context.Context, userFacingDefaultError string, rowLimit int64, dsInfo sqleng.DataSourceInfo, cnnstr string, logger log.Logger, settings backend.DataSourceInstanceSettings) (*pgxpool.Pool, *sqlengpgx.DataSourceHandler, error) {
func newPostgres(ctx context.Context, userFacingDefaultError string, rowLimit int64, dsInfo sqleng.DataSourceInfo, cnnstr string, logger log.Logger, settings backend.DataSourceInstanceSettings) (*pgxpool.Pool, *sqleng.DataSourceHandler, error) {
pgxConf, err := pgxpool.ParseConfig(cnnstr)
if err != nil {
logger.Error("postgres config creation failed", "error", err)
@@ -92,7 +40,7 @@ func newPostgresPGX(ctx context.Context, userFacingDefaultError string, rowLimit
return nil, nil, fmt.Errorf("postgres proxy creation failed")
}
pgxConf.ConnConfig.DialFunc = newPgxDialFunc(dialer)
pgxConf.ConnConfig.DialFunc = newDialFunc(dialer)
}
// by default pgx resolves hostnames to ip addresses. we must avoid this.
@@ -101,7 +49,7 @@ func newPostgresPGX(ctx context.Context, userFacingDefaultError string, rowLimit
return []string{host}, nil
}
config := sqlengpgx.DataPluginConfiguration{
config := sqleng.DataPluginConfiguration{
DSInfo: dsInfo,
MetricColumnTypes: []string{"unknown", "text", "varchar", "char", "bpchar"},
RowLimit: rowLimit,
@@ -117,7 +65,7 @@ func newPostgresPGX(ctx context.Context, userFacingDefaultError string, rowLimit
return nil, nil, err
}
handler, err := sqlengpgx.NewQueryDataHandler(userFacingDefaultError, p, config, &queryResultTransformer, newPostgresMacroEngine(dsInfo.JsonData.Timescaledb),
handler, err := sqleng.NewQueryDataHandler(userFacingDefaultError, p, config, &queryResultTransformer, newPostgresMacroEngine(dsInfo.JsonData.Timescaledb),
logger)
if err != nil {
logger.Error("Failed connecting to Postgres", "err", err)
@@ -128,7 +76,7 @@ func newPostgresPGX(ctx context.Context, userFacingDefaultError string, rowLimit
return p, handler, nil
}
func NewInstanceSettings(logger log.Logger, dataPath string) datasource.InstanceFactoryFunc {
func NewInstanceSettings(logger log.Logger) datasource.InstanceFactoryFunc {
return func(ctx context.Context, settings backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
cfg := backend.GrafanaConfigFromContext(ctx)
sqlCfg, err := cfg.SQL()
@@ -171,48 +119,26 @@ func NewInstanceSettings(logger log.Logger, dataPath string) datasource.Instance
return nil, err
}
usePGX := cfg.FeatureToggles().IsEnabled("postgresDSUsePGX")
if usePGX {
pgxlogger := logger.FromContext(ctx).With("driver", "pgx")
pgxTlsManager := newPgxTlsManager(pgxlogger)
pgxTlsSettings, err := pgxTlsManager.getTLSSettings(dsInfo)
if err != nil {
return "", err
}
// Ensure cleanupCertFiles is called after the connection is opened
defer pgxTlsManager.cleanupCertFiles(pgxTlsSettings)
cnnstr, err := generateConnectionString(dsInfo, pgxTlsSettings, usePGX, pgxlogger)
if err != nil {
return "", err
}
_, handler, err := newPostgresPGX(ctx, userFacingDefaultError, sqlCfg.RowLimit, dsInfo, cnnstr, pgxlogger, settings)
if err != nil {
pgxlogger.Error("Failed connecting to Postgres", "err", err)
return nil, err
}
pgxlogger.Debug("Successfully connected to Postgres")
return handler, nil
} else {
pqlogger := logger.FromContext(ctx).With("driver", "libpq")
tlsManager := newTLSManager(pqlogger, dataPath)
tlsSettings, err := tlsManager.getTLSSettings(dsInfo)
if err != nil {
return "", err
}
cnnstr, err := generateConnectionString(dsInfo, tlsSettings, usePGX, pqlogger)
if err != nil {
return nil, err
}
_, handler, err := newPostgres(ctx, userFacingDefaultError, sqlCfg.RowLimit, dsInfo, cnnstr, pqlogger, settings)
if err != nil {
pqlogger.Error("Failed connecting to Postgres", "err", err)
return nil, err
}
pqlogger.Debug("Successfully connected to Postgres")
return handler, nil
pgxlogger := logger.FromContext(ctx).With("driver", "pgx")
tlsManager := newTLSManager(pgxlogger)
tlsSettings, err := tlsManager.getTLSSettings(dsInfo)
if err != nil {
return "", err
}
// Ensure cleanupCertFiles is called after the connection is opened
defer tlsManager.cleanupCertFiles(tlsSettings)
cnnstr, err := generateConnectionString(dsInfo, tlsSettings, pgxlogger)
if err != nil {
return "", err
}
_, handler, err := newPostgres(ctx, userFacingDefaultError, sqlCfg.RowLimit, dsInfo, cnnstr, pgxlogger, settings)
if err != nil {
pgxlogger.Error("Failed connecting to Postgres", "err", err)
return nil, err
}
pgxlogger.Debug("Successfully connected to Postgres")
return handler, nil
}
}
@@ -302,7 +228,7 @@ func buildBaseConnectionString(params connectionParams) string {
return connStr
}
func generateConnectionString(dsInfo sqleng.DataSourceInfo, tlsSettings tlsSettings, isPGX bool, logger log.Logger) (string, error) {
func generateConnectionString(dsInfo sqleng.DataSourceInfo, tlsSettings tlsSettings, logger log.Logger) (string, error) {
params, err := parseConnectionParams(dsInfo, logger)
if err != nil {
return "", err
@@ -312,15 +238,6 @@ func generateConnectionString(dsInfo sqleng.DataSourceInfo, tlsSettings tlsSetti
connStr += fmt.Sprintf(" sslmode='%s'", escape(tlsSettings.Mode))
// there is an issue with the lib/pq module, the `verify-ca` tls mode
// does not work correctly. ( see https://github.com/lib/pq/issues/1106 )
// to workaround the problem, if the `verify-ca` mode is chosen,
// we disable sslsni.
if tlsSettings.Mode == "verify-ca" && !isPGX {
logger.Debug("Disabling sslsni for verify-ca mode")
connStr += " sslsni=0"
}
// Attach root certificate if provided
if tlsSettings.RootCertFile != "" {
logger.Debug("Setting server root certificate", "tlsRootCert", tlsSettings.RootCertFile)
pkg/tsdb/grafana-postgresql-datasource/postgres_pgx_snapshot_test.go
-196
View File
@@ -1,196 +0,0 @@
package postgres
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
"testing"
"time"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/backend/log"
"github.com/grafana/grafana-plugin-sdk-go/experimental"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/tsdb/grafana-postgresql-datasource/sqleng"
"github.com/grafana/grafana/pkg/util/testutil"
)
// These tests require a real postgres database:
// - make devenv sources=postgres_tests
// - either set the env variable GRAFANA_TEST_DB = postgres
// - or set `forceRun := true` below
//
// The tests require a PostgreSQL db named grafanadstest and a user/password grafanatest/grafanatest!
// Use the docker/blocks/postgres_tests/docker-compose.yaml to spin up a
// preconfigured Postgres server suitable for running these tests.
func TestIntegrationPostgresPGXSnapshots(t *testing.T) {
testutil.SkipIntegrationTestInShortMode(t)
// the logic in this function is copied from postgres_tests.go
shouldRunTest := func() bool {
testDbName, present := os.LookupEnv("GRAFANA_TEST_DB")
if present && testDbName == "postgres" {
return true
}
return false
}
if !shouldRunTest() {
t.Skip()
}
getCnnStr := func() string {
host := os.Getenv("POSTGRES_HOST")
if host == "" {
host = "localhost"
}
port := os.Getenv("POSTGRES_PORT")
if port == "" {
port = "5432"
}
return fmt.Sprintf("user=grafanatest password=grafanatest host=%s port=%s dbname=grafanadstest sslmode=disable",
host, port)
}
sqlQueryCommentRe := regexp.MustCompile(`^-- (.+)\n`)
readSqlFile := func(path string) (string, string) {
// the file-path is not coming from the outside,
// it is hardcoded in this file.
//nolint:gosec
sqlBytes, err := os.ReadFile(path)
require.NoError(t, err)
sql := string(sqlBytes)
// first line of the file contains the sql query to run, commented out
match := sqlQueryCommentRe.FindStringSubmatch(sql)
require.Len(t, match, 2)
rawSQL := strings.TrimSpace(match[1])
return rawSQL, sql
}
makeQuery := func(rawSQL string, format string) backend.QueryDataRequest {
queryData := map[string]string{
"rawSql": rawSQL,
"format": format,
}
queryBytes, err := json.Marshal(queryData)
require.NoError(t, err)
return backend.QueryDataRequest{
Queries: []backend.DataQuery{
{
JSON: queryBytes,
RefID: "A",
TimeRange: backend.TimeRange{
From: time.Date(2023, 12, 24, 14, 15, 22, 123456, time.UTC),
To: time.Date(2023, 12, 24, 14, 45, 13, 876543, time.UTC),
},
},
},
}
}
tt := []struct {
name string
format string
}{
{format: "time_series", name: "simple"},
{format: "time_series", name: "no_rows_long"},
{format: "time_series", name: "no_rows_wide"},
{format: "time_series", name: "7x_compat_metric_label"},
{format: "time_series", name: "convert_to_float64"},
{format: "time_series", name: "convert_to_float64_not"},
{format: "time_series", name: "fill_null"},
{format: "time_series", name: "fill_previous"},
{format: "time_series", name: "fill_value"},
{format: "time_series", name: "fill_value_wide"},
{format: "table", name: "simple"},
{format: "table", name: "multi_stat1"},
{format: "table", name: "multi_stat2"},
{format: "table", name: "no_rows"},
{format: "table", name: "types_numeric"},
{format: "table", name: "types_char"},
{format: "table", name: "types_datetime_pgx"},
{format: "table", name: "types_other"},
{format: "table", name: "types_enum"},
{format: "table", name: "types_jsonb"},
{format: "table", name: "timestamp_convert_bigint"},
{format: "table", name: "timestamp_convert_integer"},
{format: "table", name: "timestamp_convert_real"},
{format: "table", name: "timestamp_convert_double"},
{format: "table", name: "time_group_compat_case1"},
{format: "table", name: "time_group_compat_case2"},
}
for _, test := range tt {
require.True(t, test.format == "table" || test.format == "time_series")
t.Run(test.name, func(t *testing.T) {
origInterpolate := sqleng.Interpolate
t.Cleanup(func() {
sqleng.Interpolate = origInterpolate
})
sqleng.Interpolate = func(query backend.DataQuery, timeRange backend.TimeRange, timeInterval string, sql string) string {
return sql
}
jsonData := sqleng.JsonData{
MaxOpenConns: 10,
MaxIdleConns: 2,
ConnMaxLifetime: 14400,
Timescaledb: false,
Mode: "disable",
ConfigurationMethod: "file-path",
}
dsInfo := sqleng.DataSourceInfo{
JsonData: jsonData,
DecryptedSecureJSONData: map[string]string{},
}
logger := log.New()
cnnstr := getCnnStr()
p, handler, err := newPostgresPGX(context.Background(), "error", 10000, dsInfo, cnnstr, logger, backend.DataSourceInstanceSettings{})
t.Cleanup((func() {
_, err := p.Exec(context.Background(), "DROP TABLE tbl")
require.NoError(t, err)
p.Close()
}))
require.NoError(t, err)
sqlFilePath := filepath.Join("testdata", test.format, test.name+".sql")
goldenFileName := filepath.Join(test.format, test.name+".golden")
rawSQL, sql := readSqlFile(sqlFilePath)
_, err = p.Exec(context.Background(), sql)
require.NoError(t, err)
query := makeQuery(rawSQL, test.format)
result, err := handler.QueryData(context.Background(), &query)
require.Len(t, result.Responses, 1)
response, found := result.Responses["A"]
require.True(t, found)
require.NoError(t, err)
experimental.CheckGoldenJSONResponse(t, "testdata", goldenFileName, &response, updateGoldenFiles)
})
}
}
pkg/tsdb/grafana-postgresql-datasource/postgres_pgx_test.go
-1579
View File
File diff suppressed because it is too large Load Diff
pkg/tsdb/grafana-postgresql-datasource/postgres_service.go
+10 -39
View File
@@ -7,8 +7,6 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend/datasource"
"github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt"
"github.com/grafana/grafana/pkg/setting"
sqlengpgx "github.com/grafana/grafana/pkg/tsdb/grafana-postgresql-datasource/pgx"
"github.com/grafana/grafana/pkg/tsdb/grafana-postgresql-datasource/sqleng"
)
@@ -16,48 +14,30 @@ type Service struct {
im instancemgmt.InstanceManager
}
func ProvideService(cfg *setting.Cfg) *Service {
func ProvideService() *Service {
logger := backend.NewLoggerWith("logger", "tsdb.postgres")
s := &Service{
im: datasource.NewInstanceManager(NewInstanceSettings(logger, cfg.DataPath)),
im: datasource.NewInstanceManager(NewInstanceSettings(logger)),
}
return s
}
// NOTE: do not put any business logic into this method. it's whole job is to forward the call "inside"
func (s *Service) CheckHealth(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) {
cfg := backend.GrafanaConfigFromContext(ctx)
if cfg.FeatureToggles().IsEnabled("postgresDSUsePGX") {
dsHandler, err := s.getDSInfoPGX(ctx, req.PluginContext)
if err != nil {
return sqlengpgx.ErrToHealthCheckResult(err)
}
return dsHandler.CheckHealth(ctx, req)
} else {
dsHandler, err := s.getDSInfo(ctx, req.PluginContext)
if err != nil {
return sqleng.ErrToHealthCheckResult(err)
}
return dsHandler.CheckHealth(ctx, req)
dsHandler, err := s.getDSInfo(ctx, req.PluginContext)
if err != nil {
return sqleng.ErrToHealthCheckResult(err)
}
return dsHandler.CheckHealth(ctx, req)
}
// NOTE: do not put any business logic into this method. it's whole job is to forward the call "inside"
func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) {
cfg := backend.GrafanaConfigFromContext(ctx)
if cfg.FeatureToggles().IsEnabled("postgresDSUsePGX") {
dsInfo, err := s.getDSInfoPGX(ctx, req.PluginContext)
if err != nil {
return nil, err
}
return dsInfo.QueryData(ctx, req)
} else {
dsInfo, err := s.getDSInfo(ctx, req.PluginContext)
if err != nil {
return nil, err
}
return dsInfo.QueryData(ctx, req)
dsInfo, err := s.getDSInfo(ctx, req.PluginContext)
if err != nil {
return nil, err
}
return dsInfo.QueryData(ctx, req)
}
func (s *Service) getDSInfo(ctx context.Context, pluginCtx backend.PluginContext) (*sqleng.DataSourceHandler, error) {
@@ -68,12 +48,3 @@ func (s *Service) getDSInfo(ctx context.Context, pluginCtx backend.PluginContext
instance := i.(*sqleng.DataSourceHandler)
return instance, nil
}
func (s *Service) getDSInfoPGX(ctx context.Context, pluginCtx backend.PluginContext) (*sqlengpgx.DataSourceHandler, error) {
i, err := s.im.Get(ctx, pluginCtx)
if err != nil {
return nil, err
}
instance := i.(*sqlengpgx.DataSourceHandler)
return instance, nil
}
pkg/tsdb/grafana-postgresql-datasource/postgres_snapshot_test.go
+12 -14
View File
@@ -1,7 +1,6 @@
package postgres
import (
"context"
"encoding/json"
"fmt"
"os"
@@ -23,7 +22,7 @@ import (
var updateGoldenFiles = false
// These tests require a real postgres database:
// - make devenv sources=potgres_tests
// - make devenv sources=postgres_tests
// - either set the env variable GRAFANA_TEST_DB = postgres
// - or set `forceRun := true` below
//
@@ -127,6 +126,8 @@ func TestIntegrationPostgresSnapshots(t *testing.T) {
{format: "table", name: "types_char"},
{format: "table", name: "types_datetime"},
{format: "table", name: "types_other"},
{format: "table", name: "types_enum"},
{format: "table", name: "types_jsonb"},
{format: "table", name: "timestamp_convert_bigint"},
{format: "table", name: "timestamp_convert_integer"},
{format: "table", name: "timestamp_convert_real"},
@@ -148,10 +149,11 @@ func TestIntegrationPostgresSnapshots(t *testing.T) {
}
jsonData := sqleng.JsonData{
MaxOpenConns: 0,
MaxOpenConns: 10,
MaxIdleConns: 2,
ConnMaxLifetime: 14400,
Timescaledb: false,
Mode: "disable",
ConfigurationMethod: "file-path",
}
@@ -164,28 +166,24 @@ func TestIntegrationPostgresSnapshots(t *testing.T) {
cnnstr := getCnnStr()
db, handler, err := newPostgres(context.Background(), "error", 10000, dsInfo, cnnstr, logger, backend.DataSourceInstanceSettings{})
t.Cleanup((func() {
_, err := db.Exec("DROP TABLE tbl")
require.NoError(t, err)
err = db.Close()
require.NoError(t, err)
}))
p, handler, err := newPostgres(t.Context(), "error", 10000, dsInfo, cnnstr, logger, backend.DataSourceInstanceSettings{})
require.NoError(t, err)
t.Cleanup(func() {
p.Close()
})
sqlFilePath := filepath.Join("testdata", test.format, test.name+".sql")
goldenFileName := filepath.Join(test.format, test.name+".golden")
rawSQL, sql := readSqlFile(sqlFilePath)
_, err = db.Exec(sql)
_, err = p.Exec(t.Context(), sql)
require.NoError(t, err)
query := makeQuery(rawSQL, test.format)
result, err := handler.QueryData(context.Background(), &query)
result, err := handler.QueryData(t.Context(), &query)
require.Len(t, result.Responses, 1)
response, found := result.Responses["A"]
require.True(t, found)
pkg/tsdb/grafana-postgresql-datasource/postgres_test.go
+251 -76
View File
@@ -1,7 +1,7 @@
package postgres
import (
"context"
"encoding/json"
"fmt"
"math/rand"
"os"
@@ -15,20 +15,14 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb/grafana-postgresql-datasource/sqleng"
"github.com/grafana/grafana/pkg/util/testutil"
_ "github.com/lib/pq"
)
// Test generateConnectionString.
func TestIntegrationGenerateConnectionString(t *testing.T) {
testutil.SkipIntegrationTestInShortMode(t)
cfg := setting.NewCfg()
cfg.DataPath = t.TempDir()
testCases := []struct {
desc string
host string
@@ -58,15 +52,6 @@ func TestIntegrationGenerateConnectionString(t *testing.T) {
tlsSettings: tlsSettings{Mode: "verify-full"},
expConnStr: "user='user' host='host' dbname='database' password='password' sslmode='verify-full'",
},
{
desc: "verify-ca automatically adds disable-sni",
host: "host:1234",
user: "user",
password: "password",
database: "database",
tlsSettings: tlsSettings{Mode: "verify-ca"},
expConnStr: "user='user' host='host' dbname='database' password='password' port=1234 sslmode='verify-ca' sslsni=0",
},
{
desc: "TCP/port host",
host: "host:1234",
@@ -156,8 +141,6 @@ func TestIntegrationGenerateConnectionString(t *testing.T) {
}
for _, tt := range testCases {
t.Run(tt.desc, func(t *testing.T) {
logger := backend.NewLoggerWith("logger", "tsdb.postgres")
ds := sqleng.DataSourceInfo{
URL: tt.host,
User: tt.user,
@@ -165,8 +148,9 @@ func TestIntegrationGenerateConnectionString(t *testing.T) {
Database: tt.database,
UID: tt.uid,
}
logger := backend.NewLoggerWith("logger", "tsdb.postgres")
connStr, err := generateConnectionString(ds, tt.tlsSettings, false, logger)
connStr, err := generateConnectionString(ds, tt.tlsSettings, logger)
if tt.expErr == "" {
require.NoError(t, err, tt.desc)
@@ -207,10 +191,11 @@ func TestIntegrationPostgres(t *testing.T) {
}
jsonData := sqleng.JsonData{
MaxOpenConns: 0,
MaxOpenConns: 10,
MaxIdleConns: 2,
ConnMaxLifetime: 14400,
Timescaledb: false,
Mode: "disable",
ConfigurationMethod: "file-path",
}
@@ -223,7 +208,7 @@ func TestIntegrationPostgres(t *testing.T) {
cnnstr := postgresTestDBConnString()
db, exe, err := newPostgres(context.Background(), "error", 10000, dsInfo, cnnstr, logger, backend.DataSourceInstanceSettings{})
p, exe, err := newPostgres(t.Context(), "error", 10000, dsInfo, cnnstr, logger, backend.DataSourceInstanceSettings{})
require.NoError(t, err)
@@ -253,10 +238,13 @@ func TestIntegrationPostgres(t *testing.T) {
c14_timetz time with time zone,
time date,
c15_interval interval,
c16_smallint smallint
c16_smallint smallint,
c17_json json,
c18_jsonb jsonb
);
`
_, err := db.Exec(sql)
_, err := p.Exec(t.Context(), sql)
require.NoError(t, err)
sql = `
@@ -264,12 +252,11 @@ func TestIntegrationPostgres(t *testing.T) {
1,2,3,
4.5,6.7,1.1,1.2,
'char10','varchar10','text',
now(),now(),now(),now(),now(),now(),'15m'::interval,
null
now(),now(),now(),now(),now(),now(),'15m'::interval,null,
'{"key1": "value1"}'::json, '{"key2": "value2"}'::jsonb
);
`
_, err = db.Exec(sql)
_, err = p.Exec(t.Context(), sql)
require.NoError(t, err)
t.Run("When doing a table query should map Postgres column types to Go types", func(t *testing.T) {
@@ -284,14 +271,14 @@ func TestIntegrationPostgres(t *testing.T) {
},
},
}
resp, err := exe.QueryData(context.Background(), query)
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
frames := queryResult.Frames
require.Len(t, frames, 1)
require.Len(t, frames[0].Fields, 18)
require.Len(t, frames[0].Fields, 20)
require.Equal(t, int16(1), *frames[0].Fields[0].At(0).(*int16))
require.Equal(t, int32(2), *frames[0].Fields[1].At(0).(*int32))
@@ -312,14 +299,21 @@ func TestIntegrationPostgres(t *testing.T) {
require.True(t, ok)
_, ok = frames[0].Fields[12].At(0).(*time.Time)
require.True(t, ok)
_, ok = frames[0].Fields[13].At(0).(*time.Time)
_, ok = frames[0].Fields[13].At(0).(*string)
require.True(t, ok)
_, ok = frames[0].Fields[14].At(0).(*time.Time)
_, ok = frames[0].Fields[14].At(0).(*string)
require.True(t, ok)
_, ok = frames[0].Fields[15].At(0).(*time.Time)
require.True(t, ok)
require.Equal(t, "00:15:00", *frames[0].Fields[16].At(0).(*string))
require.Nil(t, frames[0].Fields[17].At(0))
_, ok = frames[0].Fields[18].At(0).(*json.RawMessage)
require.True(t, ok)
require.Equal(t, json.RawMessage(`{"key1": "value1"}`), *frames[0].Fields[18].At(0).(*json.RawMessage))
_, ok = frames[0].Fields[19].At(0).(*json.RawMessage)
require.True(t, ok)
require.Equal(t, json.RawMessage(`{"key2": "value2"}`), *frames[0].Fields[19].At(0).(*json.RawMessage))
})
})
@@ -332,7 +326,7 @@ func TestIntegrationPostgres(t *testing.T) {
)
`
_, err := db.Exec(sql)
_, err := p.Exec(t.Context(), sql)
require.NoError(t, err)
type metric struct {
@@ -359,7 +353,7 @@ func TestIntegrationPostgres(t *testing.T) {
}
for _, m := range series {
_, err := db.Exec(`INSERT INTO metric ("time", value) VALUES ($1, $2)`, m.Time.UTC(), m.Value)
_, err := p.Exec(t.Context(), `INSERT INTO metric ("time", value) VALUES ($1, $2)`, m.Time.UTC(), m.Value)
require.NoError(t, err)
}
@@ -376,7 +370,7 @@ func TestIntegrationPostgres(t *testing.T) {
},
}
resp, err := exe.QueryData(context.Background(), query)
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
@@ -408,6 +402,27 @@ func TestIntegrationPostgres(t *testing.T) {
}
})
t.Run("When doing a query without a format should default to time_series", func(t *testing.T) {
query := &backend.QueryDataRequest{
Queries: []backend.DataQuery{
{
JSON: []byte(`{
"rawSql": "SELECT $__timeGroup(time, '5m') AS time, avg(value) as value FROM metric GROUP BY 1 ORDER BY 1 "
}`),
RefID: "A",
},
},
}
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
frames := queryResult.Frames
require.Len(t, frames, 1)
require.Len(t, frames[0].Fields, 2)
})
t.Run("When doing a metric query using timeGroup and $__interval", func(t *testing.T) {
mockInterpolate := sqleng.Interpolate
sqleng.Interpolate = origInterpolate
@@ -432,7 +447,7 @@ func TestIntegrationPostgres(t *testing.T) {
},
}
resp, err := exe.QueryData(context.Background(), query)
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
frames := queryResult.Frames
@@ -460,7 +475,7 @@ func TestIntegrationPostgres(t *testing.T) {
},
}
resp, err := exe.QueryData(context.Background(), query)
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
@@ -514,7 +529,7 @@ func TestIntegrationPostgres(t *testing.T) {
},
}
resp, err := exe.QueryData(context.Background(), query)
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
@@ -540,7 +555,7 @@ func TestIntegrationPostgres(t *testing.T) {
}
for _, m := range series {
_, err := db.Exec(`INSERT INTO metric ("time", value) VALUES ($1, $2)`, m.Time.UTC(), m.Value)
_, err := p.Exec(t.Context(), `INSERT INTO metric ("time", value) VALUES ($1, $2)`, m.Time.UTC(), m.Value)
require.NoError(t, err)
}
@@ -561,7 +576,7 @@ func TestIntegrationPostgres(t *testing.T) {
},
}
resp, err := exe.QueryData(context.Background(), query)
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
@@ -596,7 +611,7 @@ func TestIntegrationPostgres(t *testing.T) {
},
}
resp, err := exe.QueryData(context.Background(), query)
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
@@ -624,10 +639,10 @@ func TestIntegrationPostgres(t *testing.T) {
ValueTwo int64
}
_, err := db.Exec("DROP TABLE IF EXISTS metric_values")
_, err := p.Exec(t.Context(), "DROP TABLE IF EXISTS metric_values")
require.NoError(t, err)
_, err = db.Exec(`CREATE TABLE metric_values (
_, err = p.Exec(t.Context(), `CREATE TABLE metric_values (
"time" TIMESTAMP NULL,
"timeInt64" BIGINT NOT NULL, "timeInt64Nullable" BIGINT NULL,
"timeFloat64" DOUBLE PRECISION NOT NULL, "timeFloat64Nullable" DOUBLE PRECISION NULL,
@@ -680,7 +695,7 @@ func TestIntegrationPostgres(t *testing.T) {
// _, err = session.InsertMulti(series)
for _, m := range series {
_, err := db.Exec(`INSERT INTO "metric_values" (
_, err := p.Exec(t.Context(), `INSERT INTO "metric_values" (
time,
"timeInt64", "timeInt64Nullable",
"timeFloat64", "timeFloat64Nullable",
@@ -713,7 +728,7 @@ func TestIntegrationPostgres(t *testing.T) {
},
}
resp, err := exe.QueryData(context.Background(), query)
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
@@ -737,7 +752,7 @@ func TestIntegrationPostgres(t *testing.T) {
},
}
resp, err := exe.QueryData(context.Background(), query)
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
@@ -761,7 +776,7 @@ func TestIntegrationPostgres(t *testing.T) {
},
}
resp, err := exe.QueryData(context.Background(), query)
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
@@ -785,7 +800,7 @@ func TestIntegrationPostgres(t *testing.T) {
},
}
resp, err := exe.QueryData(context.Background(), query)
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
@@ -809,7 +824,7 @@ func TestIntegrationPostgres(t *testing.T) {
},
}
resp, err := exe.QueryData(context.Background(), query)
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
@@ -833,7 +848,7 @@ func TestIntegrationPostgres(t *testing.T) {
},
}
resp, err := exe.QueryData(context.Background(), query)
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
@@ -857,7 +872,7 @@ func TestIntegrationPostgres(t *testing.T) {
},
}
resp, err := exe.QueryData(context.Background(), query)
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
@@ -882,7 +897,7 @@ func TestIntegrationPostgres(t *testing.T) {
},
}
resp, err := exe.QueryData(context.Background(), query)
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
@@ -906,7 +921,7 @@ func TestIntegrationPostgres(t *testing.T) {
},
}
resp, err := exe.QueryData(context.Background(), query)
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
@@ -931,7 +946,7 @@ func TestIntegrationPostgres(t *testing.T) {
},
}
resp, err := exe.QueryData(context.Background(), query)
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
@@ -963,7 +978,7 @@ func TestIntegrationPostgres(t *testing.T) {
},
}
resp, err := exe.QueryData(context.Background(), query)
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
@@ -998,7 +1013,7 @@ func TestIntegrationPostgres(t *testing.T) {
},
}
resp, err := exe.QueryData(context.Background(), query)
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
@@ -1017,9 +1032,9 @@ func TestIntegrationPostgres(t *testing.T) {
Tags string
}
_, err := db.Exec("DROP TABLE IF EXISTS event")
_, err := p.Exec(t.Context(), "DROP TABLE IF EXISTS event")
require.NoError(t, err)
_, err = db.Exec(`CREATE TABLE event (time_sec BIGINT NULL, description VARCHAR(255) NULL, tags VARCHAR(255) NULL)`)
_, err = p.Exec(t.Context(), `CREATE TABLE event (time_sec BIGINT NULL, description VARCHAR(255) NULL, tags VARCHAR(255) NULL)`)
require.NoError(t, err)
events := []*event{}
@@ -1037,7 +1052,7 @@ func TestIntegrationPostgres(t *testing.T) {
}
for _, e := range events {
_, err := db.Exec("INSERT INTO event (time_sec, description, tags) VALUES ($1, $2, $3)", e.TimeSec, e.Description, e.Tags)
_, err := p.Exec(t.Context(), "INSERT INTO event (time_sec, description, tags) VALUES ($1, $2, $3)", e.TimeSec, e.Description, e.Tags)
require.NoError(t, err)
}
@@ -1058,7 +1073,7 @@ func TestIntegrationPostgres(t *testing.T) {
},
}
resp, err := exe.QueryData(context.Background(), query)
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["Deploys"]
@@ -1085,7 +1100,7 @@ func TestIntegrationPostgres(t *testing.T) {
},
}
resp, err := exe.QueryData(context.Background(), query)
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["Tickets"]
@@ -1108,7 +1123,7 @@ func TestIntegrationPostgres(t *testing.T) {
},
}
resp, err := exe.QueryData(context.Background(), query)
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
@@ -1133,7 +1148,7 @@ func TestIntegrationPostgres(t *testing.T) {
},
}
resp, err := exe.QueryData(context.Background(), query)
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
@@ -1158,7 +1173,7 @@ func TestIntegrationPostgres(t *testing.T) {
},
}
resp, err := exe.QueryData(context.Background(), query)
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
@@ -1184,7 +1199,7 @@ func TestIntegrationPostgres(t *testing.T) {
},
}
resp, err := exe.QueryData(context.Background(), query)
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
@@ -1210,7 +1225,7 @@ func TestIntegrationPostgres(t *testing.T) {
},
}
resp, err := exe.QueryData(context.Background(), query)
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
@@ -1236,7 +1251,7 @@ func TestIntegrationPostgres(t *testing.T) {
},
}
resp, err := exe.QueryData(context.Background(), query)
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
@@ -1262,7 +1277,7 @@ func TestIntegrationPostgres(t *testing.T) {
},
}
resp, err := exe.QueryData(context.Background(), query)
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
@@ -1276,8 +1291,20 @@ func TestIntegrationPostgres(t *testing.T) {
})
t.Run("When row limit set to 1", func(t *testing.T) {
dsInfo := sqleng.DataSourceInfo{}
_, handler, err := newPostgres(context.Background(), "error", 1, dsInfo, cnnstr, logger, backend.DataSourceInstanceSettings{})
jsonData := sqleng.JsonData{
MaxOpenConns: 10,
MaxIdleConns: 2,
ConnMaxLifetime: 14400,
Timescaledb: false,
Mode: "disable",
ConfigurationMethod: "file-path",
}
dsInfo := sqleng.DataSourceInfo{
JsonData: jsonData,
DecryptedSecureJSONData: map[string]string{},
}
_, handler, err := newPostgres(t.Context(), "error", 1, dsInfo, cnnstr, logger, backend.DataSourceInstanceSettings{})
require.NoError(t, err)
@@ -1298,7 +1325,7 @@ func TestIntegrationPostgres(t *testing.T) {
},
}
resp, err := handler.QueryData(context.Background(), query)
resp, err := handler.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
@@ -1328,7 +1355,7 @@ func TestIntegrationPostgres(t *testing.T) {
},
}
resp, err := handler.QueryData(context.Background(), query)
resp, err := handler.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
@@ -1344,9 +1371,9 @@ func TestIntegrationPostgres(t *testing.T) {
})
t.Run("Given an empty table", func(t *testing.T) {
_, err := db.Exec("DROP TABLE IF EXISTS empty_obj")
_, err := p.Exec(t.Context(), "DROP TABLE IF EXISTS empty_obj")
require.NoError(t, err)
_, err = db.Exec("CREATE TABLE empty_obj (empty_key VARCHAR(255) NULL, empty_val BIGINT NULL)")
_, err = p.Exec(t.Context(), "CREATE TABLE empty_obj (empty_key VARCHAR(255) NULL, empty_val BIGINT NULL)")
require.NoError(t, err)
t.Run("When no rows are returned, should return an empty frame", func(t *testing.T) {
@@ -1366,7 +1393,7 @@ func TestIntegrationPostgres(t *testing.T) {
},
}
resp, err := exe.QueryData(context.Background(), query)
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
@@ -1376,6 +1403,154 @@ func TestIntegrationPostgres(t *testing.T) {
require.NotNil(t, frames[0].Fields)
require.Empty(t, frames[0].Fields)
})
t.Run("Should handle multiple result sets without panicking", func(t *testing.T) {
// Create a test table for the panic scenario test
sql := `
DROP TABLE IF EXISTS test_multi_results;
CREATE TABLE test_multi_results(
id integer,
name text,
value numeric
);
INSERT INTO test_multi_results VALUES
(1, 'test1', 10.5),
(2, 'test2', 20.7),
(3, 'test3', 30.2);
`
_, err := p.Exec(t.Context(), sql)
require.NoError(t, err)
t.Run("Should handle compatible multiple result sets", func(t *testing.T) {
// This query returns multiple result sets with the same structure
query := &backend.QueryDataRequest{
Queries: []backend.DataQuery{
{
RefID: "A",
JSON: []byte(`{
"rawSql": "SELECT id, name FROM test_multi_results WHERE id <= 2; SELECT id, name FROM test_multi_results WHERE id >= 2;",
"format": "table"
}`),
TimeRange: backend.TimeRange{
From: fromStart,
To: fromStart.Add(1 * time.Hour),
},
},
},
}
// This should not panic and should work correctly
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
frames := queryResult.Frames
require.Len(t, frames, 1)
// The frame should be properly constructed from both SELECT results
frame := frames[0]
require.Equal(t, 2, len(frame.Fields)) // id, name from both queries
require.Equal(t, "id", frame.Fields[0].Name)
require.Equal(t, "name", frame.Fields[1].Name)
require.Equal(t, 4, frame.Rows()) // 2 rows from first result + 2 rows from second result
})
t.Run("Should return error for incompatible multiple result sets", func(t *testing.T) {
// This query returns multiple result sets with different structures - the kind that used to cause panic
query := &backend.QueryDataRequest{
Queries: []backend.DataQuery{
{
RefID: "A",
JSON: []byte(`{
"rawSql": "SELECT id, name FROM test_multi_results WHERE id <= 2; SELECT id, value FROM test_multi_results WHERE id >= 2;",
"format": "table"
}`),
TimeRange: backend.TimeRange{
From: fromStart,
To: fromStart.Add(1 * time.Hour),
},
},
},
}
// This should not panic anymore, but should return an error instead
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
// We expect an error about column mismatch, not a panic
require.Error(t, queryResult.Error)
require.Contains(t, queryResult.Error.Error(), "column name mismatch")
})
t.Run("Should return error for incompatible number of columns", func(t *testing.T) {
// This query returns multiple result sets with different number of columns
// This should fix the error "runtime error: index out of range [1] with length 1"
query := &backend.QueryDataRequest{
Queries: []backend.DataQuery{
{
RefID: "A",
JSON: []byte(`{
"rawSql": "SELECT id, name FROM test_multi_results WHERE id = 1; SELECT id FROM test_multi_results WHERE id = 1;",
"format": "table"
}`),
TimeRange: backend.TimeRange{
From: fromStart,
To: fromStart.Add(1 * time.Hour),
},
},
},
}
// This should not panic anymore, but should return an error instead
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
// We expect an error about incompatible result structure, not a panic
require.Error(t, queryResult.Error)
require.Contains(t, queryResult.Error.Error(), "incompatible result structure: expected 2 columns, got 1 columns")
})
})
t.Run("Should handle queries with mixed statement types", func(t *testing.T) {
// This tests a scenario with UPDATE + SELECT that could cause the original panic
query := &backend.QueryDataRequest{
Queries: []backend.DataQuery{
{
RefID: "A",
JSON: []byte(`{
"rawSql": "UPDATE test_multi_results SET name = 'updated' WHERE id = 1; SELECT id, name FROM test_multi_results WHERE id = 1;",
"format": "table"
}`),
TimeRange: backend.TimeRange{
From: fromStart,
To: fromStart.Add(1 * time.Hour),
},
},
},
}
// This should not panic
resp, err := exe.QueryData(t.Context(), query)
require.NoError(t, err)
queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
frames := queryResult.Frames
require.Len(t, frames, 1)
// Should only contain data from the SELECT part
frame := frames[0]
require.Equal(t, 2, len(frame.Fields)) // id, name
require.Equal(t, 1, frame.Rows()) // 1 row
// Verify the update worked
nameField := frame.Fields[1]
nameValue := nameField.At(0).(*string)
require.Equal(t, "updated", *nameValue)
})
})
t.Run("Test Postgres connection with pgpass file", func(t *testing.T) {
@@ -1385,10 +1560,10 @@ func TestIntegrationPostgres(t *testing.T) {
cnnstr := postgresTestDBConnString()
require.NotContains(t, cnnstr, "password=", "Make sure that password is not in the connection string")
dbPgpass, _, err := newPostgres(context.Background(), "error", 10000, dsInfo, cnnstr, logger, backend.DataSourceInstanceSettings{})
pgpassPool, _, err := newPostgres(t.Context(), "error", 10000, dsInfo, cnnstr, logger, backend.DataSourceInstanceSettings{})
require.NoError(t, err)
_, err = dbPgpass.Exec("SELECT 1") // Test connection
_, err = pgpassPool.Query(t.Context(), "SELECT 1") // Test connection
require.NoError(t, err)
})
}
pkg/tsdb/grafana-postgresql-datasource/proxy.go
+2 -29
View File
@@ -3,40 +3,13 @@ package postgres
import (
"context"
"net"
"time"
"github.com/lib/pq"
"golang.org/x/net/proxy"
)
// we wrap the proxy.Dialer to become dialer that the postgres module accepts
func newPostgresProxyDialer(dialer proxy.Dialer) pq.Dialer {
return &postgresProxyDialer{d: dialer}
}
type DialFunc = func(ctx context.Context, network string, address string) (net.Conn, error)
var _ pq.Dialer = (&postgresProxyDialer{})
// postgresProxyDialer implements the postgres dialer using a proxy dialer, as their functions differ slightly
type postgresProxyDialer struct {
d proxy.Dialer
}
// Dial uses the normal proxy dial function with the updated dialer
func (p *postgresProxyDialer) Dial(network, addr string) (c net.Conn, err error) {
return p.d.Dial(network, addr)
}
// DialTimeout uses the normal postgres dial timeout function with the updated dialer
func (p *postgresProxyDialer) DialTimeout(network, address string, timeout time.Duration) (net.Conn, error) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
return p.d.(proxy.ContextDialer).DialContext(ctx, network, address)
}
type PgxDialFunc = func(ctx context.Context, network string, address string) (net.Conn, error)
func newPgxDialFunc(dialer proxy.Dialer) PgxDialFunc {
func newDialFunc(dialer proxy.Dialer) DialFunc {
return func(ctx context.Context, network string, addr string) (net.Conn, error) {
return dialer.Dial(network, addr)
}
pkg/tsdb/grafana-postgresql-datasource/proxy_test.go
+8 -7
View File
@@ -1,12 +1,11 @@
package postgres
import (
"database/sql"
"fmt"
"net"
"testing"
"github.com/lib/pq"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/stretchr/testify/require"
"golang.org/x/net/proxy"
)
@@ -25,15 +24,17 @@ func TestPostgresProxyDriver(t *testing.T) {
cnnstr := fmt.Sprintf("postgres://auser:password@%s/db?sslmode=disable", dbURL)
t.Run("Connector should use dialer context that routes through the socks proxy to db", func(t *testing.T) {
connector, err := pq.NewConnector(cnnstr)
config, err := pgxpool.ParseConfig(cnnstr)
require.NoError(t, err)
dialer := newPostgresProxyDialer(&testDialer{})
connector.Dialer(dialer)
config.ConnConfig.DialFunc = newDialFunc(&testDialer{})
db := sql.OpenDB(connector)
err = db.Ping()
pool, err := pgxpool.NewWithConfig(t.Context(), config)
require.NoError(t, err)
defer pool.Close()
err = pool.Ping(t.Context())
require.Error(t, err)
require.Contains(t, err.Error(), "test-dialer is not functional")
})
}
pkg/tsdb/grafana-postgresql-datasource/sqleng/handler_checkhealth.go
+1 -20
View File
@@ -10,11 +10,10 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/backend/log"
"github.com/lib/pq"
)
func (e *DataSourceHandler) CheckHealth(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) {
err := e.Ping()
err := e.Ping(ctx)
if err != nil {
logCheckHealthError(ctx, e.dsInfo, err)
if strings.EqualFold(req.PluginContext.User.Role, "Admin") {
@@ -64,24 +63,6 @@ func ErrToHealthCheckResult(err error) (*backend.CheckHealthResult, error) {
}
}
if errors.Is(err, pq.ErrSSLNotSupported) {
res.Message = "SSL error: Failed to connect to the server"
}
if strings.HasPrefix(err.Error(), "pq") {
res.Message = "Database error: Failed to connect to the postgres server"
if unwrappedErr := errors.Unwrap(err); unwrappedErr != nil {
details["verboseMessage"] = unwrappedErr.Error()
}
}
var pqErr *pq.Error
if errors.As(err, &pqErr) {
if pqErr != nil {
if pqErr.Code != "" {
res.Message += fmt.Sprintf(". Postgres error code: %s", pqErr.Code.Name())
}
details["verboseMessage"] = pqErr.Message
}
}
if errors.Is(err, ErrParsingPostgresURL) {
res.Message = fmt.Sprintf("Connection string error: %s", ErrParsingPostgresURL.Error())
if unwrappedErr := errors.Unwrap(err); unwrappedErr != nil {
pkg/tsdb/grafana-postgresql-datasource/sqleng/handler_checkhealth_test.go
-10
View File
@@ -7,7 +7,6 @@ import (
"testing"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/lib/pq"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -31,15 +30,6 @@ func TestErrToHealthCheckResult(t *testing.T) {
JSONDetails: []byte(`{"errorDetailsLink":"https://grafana.com/docs/grafana/latest/datasources/postgres","verboseMessage":"foo\nread tcp: some op"}`),
},
},
{
name: "db error",
err: errors.Join(errors.New("foo"), &pq.Error{Message: pq.ErrCouldNotDetectUsername.Error(), Code: pq.ErrorCode("28P01")}),
want: &backend.CheckHealthResult{
Status: backend.HealthStatusError,
Message: "foo\npq: pq: Could not detect default username. Please provide one explicitly. Postgres error code: invalid_password",
JSONDetails: []byte(`{"errorDetailsLink":"https://grafana.com/docs/grafana/latest/datasources/postgres","verboseMessage":"pq: Could not detect default username. Please provide one explicitly"}`),
},
},
{
name: "regular error",
err: errors.New("internal server error"),
pkg/tsdb/grafana-postgresql-datasource/sqleng/sql_engine.go
+343 -97
View File
@@ -2,7 +2,6 @@ package sqleng
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
@@ -19,6 +18,9 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend/log"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana-plugin-sdk-go/data/sqlutil"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgtype"
"github.com/jackc/pgx/v5/pgxpool"
)
// MetaKeyExecutedQueryString is the key where the executed query should get stored
@@ -81,13 +83,13 @@ type DataPluginConfiguration struct {
type DataSourceHandler struct {
macroEngine SQLMacroEngine
queryResultTransformer SqlQueryResultTransformer
db *sql.DB
timeColumnNames []string
metricColumnTypes []string
log log.Logger
dsInfo DataSourceInfo
rowLimit int64
userError string
pool *pgxpool.Pool
}
type QueryJson struct {
@@ -112,7 +114,7 @@ func (e *DataSourceHandler) TransformQueryError(logger log.Logger, err error) er
return e.queryResultTransformer.TransformQueryError(logger, err)
}
func NewQueryDataHandler(userFacingDefaultError string, db *sql.DB, config DataPluginConfiguration, queryResultTransformer SqlQueryResultTransformer,
func NewQueryDataHandler(userFacingDefaultError string, p *pgxpool.Pool, config DataPluginConfiguration, queryResultTransformer SqlQueryResultTransformer,
macroEngine SQLMacroEngine, log log.Logger) (*DataSourceHandler, error) {
queryDataHandler := DataSourceHandler{
queryResultTransformer: queryResultTransformer,
@@ -132,7 +134,7 @@ func NewQueryDataHandler(userFacingDefaultError string, db *sql.DB, config DataP
queryDataHandler.metricColumnTypes = config.MetricColumnTypes
}
queryDataHandler.db = db
queryDataHandler.pool = p
return &queryDataHandler, nil
}
@@ -143,16 +145,16 @@ type DBDataResponse struct {
func (e *DataSourceHandler) Dispose() {
e.log.Debug("Disposing DB...")
if e.db != nil {
if err := e.db.Close(); err != nil {
e.log.Error("Failed to dispose db", "error", err)
}
if e.pool != nil {
e.pool.Close()
}
e.log.Debug("DB disposed")
}
func (e *DataSourceHandler) Ping() error {
return e.db.Ping()
func (e *DataSourceHandler) Ping(ctx context.Context) error {
return e.pool.Ping(ctx)
}
func (e *DataSourceHandler) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) {
@@ -167,13 +169,13 @@ func (e *DataSourceHandler) QueryData(ctx context.Context, req *backend.QueryDat
}
err := json.Unmarshal(query.JSON, &queryjson)
if err != nil {
return nil, fmt.Errorf("error unmarshal query json: %w", err)
return nil, backend.DownstreamErrorf("error unmarshal query json: %s", err.Error())
}
// the fill-params are only stored inside this function, during query-interpolation. we do not support
// sending them in "from the outside"
if queryjson.Fill || queryjson.FillInterval != 0.0 || queryjson.FillMode != "" || queryjson.FillValue != 0.0 {
return nil, fmt.Errorf("query fill-parameters not supported")
return nil, backend.DownstreamErrorf("query fill-parameters not supported")
}
if queryjson.RawSql == "" {
@@ -181,7 +183,7 @@ func (e *DataSourceHandler) QueryData(ctx context.Context, req *backend.QueryDat
}
wg.Add(1)
go e.executeQuery(query, &wg, ctx, ch, queryjson)
go e.executeQuery(ctx, query, &wg, ch, queryjson)
}
wg.Wait()
@@ -196,8 +198,21 @@ func (e *DataSourceHandler) QueryData(ctx context.Context, req *backend.QueryDat
return result, nil
}
func (e *DataSourceHandler) executeQuery(query backend.DataQuery, wg *sync.WaitGroup, queryContext context.Context,
ch chan DBDataResponse, queryJson QueryJson) {
func (e *DataSourceHandler) execQuery(ctx context.Context, query string) ([]*pgconn.Result, error) {
c, err := e.pool.Acquire(ctx)
if err != nil {
return nil, backend.DownstreamErrorf("failed to acquire connection: %w", err)
}
defer c.Release()
mrr := c.Conn().PgConn().Exec(ctx, query)
// Close returns the first error that occurred during the MultiResultReader's use. We will log that later.
defer mrr.Close() //nolint:errcheck
return mrr.ReadAll()
}
func (e *DataSourceHandler) executeQuery(queryContext context.Context, query backend.DataQuery, wg *sync.WaitGroup,
ch chan DBDataResponse, queryJSON QueryJson) {
defer wg.Done()
queryResult := DBDataResponse{
dataResponse: backend.DataResponse{},
@@ -205,84 +220,77 @@ func (e *DataSourceHandler) executeQuery(query backend.DataQuery, wg *sync.WaitG
}
logger := e.log.FromContext(queryContext)
defer e.handlePanic(logger, &queryResult, ch)
defer func() {
if r := recover(); r != nil {
logger.Error("ExecuteQuery panic", "error", r, "stack", string(debug.Stack()))
if theErr, ok := r.(error); ok {
queryResult.dataResponse.Error = theErr
queryResult.dataResponse.ErrorSource = backend.ErrorSourcePlugin
} else if theErrString, ok := r.(string); ok {
queryResult.dataResponse.Error = errors.New(theErrString)
queryResult.dataResponse.ErrorSource = backend.ErrorSourcePlugin
} else {
queryResult.dataResponse.Error = fmt.Errorf("unexpected error - %s", e.userError)
queryResult.dataResponse.ErrorSource = backend.ErrorSourceDownstream
}
ch <- queryResult
}
}()
if queryJson.RawSql == "" {
if queryJSON.RawSql == "" {
panic("Query model property rawSql should not be empty at this point")
}
timeRange := query.TimeRange
errAppendDebug := func(frameErr string, err error, query string, source backend.ErrorSource) {
var emptyFrame data.Frame
emptyFrame.SetMeta(&data.FrameMeta{
ExecutedQueryString: query,
})
if isDownstreamError(err) {
source = backend.ErrorSourceDownstream
}
queryResult.dataResponse.Error = fmt.Errorf("%s: %w", frameErr, err)
queryResult.dataResponse.ErrorSource = source
queryResult.dataResponse.Frames = data.Frames{&emptyFrame}
ch <- queryResult
}
// global substitutions
interpolatedQuery := Interpolate(query, timeRange, e.dsInfo.JsonData.TimeInterval, queryJson.RawSql)
interpolatedQuery := Interpolate(query, query.TimeRange, e.dsInfo.JsonData.TimeInterval, queryJSON.RawSql)
// data source specific substitutions
interpolatedQuery, err := e.macroEngine.Interpolate(&query, timeRange, interpolatedQuery)
interpolatedQuery, err := e.macroEngine.Interpolate(&query, query.TimeRange, interpolatedQuery)
if err != nil {
errAppendDebug("interpolation failed", e.TransformQueryError(logger, err), interpolatedQuery, backend.ErrorSourcePlugin)
e.handleQueryError("interpolation failed", e.TransformQueryError(logger, err), interpolatedQuery, backend.ErrorSourceDownstream, ch, queryResult)
return
}
rows, err := e.db.QueryContext(queryContext, interpolatedQuery)
results, err := e.execQuery(queryContext, interpolatedQuery)
if err != nil {
errAppendDebug("db query error", e.TransformQueryError(logger, err), interpolatedQuery, backend.ErrorSourceDownstream)
e.handleQueryError("db query error", e.TransformQueryError(logger, err), interpolatedQuery, backend.ErrorSourceDownstream, ch, queryResult)
return
}
defer func() {
if err := rows.Close(); err != nil {
logger.Warn("Failed to close rows", "err", err)
qm, err := e.newProcessCfg(queryContext, query, results, interpolatedQuery)
if err != nil {
e.handleQueryError("failed to get configurations", err, interpolatedQuery, backend.ErrorSourceDownstream, ch, queryResult)
return
}
frame, err := convertResultsToFrame(results, e.rowLimit)
if err != nil {
e.handleQueryError("convert frame from rows error", err, interpolatedQuery, backend.ErrorSourceDownstream, ch, queryResult)
return
}
e.processFrame(frame, qm, queryResult, ch, logger)
}
func (e *DataSourceHandler) handleQueryError(frameErr string, err error, query string, source backend.ErrorSource, ch chan DBDataResponse, queryResult DBDataResponse) {
var emptyFrame data.Frame
emptyFrame.SetMeta(&data.FrameMeta{ExecutedQueryString: query})
if isDownstreamError(err) {
source = backend.ErrorSourceDownstream
}
queryResult.dataResponse.Error = fmt.Errorf("%s: %w", frameErr, err)
queryResult.dataResponse.ErrorSource = source
queryResult.dataResponse.Frames = data.Frames{&emptyFrame}
ch <- queryResult
}
func (e *DataSourceHandler) handlePanic(logger log.Logger, queryResult *DBDataResponse, ch chan DBDataResponse) {
if r := recover(); r != nil {
logger.Error("ExecuteQuery panic", "error", r, "stack", string(debug.Stack()))
if theErr, ok := r.(error); ok {
queryResult.dataResponse.Error = theErr
queryResult.dataResponse.ErrorSource = backend.ErrorSourcePlugin
} else if theErrString, ok := r.(string); ok {
queryResult.dataResponse.Error = errors.New(theErrString)
queryResult.dataResponse.ErrorSource = backend.ErrorSourcePlugin
} else {
queryResult.dataResponse.Error = fmt.Errorf("unexpected error - %s", e.userError)
queryResult.dataResponse.ErrorSource = backend.ErrorSourceDownstream
}
}()
qm, err := e.newProcessCfg(query, queryContext, rows, interpolatedQuery)
if err != nil {
errAppendDebug("failed to get configurations", err, interpolatedQuery, backend.ErrorSourcePlugin)
return
}
// Convert row.Rows to dataframe
stringConverters := e.queryResultTransformer.GetConverterList()
frame, err := sqlutil.FrameFromRows(rows, e.rowLimit, sqlutil.ToConverters(stringConverters...)...)
if err != nil {
errAppendDebug("convert frame from rows error", err, interpolatedQuery, backend.ErrorSourcePlugin)
return
ch <- *queryResult
}
}
func (e *DataSourceHandler) processFrame(frame *data.Frame, qm *dataQueryModel, queryResult DBDataResponse, ch chan DBDataResponse, logger log.Logger) {
if frame.Meta == nil {
frame.Meta = &data.FrameMeta{}
}
frame.Meta.ExecutedQueryString = interpolatedQuery
frame.Meta.ExecutedQueryString = qm.InterpolatedQuery
// If no rows were returned, clear any previously set `Fields` with a single empty `data.Field` slice.
// Then assign `queryResult.dataResponse.Frames` the current single frame with that single empty Field.
@@ -296,14 +304,14 @@ func (e *DataSourceHandler) executeQuery(query backend.DataQuery, wg *sync.WaitG
}
if err := convertSQLTimeColumnsToEpochMS(frame, qm); err != nil {
errAppendDebug("converting time columns failed", err, interpolatedQuery, backend.ErrorSourcePlugin)
e.handleQueryError("converting time columns failed", err, qm.InterpolatedQuery, backend.ErrorSourceDownstream, ch, queryResult)
return
}
if qm.Format == dataQueryFormatSeries {
// time series has to have time column
if qm.timeIndex == -1 {
errAppendDebug("db has no time column", errors.New("time column is missing; make sure your data includes a time column for time series format or switch to a table format that doesn't require it"), interpolatedQuery, backend.ErrorSourceDownstream)
e.handleQueryError("db has no time column", errors.New("time column is missing; make sure your data includes a time column for time series format or switch to a table format that doesn't require it"), qm.InterpolatedQuery, backend.ErrorSourceDownstream, ch, queryResult)
return
}
@@ -321,7 +329,7 @@ func (e *DataSourceHandler) executeQuery(query backend.DataQuery, wg *sync.WaitG
var err error
if frame, err = convertSQLValueColumnToFloat(frame, i); err != nil {
errAppendDebug("convert value to float failed", err, interpolatedQuery, backend.ErrorSourcePlugin)
e.handleQueryError("convert value to float failed", err, qm.InterpolatedQuery, backend.ErrorSourceDownstream, ch, queryResult)
return
}
}
@@ -332,7 +340,7 @@ func (e *DataSourceHandler) executeQuery(query backend.DataQuery, wg *sync.WaitG
originalData := frame
frame, err = data.LongToWide(frame, qm.FillMissing)
if err != nil {
errAppendDebug("failed to convert long to wide series when converting from dataframe", err, interpolatedQuery, backend.ErrorSourcePlugin)
e.handleQueryError("failed to convert long to wide series when converting from dataframe", err, qm.InterpolatedQuery, backend.ErrorSourceDownstream, ch, queryResult)
return
}
@@ -364,6 +372,7 @@ func (e *DataSourceHandler) executeQuery(query backend.DataQuery, wg *sync.WaitG
if err != nil {
logger.Error("Failed to resample dataframe", "err", err)
frame.AppendNotices(data.Notice{Text: "Failed to resample dataframe", Severity: data.NoticeSeverityWarning})
return
}
}
}
@@ -384,15 +393,38 @@ var Interpolate = func(query backend.DataQuery, timeRange backend.TimeRange, tim
return sql
}
func (e *DataSourceHandler) newProcessCfg(query backend.DataQuery, queryContext context.Context,
rows *sql.Rows, interpolatedQuery string) (*dataQueryModel, error) {
columnNames, err := rows.Columns()
if err != nil {
return nil, err
func (e *DataSourceHandler) newProcessCfg(queryContext context.Context, query backend.DataQuery,
results []*pgconn.Result, interpolatedQuery string) (*dataQueryModel, error) {
// Calculate total number of fields to preallocate slices
totalFields := 0
for _, result := range results {
totalFields += len(result.FieldDescriptions)
}
columnTypes, err := rows.ColumnTypes()
if err != nil {
return nil, err
columnNames := make([]string, 0, totalFields)
columnTypes := make([]string, 0, totalFields)
// The results will contain column information in the metadata
for _, result := range results {
// Get column names from the result metadata
for _, field := range result.FieldDescriptions {
columnNames = append(columnNames, field.Name)
pqtype, ok := pgtype.NewMap().TypeForOID(field.DataTypeOID)
if !ok {
// Handle special cases for field types
switch field.DataTypeOID {
case pgtype.TimetzOID:
columnTypes = append(columnTypes, "timetz")
// money type is 790
case 790:
columnTypes = append(columnTypes, "money")
default:
columnTypes = append(columnTypes, "unknown")
}
} else {
columnTypes = append(columnTypes, pqtype.Name)
}
}
}
qm := &dataQueryModel{
@@ -405,23 +437,23 @@ func (e *DataSourceHandler) newProcessCfg(query backend.DataQuery, queryContext
queryContext: queryContext,
}
queryJson := QueryJson{}
err = json.Unmarshal(query.JSON, &queryJson)
queryJSON := QueryJson{}
err := json.Unmarshal(query.JSON, &queryJSON)
if err != nil {
return nil, err
}
if queryJson.Fill {
if queryJSON.Fill {
qm.FillMissing = &data.FillMissing{}
qm.Interval = time.Duration(queryJson.FillInterval * float64(time.Second))
switch strings.ToLower(queryJson.FillMode) {
qm.Interval = time.Duration(queryJSON.FillInterval * float64(time.Second))
switch strings.ToLower(queryJSON.FillMode) {
case "null":
qm.FillMissing.Mode = data.FillModeNull
case "previous":
qm.FillMissing.Mode = data.FillModePrevious
case "value":
qm.FillMissing.Mode = data.FillModeValue
qm.FillMissing.Value = queryJson.FillValue
qm.FillMissing.Value = queryJSON.FillValue
default:
}
}
@@ -429,13 +461,14 @@ func (e *DataSourceHandler) newProcessCfg(query backend.DataQuery, queryContext
qm.TimeRange.From = query.TimeRange.From.UTC()
qm.TimeRange.To = query.TimeRange.To.UTC()
switch queryJson.Format {
case "time_series":
qm.Format = dataQueryFormatSeries
// Default to time_series if no format is provided
switch queryJSON.Format {
case "table":
qm.Format = dataQueryFormatTable
case "time_series":
fallthrough
default:
panic(fmt.Sprintf("Unrecognized query model format: %q", queryJson.Format))
qm.Format = dataQueryFormatSeries
}
for i, col := range qm.columnNames {
@@ -456,7 +489,7 @@ func (e *DataSourceHandler) newProcessCfg(query backend.DataQuery, queryContext
qm.metricIndex = i
default:
if qm.metricIndex == -1 {
columnType := qm.columnTypes[i].DatabaseTypeName()
columnType := qm.columnTypes[i]
for _, mct := range e.metricColumnTypes {
if columnType == mct {
qm.metricIndex = i
@@ -487,7 +520,7 @@ type dataQueryModel struct {
FillMissing *data.FillMissing // property not set until after Interpolate()
Interval time.Duration
columnNames []string
columnTypes []*sql.ColumnType
columnTypes []string
timeIndex int
timeEndIndex int
metricIndex int
@@ -511,6 +544,219 @@ func convertSQLTimeColumnsToEpochMS(frame *data.Frame, qm *dataQueryModel) error
return nil
}
func convertResultsToFrame(results []*pgconn.Result, rowLimit int64) (*data.Frame, error) {
m := pgtype.NewMap()
// Find the first SELECT result to establish the frame structure
var firstSelectResult *pgconn.Result
for _, result := range results {
if result.CommandTag.Select() {
firstSelectResult = result
break
}
}
// If no SELECT results found, return empty frame
if firstSelectResult == nil {
return data.NewFrame(""), nil
}
// Create frame structure based on the first SELECT result
fields := make(data.Fields, len(firstSelectResult.FieldDescriptions))
fieldTypes, err := getFieldTypesFromDescriptions(firstSelectResult.FieldDescriptions, m)
if err != nil {
return nil, err
}
for i, v := range firstSelectResult.FieldDescriptions {
fields[i] = data.NewFieldFromFieldType(fieldTypes[i], 0)
fields[i].Name = v.Name
}
frame := *data.NewFrame("", fields...)
// Process all SELECT results, but validate column compatibility
for _, result := range results {
// Skip non-select statements
if !result.CommandTag.Select() {
continue
}
// Validate that this result has the same structure as the frame
if len(result.FieldDescriptions) != len(frame.Fields) {
return nil, fmt.Errorf("incompatible result structure: expected %d columns, got %d columns",
len(frame.Fields), len(result.FieldDescriptions))
}
// Validate column names and types match
for i, fd := range result.FieldDescriptions {
if fd.Name != frame.Fields[i].Name {
return nil, fmt.Errorf("column name mismatch at position %d: expected %q, got %q",
i, frame.Fields[i].Name, fd.Name)
}
}
fieldDescriptions := result.FieldDescriptions
for rowIdx := range result.Rows {
if rowIdx == int(rowLimit) {
frame.AppendNotices(data.Notice{
Severity: data.NoticeSeverityWarning,
Text: fmt.Sprintf("Results have been limited to %v because the SQL row limit was reached", rowLimit),
})
break
}
row := make([]any, len(fieldDescriptions))
for colIdx, fd := range fieldDescriptions {
rawValue := result.Rows[rowIdx][colIdx]
if rawValue == nil {
row[colIdx] = nil
continue
}
convertedValue, err := convertPostgresValue(rawValue, fd, m)
if err != nil {
return nil, err
}
row[colIdx] = convertedValue
}
// Validate row length matches frame field count before appending
if len(row) != len(frame.Fields) {
return nil, fmt.Errorf("row data length mismatch: expected %d values, got %d values",
len(frame.Fields), len(row))
}
frame.AppendRow(row...)
}
}
return &frame, nil
}
// convertPostgresValue converts a raw PostgreSQL value to the appropriate Go type
func convertPostgresValue(rawValue []byte, fd pgconn.FieldDescription, m *pgtype.Map) (interface{}, error) {
dataTypeOID := fd.DataTypeOID
format := fd.Format
// Convert based on type
switch fd.DataTypeOID {
case pgtype.Int2OID:
var d *int16
scanPlan := m.PlanScan(dataTypeOID, format, &d)
err := scanPlan.Scan(rawValue, &d)
if err != nil {
return nil, err
}
return d, nil
case pgtype.Int4OID:
var d *int32
scanPlan := m.PlanScan(dataTypeOID, format, &d)
err := scanPlan.Scan(rawValue, &d)
if err != nil {
return nil, err
}
return d, nil
case pgtype.Int8OID:
var d *int64
scanPlan := m.PlanScan(dataTypeOID, format, &d)
err := scanPlan.Scan(rawValue, &d)
if err != nil {
return nil, err
}
return d, nil
case pgtype.NumericOID, pgtype.Float8OID, pgtype.Float4OID:
var d *float64
scanPlan := m.PlanScan(dataTypeOID, format, &d)
err := scanPlan.Scan(rawValue, &d)
if err != nil {
return nil, err
}
return d, nil
case pgtype.BoolOID:
var d *bool
scanPlan := m.PlanScan(dataTypeOID, format, &d)
err := scanPlan.Scan(rawValue, &d)
if err != nil {
return nil, err
}
return d, nil
case pgtype.ByteaOID:
d, err := pgtype.ByteaCodec.DecodeValue(pgtype.ByteaCodec{}, m, dataTypeOID, format, rawValue)
if err != nil {
return nil, err
}
str := string(d.([]byte))
return &str, nil
case pgtype.TimestampOID, pgtype.TimestamptzOID, pgtype.DateOID:
var d *time.Time
scanPlan := m.PlanScan(dataTypeOID, format, &d)
err := scanPlan.Scan(rawValue, &d)
if err != nil {
return nil, err
}
return d, nil
case pgtype.TimeOID, pgtype.TimetzOID:
var d *string
scanPlan := m.PlanScan(dataTypeOID, format, &d)
err := scanPlan.Scan(rawValue, &d)
if err != nil {
return nil, err
}
return d, nil
case pgtype.JSONOID, pgtype.JSONBOID:
var d *string
scanPlan := m.PlanScan(dataTypeOID, format, &d)
err := scanPlan.Scan(rawValue, &d)
if err != nil {
return nil, err
}
// Handle null JSON values
if d == nil {
return nil, nil
}
j := json.RawMessage(*d)
return &j, nil
default:
var d *string
scanPlan := m.PlanScan(dataTypeOID, format, &d)
err := scanPlan.Scan(rawValue, &d)
if err != nil {
return nil, err
}
return d, nil
}
}
func getFieldTypesFromDescriptions(fieldDescriptions []pgconn.FieldDescription, m *pgtype.Map) ([]data.FieldType, error) {
fieldTypes := make([]data.FieldType, len(fieldDescriptions))
for i, v := range fieldDescriptions {
typeName, ok := m.TypeForOID(v.DataTypeOID)
if !ok {
fieldTypes[i] = data.FieldTypeNullableString
} else {
switch typeName.Name {
case "int2":
fieldTypes[i] = data.FieldTypeNullableInt16
case "int4":
fieldTypes[i] = data.FieldTypeNullableInt32
case "int8":
fieldTypes[i] = data.FieldTypeNullableInt64
case "float4", "float8", "numeric":
fieldTypes[i] = data.FieldTypeNullableFloat64
case "bool":
fieldTypes[i] = data.FieldTypeNullableBool
case "timestamptz", "timestamp", "date":
fieldTypes[i] = data.FieldTypeNullableTime
case "json", "jsonb":
fieldTypes[i] = data.FieldTypeNullableJSON
default:
fieldTypes[i] = data.FieldTypeNullableString
}
}
}
return fieldTypes, nil
}
// convertSQLTimeColumnToEpochMS converts column named time to unix timestamp in milliseconds
// to make native datetime types and epoch dates work in annotation and table queries.
func convertSQLTimeColumnToEpochMS(frame *data.Frame, timeIndex int) error {
pkg/tsdb/grafana-postgresql-datasource/sqleng/sql_engine_test.go
+242
View File
@@ -9,6 +9,8 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana-plugin-sdk-go/data/sqlutil"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgtype"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -425,6 +427,246 @@ func TestSQLEngine(t *testing.T) {
})
}
func TestConvertResultsToFrame(t *testing.T) {
// Import the pgx packages needed for testing
// These imports are included in the main file but need to be accessible for tests
t.Run("convertResultsToFrame with single result", func(t *testing.T) {
// Create mock field descriptions
fieldDescs := []pgconn.FieldDescription{
{Name: "id", DataTypeOID: pgtype.Int4OID},
{Name: "name", DataTypeOID: pgtype.TextOID},
{Name: "value", DataTypeOID: pgtype.Float8OID},
}
// Create mock result data
mockRows := [][][]byte{
{[]byte("1"), []byte("test1"), []byte("10.5")},
{[]byte("2"), []byte("test2"), []byte("20.7")},
}
// Create mock result
result := &pgconn.Result{
FieldDescriptions: fieldDescs,
Rows: mockRows,
}
result.CommandTag = pgconn.NewCommandTag("SELECT 2")
results := []*pgconn.Result{result}
frame, err := convertResultsToFrame(results, 1000)
require.NoError(t, err)
require.NotNil(t, frame)
require.Equal(t, 3, len(frame.Fields))
require.Equal(t, 2, frame.Rows())
// Verify field names
require.Equal(t, "id", frame.Fields[0].Name)
require.Equal(t, "name", frame.Fields[1].Name)
require.Equal(t, "value", frame.Fields[2].Name)
})
t.Run("convertResultsToFrame with multiple compatible results", func(t *testing.T) {
// Create mock field descriptions (same structure for both results)
fieldDescs := []pgconn.FieldDescription{
{Name: "id", DataTypeOID: pgtype.Int4OID},
{Name: "name", DataTypeOID: pgtype.TextOID},
}
// Create first result
mockRows1 := [][][]byte{
{[]byte("1"), []byte("test1")},
{[]byte("2"), []byte("test2")},
}
result1 := &pgconn.Result{
FieldDescriptions: fieldDescs,
Rows: mockRows1,
}
result1.CommandTag = pgconn.NewCommandTag("SELECT 2")
// Create second result with same structure
mockRows2 := [][][]byte{
{[]byte("3"), []byte("test3")},
{[]byte("4"), []byte("test4")},
}
result2 := &pgconn.Result{
FieldDescriptions: fieldDescs,
Rows: mockRows2,
}
result2.CommandTag = pgconn.NewCommandTag("SELECT 2")
results := []*pgconn.Result{result1, result2}
frame, err := convertResultsToFrame(results, 1000)
require.NoError(t, err)
require.NotNil(t, frame)
require.Equal(t, 2, len(frame.Fields))
require.Equal(t, 4, frame.Rows()) // Should have rows from both results
// Verify field names
require.Equal(t, "id", frame.Fields[0].Name)
require.Equal(t, "name", frame.Fields[1].Name)
})
t.Run("convertResultsToFrame with row limit", func(t *testing.T) {
// Create mock field descriptions
fieldDescs := []pgconn.FieldDescription{
{Name: "id", DataTypeOID: pgtype.Int4OID},
}
// Create mock result data with 3 rows
mockRows := [][][]byte{
{[]byte("1")},
{[]byte("2")},
{[]byte("3")},
}
result := &pgconn.Result{
FieldDescriptions: fieldDescs,
Rows: mockRows,
}
result.CommandTag = pgconn.NewCommandTag("SELECT 3")
results := []*pgconn.Result{result}
// Set row limit to 2
frame, err := convertResultsToFrame(results, 2)
require.NoError(t, err)
require.NotNil(t, frame)
require.Equal(t, 1, len(frame.Fields))
require.Equal(t, 2, frame.Rows()) // Should be limited to 2 rows
// Should have a notice about the limit
require.NotNil(t, frame.Meta)
require.Len(t, frame.Meta.Notices, 1)
require.Contains(t, frame.Meta.Notices[0].Text, "Results have been limited to 2")
})
t.Run("convertResultsToFrame with mixed SELECT and non-SELECT results", func(t *testing.T) {
// Create a non-SELECT result (should be skipped)
nonSelectResult := &pgconn.Result{}
nonSelectResult.CommandTag = pgconn.NewCommandTag("UPDATE 1")
// Create a SELECT result
fieldDescs := []pgconn.FieldDescription{
{Name: "id", DataTypeOID: pgtype.Int4OID},
}
mockRows := [][][]byte{
{[]byte("1")},
}
selectResult := &pgconn.Result{
FieldDescriptions: fieldDescs,
Rows: mockRows,
}
selectResult.CommandTag = pgconn.NewCommandTag("SELECT 1")
results := []*pgconn.Result{nonSelectResult, selectResult}
frame, err := convertResultsToFrame(results, 1000)
require.NoError(t, err)
require.NotNil(t, frame)
require.Equal(t, 1, len(frame.Fields))
require.Equal(t, 1, frame.Rows())
})
t.Run("convertResultsToFrame with no SELECT results", func(t *testing.T) {
// Create only non-SELECT results
result1 := &pgconn.Result{}
result1.CommandTag = pgconn.NewCommandTag("UPDATE 1")
result2 := &pgconn.Result{}
result2.CommandTag = pgconn.NewCommandTag("INSERT 1")
results := []*pgconn.Result{result1, result2}
frame, err := convertResultsToFrame(results, 1000)
require.NoError(t, err)
require.NotNil(t, frame)
require.Equal(t, 0, len(frame.Fields))
require.Equal(t, 0, frame.Rows())
})
t.Run("convertResultsToFrame with multiple results and row limit per result", func(t *testing.T) {
// Create mock field descriptions (same structure for both results)
fieldDescs := []pgconn.FieldDescription{
{Name: "id", DataTypeOID: pgtype.Int4OID},
}
// Create first result with 3 rows
mockRows1 := [][][]byte{
{[]byte("1")},
{[]byte("2")},
{[]byte("3")},
}
result1 := &pgconn.Result{
FieldDescriptions: fieldDescs,
Rows: mockRows1,
}
result1.CommandTag = pgconn.NewCommandTag("SELECT 3")
// Create second result with 3 rows
mockRows2 := [][][]byte{
{[]byte("4")},
{[]byte("5")},
{[]byte("6")},
}
result2 := &pgconn.Result{
FieldDescriptions: fieldDescs,
Rows: mockRows2,
}
result2.CommandTag = pgconn.NewCommandTag("SELECT 3")
results := []*pgconn.Result{result1, result2}
// Set row limit to 2 (should limit each result to 2 rows)
frame, err := convertResultsToFrame(results, 2)
require.NoError(t, err)
require.NotNil(t, frame)
require.Equal(t, 1, len(frame.Fields))
require.Equal(t, 4, frame.Rows()) // 2 rows from each result
// Should have notices about the limit from both results
require.NotNil(t, frame.Meta)
require.Len(t, frame.Meta.Notices, 2)
require.Contains(t, frame.Meta.Notices[0].Text, "Results have been limited to 2")
require.Contains(t, frame.Meta.Notices[1].Text, "Results have been limited to 2")
})
t.Run("convertResultsToFrame handles null values correctly", func(t *testing.T) {
// Create mock field descriptions
fieldDescs := []pgconn.FieldDescription{
{Name: "id", DataTypeOID: pgtype.Int4OID},
{Name: "name", DataTypeOID: pgtype.TextOID},
}
// Create mock result data with null values
mockRows := [][][]byte{
{[]byte("1"), nil}, // null name
{nil, []byte("test2")}, // null id
}
result := &pgconn.Result{
FieldDescriptions: fieldDescs,
Rows: mockRows,
}
result.CommandTag = pgconn.NewCommandTag("SELECT 2")
results := []*pgconn.Result{result}
frame, err := convertResultsToFrame(results, 1000)
require.NoError(t, err)
require.NotNil(t, frame)
require.Equal(t, 2, len(frame.Fields))
require.Equal(t, 2, frame.Rows())
// Check that null values are handled correctly
// The exact representation depends on the field type, but should not panic
require.NotPanics(t, func() {
frame.Fields[0].At(1) // null id
frame.Fields[1].At(0) // null name
})
})
}
type testQueryResultTransformer struct {
transformQueryErrorWasCalled bool
}
pkg/tsdb/grafana-postgresql-datasource/standalone/main.go
+1 -4
View File
@@ -6,16 +6,13 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/backend/datasource"
"github.com/grafana/grafana-plugin-sdk-go/backend/log"
"github.com/grafana/grafana/pkg/setting"
postgres "github.com/grafana/grafana/pkg/tsdb/grafana-postgresql-datasource"
)
func main() {
// No need to pass logger name, it will be set by the plugin SDK
logger := backend.NewLoggerWith()
// TODO: get rid of setting.NewCfg() once PostgresDSUsePGX is removed
cfg := setting.NewCfg()
if err := datasource.Manage("grafana-postgresql-datasource", postgres.NewInstanceSettings(logger, cfg.DataPath), datasource.ManageOpts{}); err != nil {
if err := datasource.Manage("grafana-postgresql-datasource", postgres.NewInstanceSettings(logger), datasource.ManageOpts{}); err != nil {
log.DefaultLogger.Error(err.Error())
os.Exit(1)
}
pkg/tsdb/grafana-postgresql-datasource/testdata/table/types_datetime.golden.jsonc
Vendored
+26 -38
View File
@@ -9,14 +9,14 @@
// }
// Name:
// Dimensions: 12 Fields by 2 Rows
// +----------------------------------------+----------------------------------------+--------------------------------------+--------------------------------------+---------------------------------+---------------------------------+--------------------------------------+--------------------------------------+----------------------------------------+----------------------------------------+-----------------+-----------------+
// | Name: ts | Name: tsnn | Name: tsz | Name: tsznn | Name: d | Name: dnn | Name: t | Name: tnn | Name: tz | Name: tznn | Name: i | Name: inn |
// | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: |
// | Type: []*time.Time | Type: []*time.Time | Type: []*time.Time | Type: []*time.Time | Type: []*time.Time | Type: []*time.Time | Type: []*time.Time | Type: []*time.Time | Type: []*time.Time | Type: []*time.Time | Type: []*string | Type: []*string |
// +----------------------------------------+----------------------------------------+--------------------------------------+--------------------------------------+---------------------------------+---------------------------------+--------------------------------------+--------------------------------------+----------------------------------------+----------------------------------------+-----------------+-----------------+
// | 2023-11-15 05:06:07.123456 +0000 +0000 | 2023-11-15 05:06:08.123456 +0000 +0000 | 2021-07-22 11:22:33.654321 +0000 UTC | 2021-07-22 11:22:34.654321 +0000 UTC | 2023-12-20 00:00:00 +0000 +0000 | 2023-12-21 00:00:00 +0000 +0000 | 0000-01-01 12:34:56.234567 +0000 UTC | 0000-01-01 12:34:57.234567 +0000 UTC | 0000-01-01 23:12:36.765432 +0100 +0100 | 0000-01-01 23:12:37.765432 +0100 +0100 | 00:00:00.987654 | 00:00:00.887654 |
// | null | 2023-11-15 05:06:09.123456 +0000 +0000 | null | 2021-07-22 11:22:35.654321 +0000 UTC | null | 2023-12-22 00:00:00 +0000 +0000 | null | 0000-01-01 12:34:58.234567 +0000 UTC | null | 0000-01-01 23:12:38.765432 +0100 +0100 | null | 00:00:00.787654 |
// +----------------------------------------+----------------------------------------+--------------------------------------+--------------------------------------+---------------------------------+---------------------------------+--------------------------------------+--------------------------------------+----------------------------------------+----------------------------------------+-----------------+-----------------+
// +--------------------------------------+--------------------------------------+----------------------------------------+----------------------------------------+-------------------------------+-------------------------------+-----------------+-----------------+--------------------+--------------------+-----------------+-----------------+
// | Name: ts | Name: tsnn | Name: tsz | Name: tsznn | Name: d | Name: dnn | Name: t | Name: tnn | Name: tz | Name: tznn | Name: i | Name: inn |
// | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: |
// | Type: []*time.Time | Type: []*time.Time | Type: []*time.Time | Type: []*time.Time | Type: []*time.Time | Type: []*time.Time | Type: []*string | Type: []*string | Type: []*string | Type: []*string | Type: []*string | Type: []*string |
// +--------------------------------------+--------------------------------------+----------------------------------------+----------------------------------------+-------------------------------+-------------------------------+-----------------+-----------------+--------------------+--------------------+-----------------+-----------------+
// | 2023-11-15 05:06:07.123456 +0000 UTC | 2023-11-15 05:06:08.123456 +0000 UTC | 2021-07-22 11:22:33.654321 +0000 +0000 | 2021-07-22 11:22:34.654321 +0000 +0000 | 2023-12-20 00:00:00 +0000 UTC | 2023-12-21 00:00:00 +0000 UTC | 12:34:56.234567 | 12:34:57.234567 | 23:12:36.765432+01 | 23:12:37.765432+01 | 00:00:00.987654 | 00:00:00.887654 |
// | null | 2023-11-15 05:06:09.123456 +0000 UTC | null | 2021-07-22 11:22:35.654321 +0000 +0000 | null | 2023-12-22 00:00:00 +0000 UTC | null | 12:34:58.234567 | null | 23:12:38.765432+01 | null | 00:00:00.787654 |
// +--------------------------------------+--------------------------------------+----------------------------------------+----------------------------------------+-------------------------------+-------------------------------+-----------------+-----------------+--------------------+--------------------+-----------------+-----------------+
//
//
// 🌟 This was machine generated. Do not edit. 🌟
@@ -83,33 +83,33 @@
},
{
"name": "t",
"type": "time",
"type": "string",
"typeInfo": {
"frame": "time.Time",
"frame": "string",
"nullable": true
}
},
{
"name": "tnn",
"type": "time",
"type": "string",
"typeInfo": {
"frame": "time.Time",
"frame": "string",
"nullable": true
}
},
{
"name": "tz",
"type": "time",
"type": "string",
"typeInfo": {
"frame": "time.Time",
"frame": "string",
"nullable": true
}
},
{
"name": "tznn",
"type": "time",
"type": "string",
"typeInfo": {
"frame": "time.Time",
"frame": "string",
"nullable": true
}
},
@@ -158,20 +158,20 @@
1703203200000
],
[
-62167173903766,
"12:34:56.234567",
null
],
[
-62167173902766,
-62167173901766
"12:34:57.234567",
"12:34:58.234567"
],
[
-62167139243235,
"23:12:36.765432+01",
null
],
[
-62167139242235,
-62167139241235
"23:12:37.765432+01",
"23:12:38.765432+01"
],
[
"00:00:00.987654",
@@ -201,22 +201,10 @@
],
null,
null,
[
567000,
0
],
[
567000,
567000
],
[
432000,
0
],
[
432000,
432000
],
null,
null,
null,
null,
null,
null
]
pkg/tsdb/grafana-postgresql-datasource/testdata/table/types_datetime_pgx.golden.jsonc
Vendored
-214
View File
@@ -1,214 +0,0 @@
// 🌟 This was machine generated. Do not edit. 🌟
//
// Frame[0] {
// "typeVersion": [
// 0,
// 0
// ],
// "executedQueryString": "SELECT * FROM tbl"
// }
// Name:
// Dimensions: 12 Fields by 2 Rows
// +--------------------------------------+--------------------------------------+----------------------------------------+----------------------------------------+-------------------------------+-------------------------------+-----------------+-----------------+--------------------+--------------------+-----------------+-----------------+
// | Name: ts | Name: tsnn | Name: tsz | Name: tsznn | Name: d | Name: dnn | Name: t | Name: tnn | Name: tz | Name: tznn | Name: i | Name: inn |
// | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: |
// | Type: []*time.Time | Type: []*time.Time | Type: []*time.Time | Type: []*time.Time | Type: []*time.Time | Type: []*time.Time | Type: []*string | Type: []*string | Type: []*string | Type: []*string | Type: []*string | Type: []*string |
// +--------------------------------------+--------------------------------------+----------------------------------------+----------------------------------------+-------------------------------+-------------------------------+-----------------+-----------------+--------------------+--------------------+-----------------+-----------------+
// | 2023-11-15 05:06:07.123456 +0000 UTC | 2023-11-15 05:06:08.123456 +0000 UTC | 2021-07-22 11:22:33.654321 +0000 +0000 | 2021-07-22 11:22:34.654321 +0000 +0000 | 2023-12-20 00:00:00 +0000 UTC | 2023-12-21 00:00:00 +0000 UTC | 12:34:56.234567 | 12:34:57.234567 | 23:12:36.765432+01 | 23:12:37.765432+01 | 00:00:00.987654 | 00:00:00.887654 |
// | null | 2023-11-15 05:06:09.123456 +0000 UTC | null | 2021-07-22 11:22:35.654321 +0000 +0000 | null | 2023-12-22 00:00:00 +0000 UTC | null | 12:34:58.234567 | null | 23:12:38.765432+01 | null | 00:00:00.787654 |
// +--------------------------------------+--------------------------------------+----------------------------------------+----------------------------------------+-------------------------------+-------------------------------+-----------------+-----------------+--------------------+--------------------+-----------------+-----------------+
//
//
// 🌟 This was machine generated. Do not edit. 🌟
{
"status": 200,
"frames": [
{
"schema": {
"meta": {
"typeVersion": [
0,
0
],
"executedQueryString": "SELECT * FROM tbl"
},
"fields": [
{
"name": "ts",
"type": "time",
"typeInfo": {
"frame": "time.Time",
"nullable": true
}
},
{
"name": "tsnn",
"type": "time",
"typeInfo": {
"frame": "time.Time",
"nullable": true
}
},
{
"name": "tsz",
"type": "time",
"typeInfo": {
"frame": "time.Time",
"nullable": true
}
},
{
"name": "tsznn",
"type": "time",
"typeInfo": {
"frame": "time.Time",
"nullable": true
}
},
{
"name": "d",
"type": "time",
"typeInfo": {
"frame": "time.Time",
"nullable": true
}
},
{
"name": "dnn",
"type": "time",
"typeInfo": {
"frame": "time.Time",
"nullable": true
}
},
{
"name": "t",
"type": "string",
"typeInfo": {
"frame": "string",
"nullable": true
}
},
{
"name": "tnn",
"type": "string",
"typeInfo": {
"frame": "string",
"nullable": true
}
},
{
"name": "tz",
"type": "string",
"typeInfo": {
"frame": "string",
"nullable": true
}
},
{
"name": "tznn",
"type": "string",
"typeInfo": {
"frame": "string",
"nullable": true
}
},
{
"name": "i",
"type": "string",
"typeInfo": {
"frame": "string",
"nullable": true
}
},
{
"name": "inn",
"type": "string",
"typeInfo": {
"frame": "string",
"nullable": true
}
}
]
},
"data": {
"values": [
[
1700024767123,
null
],
[
1700024768123,
1700024769123
],
[
1626952953654,
null
],
[
1626952954654,
1626952955654
],
[
1703030400000,
null
],
[
1703116800000,
1703203200000
],
[
"12:34:56.234567",
null
],
[
"12:34:57.234567",
"12:34:58.234567"
],
[
"23:12:36.765432+01",
null
],
[
"23:12:37.765432+01",
"23:12:38.765432+01"
],
[
"00:00:00.987654",
null
],
[
"00:00:00.887654",
"00:00:00.787654"
]
],
"nanos": [
[
456000,
0
],
[
456000,
456000
],
[
321000,
0
],
[
321000,
321000
],
null,
null,
null,
null,
null,
null,
null,
null
]
}
}
]
}
pkg/tsdb/grafana-postgresql-datasource/testdata/table/types_datetime_pgx.sql
Vendored
-44
View File
@@ -1,44 +0,0 @@
-- SELECT * FROM tbl
-- test all date/time-based postgres data types
CREATE TEMPORARY TABLE tbl (
ts timestamp,
tsnn timestamp NOT NULL,
tsz timestamp with time zone,
tsznn timestamp with time zone NOT NULL,
d date,
dnn date NOT NULL,
t time,
tnn time NOT NULL,
tz time with time zone,
tznn time with time zone NOT NULL,
i interval,
inn interval NOT NULL
);
INSERT INTO tbl (ts, tsnn, tsz, tsznn, d, dnn, t, tnn, tz, tznn, i, inn) VALUES (
'2023-11-15 05:06:07.123456',
'2023-11-15 05:06:08.123456',
'2021-07-22 13:22:33.654321 Europe/Berlin',
'2021-07-22 13:22:34.654321 Europe/Berlin',
'2023-12-20',
'2023-12-21',
'12:34:56.234567',
'12:34:57.234567',
'23:12:36.765432+1',
'23:12:37.765432+1',
'987654 microsecond',
'887654 microsecond'
), (
NULL,
'2023-11-15 05:06:09.123456',
NULL,
'2021-07-22 13:22:35.654321 Europe/Berlin',
NULL,
'2023-12-22',
NULL,
'12:34:58.234567',
NULL,
'23:12:38.765432+1',
NULL,
'787654 microsecond'
);
pkg/tsdb/grafana-postgresql-datasource/tlsmanager.go
+92 -182
View File
@@ -3,48 +3,15 @@ package postgres
import (
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"github.com/grafana/grafana-plugin-sdk-go/backend/log"
"github.com/grafana/grafana/pkg/tsdb/grafana-postgresql-datasource/sqleng"
)
var validateCertFunc = validateCertFilePaths
var writeCertFileFunc = writeCertFile
type certFileType int
const (
rootCert = iota
clientCert
clientKey
)
type tlsSettingsProvider interface {
getTLSSettings(dsInfo sqleng.DataSourceInfo) (tlsSettings, error)
}
type datasourceCacheManager struct {
locker *locker
cache sync.Map
}
type tlsManager struct {
logger log.Logger
dsCacheInstance datasourceCacheManager
dataPath string
}
func newTLSManager(logger log.Logger, dataPath string) tlsSettingsProvider {
return &tlsManager{
logger: logger,
dataPath: dataPath,
dsCacheInstance: datasourceCacheManager{locker: newLocker()},
}
type TLSManager struct {
logger log.Logger
}
type tlsSettings struct {
@@ -55,178 +22,122 @@ type tlsSettings struct {
CertKeyFile string
}
func (m *tlsManager) getTLSSettings(dsInfo sqleng.DataSourceInfo) (tlsSettings, error) {
tlsconfig := tlsSettings{
func newTLSManager(logger log.Logger) *TLSManager {
return &TLSManager{
logger: logger,
}
}
// getTLSSettings retrieves TLS settings and handles certificate file creation if needed.
func (m *TLSManager) getTLSSettings(dsInfo sqleng.DataSourceInfo) (tlsSettings, error) {
tlsConfig := tlsSettings{
Mode: dsInfo.JsonData.Mode,
}
isTLSDisabled := (tlsconfig.Mode == "disable")
if isTLSDisabled {
if tlsConfig.Mode == "disable" {
m.logger.Debug("Postgres TLS/SSL is disabled")
return tlsconfig, nil
return tlsConfig, nil
}
m.logger.Debug("Postgres TLS/SSL is enabled", "tlsMode", tlsconfig.Mode)
tlsConfig.ConfigurationMethod = dsInfo.JsonData.ConfigurationMethod
tlsConfig.RootCertFile = dsInfo.JsonData.RootCertFile
tlsConfig.CertFile = dsInfo.JsonData.CertFile
tlsConfig.CertKeyFile = dsInfo.JsonData.CertKeyFile
tlsconfig.ConfigurationMethod = dsInfo.JsonData.ConfigurationMethod
tlsconfig.RootCertFile = dsInfo.JsonData.RootCertFile
tlsconfig.CertFile = dsInfo.JsonData.CertFile
tlsconfig.CertKeyFile = dsInfo.JsonData.CertKeyFile
if tlsconfig.ConfigurationMethod == "file-content" {
if err := m.writeCertFiles(dsInfo, &tlsconfig); err != nil {
return tlsconfig, err
if tlsConfig.ConfigurationMethod == "file-content" {
if err := m.createCertFiles(dsInfo, &tlsConfig); err != nil {
return tlsConfig, fmt.Errorf("failed to create TLS certificate files: %w", err)
}
} else {
if err := validateCertFunc(tlsconfig.RootCertFile, tlsconfig.CertFile, tlsconfig.CertKeyFile); err != nil {
return tlsconfig, err
if err := validateCertFunc(tlsConfig.RootCertFile, tlsConfig.CertFile, tlsConfig.CertKeyFile); err != nil {
return tlsConfig, fmt.Errorf("invalid TLS certificate file paths: %w", err)
}
}
return tlsconfig, nil
return tlsConfig, nil
}
func (t certFileType) String() string {
switch t {
case rootCert:
return "root certificate"
case clientCert:
return "client certificate"
case clientKey:
return "client key"
default:
panic(fmt.Sprintf("Unrecognized certFileType %d", t))
}
}
// createCertFiles writes certificate files to temporary locations.
func (m *TLSManager) createCertFiles(dsInfo sqleng.DataSourceInfo, tlsConfig *tlsSettings) error {
m.logger.Debug("Writing TLS certificate files to temporary locations")
func getFileName(dataDir string, fileType certFileType) string {
var filename string
switch fileType {
case rootCert:
filename = "root.crt"
case clientCert:
filename = "client.crt"
case clientKey:
filename = "client.key"
default:
panic(fmt.Sprintf("unrecognized certFileType %s", fileType.String()))
}
generatedFilePath := filepath.Join(dataDir, filename)
return generatedFilePath
}
// writeCertFile writes a certificate file.
func writeCertFile(logger log.Logger, fileContent string, generatedFilePath string) error {
fileContent = strings.TrimSpace(fileContent)
if fileContent != "" {
logger.Debug("Writing cert file", "path", generatedFilePath)
if err := os.WriteFile(generatedFilePath, []byte(fileContent), 0600); err != nil {
return err
}
// Make sure the file has the permissions expected by the Postgresql driver, otherwise it will bail
if err := os.Chmod(generatedFilePath, 0600); err != nil {
return err
}
return nil
}
logger.Debug("Deleting cert file since no content is provided", "path", generatedFilePath)
exists, err := fileExists(generatedFilePath)
if err != nil {
var err error
if tlsConfig.RootCertFile, err = m.writeCertFile("root-*.crt", dsInfo.DecryptedSecureJSONData["tlsCACert"]); err != nil {
return err
}
if exists {
if err := os.Remove(generatedFilePath); err != nil {
return fmt.Errorf("failed to remove %q: %w", generatedFilePath, err)
}
if tlsConfig.CertFile, err = m.writeCertFile("client-*.crt", dsInfo.DecryptedSecureJSONData["tlsClientCert"]); err != nil {
return err
}
if tlsConfig.CertKeyFile, err = m.writeCertFile("client-*.key", dsInfo.DecryptedSecureJSONData["tlsClientKey"]); err != nil {
return err
}
return nil
}
func (m *tlsManager) writeCertFiles(dsInfo sqleng.DataSourceInfo, tlsconfig *tlsSettings) error {
m.logger.Debug("Writing TLS certificate files to disk")
tlsRootCert := dsInfo.DecryptedSecureJSONData["tlsCACert"]
tlsClientCert := dsInfo.DecryptedSecureJSONData["tlsClientCert"]
tlsClientKey := dsInfo.DecryptedSecureJSONData["tlsClientKey"]
if tlsRootCert == "" && tlsClientCert == "" && tlsClientKey == "" {
m.logger.Debug("No TLS/SSL certificates provided")
// writeCertFile writes a single certificate file to a temporary location.
func (m *TLSManager) writeCertFile(pattern, content string) (string, error) {
if content == "" {
return "", nil
}
// Calculate all files path
workDir := filepath.Join(m.dataPath, "tls", dsInfo.UID+"generatedTLSCerts")
tlsconfig.RootCertFile = getFileName(workDir, rootCert)
tlsconfig.CertFile = getFileName(workDir, clientCert)
tlsconfig.CertKeyFile = getFileName(workDir, clientKey)
// Find datasource in the cache, if found, skip writing files
cacheKey := strconv.Itoa(int(dsInfo.ID))
m.dsCacheInstance.locker.RLock(cacheKey)
item, ok := m.dsCacheInstance.cache.Load(cacheKey)
m.dsCacheInstance.locker.RUnlock(cacheKey)
if ok {
if !item.(time.Time).Before(dsInfo.Updated) {
return nil
}
}
m.dsCacheInstance.locker.Lock(cacheKey)
defer m.dsCacheInstance.locker.Unlock(cacheKey)
item, ok = m.dsCacheInstance.cache.Load(cacheKey)
if ok {
if !item.(time.Time).Before(dsInfo.Updated) {
return nil
}
}
// Write certification directory and files
exists, err := fileExists(workDir)
m.logger.Debug("Writing certificate file", "pattern", pattern)
file, err := os.CreateTemp("", pattern)
if err != nil {
return err
return "", fmt.Errorf("failed to create temporary file: %w", err)
}
if !exists {
if err := os.MkdirAll(workDir, 0700); err != nil {
return err
defer func() {
if err := file.Close(); err != nil {
m.logger.Error("Failed to close file", "error", err)
}
}()
if _, err := file.WriteString(content); err != nil {
return "", fmt.Errorf("failed to write to temporary file: %w", err)
}
if err = writeCertFileFunc(m.logger, tlsRootCert, tlsconfig.RootCertFile); err != nil {
return err
}
if err = writeCertFileFunc(m.logger, tlsClientCert, tlsconfig.CertFile); err != nil {
return err
}
if err = writeCertFileFunc(m.logger, tlsClientKey, tlsconfig.CertKeyFile); err != nil {
return err
}
// we do not want to point to cert-files that do not exist
if tlsRootCert == "" {
tlsconfig.RootCertFile = ""
}
if tlsClientCert == "" {
tlsconfig.CertFile = ""
}
if tlsClientKey == "" {
tlsconfig.CertKeyFile = ""
}
// Update datasource cache
m.dsCacheInstance.cache.Store(cacheKey, dsInfo.Updated)
return nil
return file.Name(), nil
}
// validateCertFilePaths validates configured certificate file paths.
func validateCertFilePaths(rootCert, clientCert, clientKey string) error {
for _, fpath := range []string{rootCert, clientCert, clientKey} {
if fpath == "" {
// cleanupCertFiles removes temporary certificate files.
func (m *TLSManager) cleanupCertFiles(tlsConfig tlsSettings) {
// Only clean up if the configuration method is "file-content"
if tlsConfig.ConfigurationMethod != "file-content" {
m.logger.Debug("Skipping cleanup of TLS certificate files")
return
}
m.logger.Debug("Cleaning up TLS certificate files")
files := []struct {
path string
name string
}{
{tlsConfig.RootCertFile, "root certificate"},
{tlsConfig.CertFile, "client certificate"},
{tlsConfig.CertKeyFile, "client key"},
}
for _, file := range files {
if file.path == "" {
continue
}
exists, err := fileExists(fpath)
if err := os.Remove(file.path); err != nil {
m.logger.Error("Failed to remove file", "type", file.name, "path", file.path, "error", err)
} else {
m.logger.Debug("Successfully removed file", "type", file.name, "path", file.path)
}
}
}
// validateCertFilePaths validates the existence of configured certificate file paths.
func validateCertFilePaths(rootCert, clientCert, clientKey string) error {
for _, path := range []string{rootCert, clientCert, clientKey} {
if path == "" {
continue
}
exists, err := fileExists(path)
if err != nil {
return err
return fmt.Errorf("error checking file existence: %w", err)
}
if !exists {
return sqleng.ErrCertFileNotExist
@@ -235,15 +146,14 @@ func validateCertFilePaths(rootCert, clientCert, clientKey string) error {
return nil
}
// Exists determines whether a file/directory exists or not.
func fileExists(fpath string) (bool, error) {
_, err := os.Stat(fpath)
// fileExists checks if a file exists at the given path.
func fileExists(path string) (bool, error) {
_, err := os.Stat(path)
if err != nil {
if !os.IsNotExist(err) {
return false, err
if os.IsNotExist(err) {
return false, nil
}
return false, nil
return false, err
}
return true, nil
}
pkg/tsdb/grafana-postgresql-datasource/tlsmanager_test.go
+141 -228
View File
@@ -2,176 +2,21 @@ package postgres
import (
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"testing"
"time"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/backend/log"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb/grafana-postgresql-datasource/sqleng"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
_ "github.com/lib/pq"
)
var writeCertFileCallNum int
// TestDataSourceCacheManager is to test the Cache manager
func TestDataSourceCacheManager(t *testing.T) {
cfg := setting.NewCfg()
cfg.DataPath = t.TempDir()
mng := tlsManager{
logger: backend.NewLoggerWith("logger", "tsdb.postgres"),
dsCacheInstance: datasourceCacheManager{locker: newLocker()},
dataPath: cfg.DataPath,
}
jsonData := sqleng.JsonData{
Mode: "verify-full",
ConfigurationMethod: "file-content",
}
secureJSONData := map[string]string{
"tlsClientCert": "I am client certification",
"tlsClientKey": "I am client key",
"tlsCACert": "I am CA certification",
}
updateTime := time.Now().Add(-5 * time.Minute)
mockValidateCertFilePaths()
t.Cleanup(resetValidateCertFilePaths)
t.Run("Check datasource cache creation", func(t *testing.T) {
var wg sync.WaitGroup
wg.Add(10)
for id := int64(1); id <= 10; id++ {
go func(id int64) {
ds := sqleng.DataSourceInfo{
ID: id,
Updated: updateTime,
Database: "database",
JsonData: jsonData,
DecryptedSecureJSONData: secureJSONData,
UID: "testData",
}
s := tlsSettings{}
err := mng.writeCertFiles(ds, &s)
require.NoError(t, err)
wg.Done()
}(id)
}
wg.Wait()
t.Run("check cache creation is succeed", func(t *testing.T) {
for id := int64(1); id <= 10; id++ {
updated, ok := mng.dsCacheInstance.cache.Load(strconv.Itoa(int(id)))
require.True(t, ok)
require.Equal(t, updateTime, updated)
}
})
})
t.Run("Check datasource cache modification", func(t *testing.T) {
t.Run("check when version not changed, cache and files are not updated", func(t *testing.T) {
mockWriteCertFile()
t.Cleanup(resetWriteCertFile)
var wg1 sync.WaitGroup
wg1.Add(5)
for id := int64(1); id <= 5; id++ {
go func(id int64) {
ds := sqleng.DataSourceInfo{
ID: 1,
Updated: updateTime,
Database: "database",
JsonData: jsonData,
DecryptedSecureJSONData: secureJSONData,
UID: "testData",
}
s := tlsSettings{}
err := mng.writeCertFiles(ds, &s)
require.NoError(t, err)
wg1.Done()
}(id)
}
wg1.Wait()
assert.Equal(t, writeCertFileCallNum, 0)
})
t.Run("cache is updated with the last datasource version", func(t *testing.T) {
dsV2 := sqleng.DataSourceInfo{
ID: 1,
Updated: updateTime.Add(time.Minute),
Database: "database",
JsonData: jsonData,
DecryptedSecureJSONData: secureJSONData,
UID: "testData",
}
dsV3 := sqleng.DataSourceInfo{
ID: 1,
Updated: updateTime.Add(2 * time.Minute),
Database: "database",
JsonData: jsonData,
DecryptedSecureJSONData: secureJSONData,
UID: "testData",
}
s := tlsSettings{}
err := mng.writeCertFiles(dsV2, &s)
require.NoError(t, err)
err = mng.writeCertFiles(dsV3, &s)
require.NoError(t, err)
version, ok := mng.dsCacheInstance.cache.Load("1")
require.True(t, ok)
require.Equal(t, updateTime.Add(2*time.Minute), version)
})
})
}
// Test getFileName
func TestGetFileName(t *testing.T) {
testCases := []struct {
desc string
datadir string
fileType certFileType
expErr string
expectedGeneratedPath string
}{
{
desc: "Get File Name for root certification",
datadir: ".",
fileType: rootCert,
expectedGeneratedPath: "root.crt",
},
{
desc: "Get File Name for client certification",
datadir: ".",
fileType: clientCert,
expectedGeneratedPath: "client.crt",
},
{
desc: "Get File Name for client certification",
datadir: ".",
fileType: clientKey,
expectedGeneratedPath: "client.key",
},
}
for _, tt := range testCases {
t.Run(tt.desc, func(t *testing.T) {
generatedPath := getFileName(tt.datadir, tt.fileType)
assert.Equal(t, tt.expectedGeneratedPath, generatedPath)
})
}
}
// Test getTLSSettings.
func TestGetTLSSettings(t *testing.T) {
cfg := setting.NewCfg()
cfg.DataPath = t.TempDir()
mockValidateCertFilePaths()
t.Cleanup(resetValidateCertFilePaths)
@@ -216,75 +61,13 @@ func TestGetTLSSettings(t *testing.T) {
CertKeyFile: "i/am/coding/client.key",
},
},
{
desc: "Custom TLS mode verify-full with certificate files content",
updated: updatedTime.Add(2 * time.Minute),
uid: "xxx",
jsonData: sqleng.JsonData{
Mode: "verify-full",
ConfigurationMethod: "file-content",
},
secureJSONData: map[string]string{
"tlsCACert": "I am CA certification",
"tlsClientCert": "I am client certification",
"tlsClientKey": "I am client key",
},
tlsSettings: tlsSettings{
Mode: "verify-full",
ConfigurationMethod: "file-content",
RootCertFile: filepath.Join(cfg.DataPath, "tls", "xxxgeneratedTLSCerts", "root.crt"),
CertFile: filepath.Join(cfg.DataPath, "tls", "xxxgeneratedTLSCerts", "client.crt"),
CertKeyFile: filepath.Join(cfg.DataPath, "tls", "xxxgeneratedTLSCerts", "client.key"),
},
},
{
desc: "Custom TLS mode verify-ca with no client certificates with certificate files content",
updated: updatedTime.Add(3 * time.Minute),
uid: "xxx",
jsonData: sqleng.JsonData{
Mode: "verify-ca",
ConfigurationMethod: "file-content",
},
secureJSONData: map[string]string{
"tlsCACert": "I am CA certification",
},
tlsSettings: tlsSettings{
Mode: "verify-ca",
ConfigurationMethod: "file-content",
RootCertFile: filepath.Join(cfg.DataPath, "tls", "xxxgeneratedTLSCerts", "root.crt"),
CertFile: "",
CertKeyFile: "",
},
},
{
desc: "Custom TLS mode require with client certificates and no root certificate with certificate files content",
updated: updatedTime.Add(4 * time.Minute),
uid: "xxx",
jsonData: sqleng.JsonData{
Mode: "require",
ConfigurationMethod: "file-content",
},
secureJSONData: map[string]string{
"tlsClientCert": "I am client certification",
"tlsClientKey": "I am client key",
},
tlsSettings: tlsSettings{
Mode: "require",
ConfigurationMethod: "file-content",
RootCertFile: "",
CertFile: filepath.Join(cfg.DataPath, "tls", "xxxgeneratedTLSCerts", "client.crt"),
CertKeyFile: filepath.Join(cfg.DataPath, "tls", "xxxgeneratedTLSCerts", "client.key"),
},
},
}
for _, tt := range testCases {
t.Run(tt.desc, func(t *testing.T) {
var settings tlsSettings
var err error
mng := tlsManager{
logger: backend.NewLoggerWith("logger", "tsdb.postgres"),
dsCacheInstance: datasourceCacheManager{locker: newLocker()},
dataPath: cfg.DataPath,
mng := TLSManager{
logger: backend.NewLoggerWith("logger", "tsdb.postgres"),
}
ds := sqleng.DataSourceInfo{
@@ -318,15 +101,145 @@ func resetValidateCertFilePaths() {
validateCertFunc = validateCertFilePaths
}
func mockWriteCertFile() {
writeCertFileCallNum = 0
writeCertFileFunc = func(logger log.Logger, fileContent string, generatedFilePath string) error {
writeCertFileCallNum++
return nil
func TestTLSManager_GetTLSSettings(t *testing.T) {
logger := log.New()
tlsManager := newTLSManager(logger)
dsInfo := sqleng.DataSourceInfo{
JsonData: sqleng.JsonData{
Mode: "require",
ConfigurationMethod: "file-content",
},
DecryptedSecureJSONData: map[string]string{
"tlsCACert": "root-cert-content",
"tlsClientCert": "client-cert-content",
"tlsClientKey": "client-key-content",
},
}
tlsConfig, err := tlsManager.getTLSSettings(dsInfo)
require.NoError(t, err)
assert.Equal(t, "require", tlsConfig.Mode)
assert.NotEmpty(t, tlsConfig.RootCertFile)
assert.NotEmpty(t, tlsConfig.CertFile)
assert.NotEmpty(t, tlsConfig.CertKeyFile)
// Cleanup temporary files
tlsManager.cleanupCertFiles(tlsConfig)
assert.NoFileExists(t, tlsConfig.RootCertFile)
assert.NoFileExists(t, tlsConfig.CertFile)
assert.NoFileExists(t, tlsConfig.CertKeyFile)
}
func resetWriteCertFile() {
writeCertFileCallNum = 0
writeCertFileFunc = writeCertFile
func TestTLSManager_CleanupCertFiles_FilePath(t *testing.T) {
logger := log.New()
tlsManager := newTLSManager(logger)
// Create temporary files for testing
rootCertFile, err := tlsManager.writeCertFile("root-*.crt", "root-cert-content")
require.NoError(t, err)
clientCertFile, err := tlsManager.writeCertFile("client-*.crt", "client-cert-content")
require.NoError(t, err)
clientKeyFile, err := tlsManager.writeCertFile("client-*.key", "client-key-content")
require.NoError(t, err)
// Simulate a configuration where the method is "file-path"
tlsConfig := tlsSettings{
ConfigurationMethod: "file-path",
RootCertFile: rootCertFile,
CertFile: clientCertFile,
CertKeyFile: clientKeyFile,
}
// Call cleanupCertFiles
tlsManager.cleanupCertFiles(tlsConfig)
// Verify the files are NOT deleted
assert.FileExists(t, rootCertFile, "Root certificate file should not be deleted")
assert.FileExists(t, clientCertFile, "Client certificate file should not be deleted")
assert.FileExists(t, clientKeyFile, "Client key file should not be deleted")
// Cleanup the files manually
err = os.Remove(rootCertFile)
require.NoError(t, err)
err = os.Remove(clientCertFile)
require.NoError(t, err)
err = os.Remove(clientKeyFile)
require.NoError(t, err)
}
func TestTLSManager_CreateCertFiles(t *testing.T) {
logger := log.New()
tlsManager := newTLSManager(logger)
dsInfo := sqleng.DataSourceInfo{
DecryptedSecureJSONData: map[string]string{
"tlsCACert": "root-cert-content",
"tlsClientCert": "client-cert-content",
"tlsClientKey": "client-key-content",
},
}
tlsConfig := tlsSettings{
ConfigurationMethod: "file-content",
}
err := tlsManager.createCertFiles(dsInfo, &tlsConfig)
require.NoError(t, err)
assert.FileExists(t, tlsConfig.RootCertFile)
assert.FileExists(t, tlsConfig.CertFile)
assert.FileExists(t, tlsConfig.CertKeyFile)
// Cleanup temporary files
tlsManager.cleanupCertFiles(tlsConfig)
assert.NoFileExists(t, tlsConfig.RootCertFile)
assert.NoFileExists(t, tlsConfig.CertFile)
assert.NoFileExists(t, tlsConfig.CertKeyFile)
}
func TestTLSManager_WriteCertFile(t *testing.T) {
logger := log.New()
tlsManager := newTLSManager(logger)
// Test writing a valid certificate file
filePath, err := tlsManager.writeCertFile("test-*.crt", "test-cert-content")
require.NoError(t, err)
assert.FileExists(t, filePath)
content, err := os.ReadFile(filepath.Clean(filePath))
require.NoError(t, err)
assert.Equal(t, "test-cert-content", string(content))
// Cleanup the file
err = os.Remove(filePath)
require.NoError(t, err)
assert.NoFileExists(t, filePath)
}
func TestTLSManager_CleanupCertFiles(t *testing.T) {
logger := log.New()
tlsManager := newTLSManager(logger)
// Create temporary files for testing
rootCertFile, err := tlsManager.writeCertFile("root-*.crt", "root-cert-content")
require.NoError(t, err)
clientCertFile, err := tlsManager.writeCertFile("client-*.crt", "client-cert-content")
require.NoError(t, err)
clientKeyFile, err := tlsManager.writeCertFile("client-*.key", "client-key-content")
require.NoError(t, err)
tlsConfig := tlsSettings{
ConfigurationMethod: "file-content",
RootCertFile: rootCertFile,
CertFile: clientCertFile,
CertKeyFile: clientKeyFile,
}
// Cleanup the files
tlsManager.cleanupCertFiles(tlsConfig)
// Verify the files are deleted
assert.NoFileExists(t, rootCertFile)
assert.NoFileExists(t, clientCertFile)
assert.NoFileExists(t, clientKeyFile)
}
public/app/plugins/datasource/grafana-postgresql-datasource/configuration/ConfigurationEditor.tsx
+18 -22
View File
@@ -11,7 +11,6 @@ import {
import { ConfigSection, ConfigSubSection, DataSourceDescription, EditorStack } from '@grafana/plugin-ui';
import { config } from '@grafana/runtime';
import {
ConnectionLimits,
Divider,
MaxLifetimeField,
MaxOpenConnectionsField,
@@ -19,16 +18,16 @@ import {
useMigrateDatabaseFields,
} from '@grafana/sql';
import {
Input,
Select,
SecretInput,
Field,
Tooltip,
Label,
Icon,
Switch,
SecureSocksProxySettings,
Collapse,
Field,
Icon,
Input,
Label,
SecretInput,
SecureSocksProxySettings,
Select,
Switch,
Tooltip,
} from '@grafana/ui';
import { PostgresOptions, PostgresTLSMethods, PostgresTLSModes, SecureJsonData } from '../types';
@@ -412,18 +411,15 @@ export const PostgresConfigEditor = (props: DataSourcePluginOptionsEditorProps<P
</Field>
</ConfigSubSection>
{config.featureToggles.postgresDSUsePGX ? (
<ConfigSubSection title="Connection limits">
<MaxOpenConnectionsField
labelWidth={WIDTH_LONG}
jsonData={jsonData}
onMaxConnectionsChanged={onMaxConnectionsChanged}
/>
<MaxLifetimeField labelWidth={WIDTH_LONG} jsonData={jsonData} onMaxLifetimeChanged={onMaxLifetimeChanged} />
</ConfigSubSection>
) : (
<ConnectionLimits options={options} onOptionsChange={onOptionsChange} />
)}
<ConfigSubSection title="Connection limits">
<MaxOpenConnectionsField
labelWidth={WIDTH_LONG}
jsonData={jsonData}
onMaxConnectionsChanged={onMaxConnectionsChanged}
/>
<MaxLifetimeField labelWidth={WIDTH_LONG} jsonData={jsonData} onMaxLifetimeChanged={onMaxLifetimeChanged} />
</ConfigSubSection>
{config.secureSocksDSProxyEnabled && (
<SecureSocksProxySettings options={options} onOptionsChange={onOptionsChange} />
)}