Encryption: Refactor securejsondata.SecureJsonData to stop relying on global functions (#38865)

* Encryption: Add support to encrypt/decrypt sjd

* Add datasources.Service as a proxy to datasources db operations

* Encrypt ds.SecureJsonData before calling SQLStore

* Move ds cache code into ds service

* Fix tlsmanager tests

* Fix pluginproxy tests

* Remove some securejsondata.GetEncryptedJsonData usages

* Add pluginsettings.Service as a proxy for plugin settings db operations

* Add AlertNotificationService as a proxy for alert notification db operations

* Remove some securejsondata.GetEncryptedJsonData usages

* Remove more securejsondata.GetEncryptedJsonData usages

* Fix lint errors

* Minor fixes

* Remove encryption global functions usages from ngalert

* Fix lint errors

* Minor fixes

* Minor fixes

* Remove securejsondata.DecryptedValue usage

* Refactor the refactor

* Remove securejsondata.DecryptedValue usage

* Move securejsondata to migrations package

* Move securejsondata to migrations package

* Minor fix

* Fix integration test

* Fix integration tests

* Undo undesired changes

* Fix tests

* Add context.Context into encryption methods

* Fix tests

* Fix tests

* Fix tests

* Trigger CI

* Fix test

* Add names to params of encryption service interface

* Remove bus from CacheServiceImpl

* Add logging

* Add keys to logger

Co-authored-by: Emil Tullstedt <emil.tullstedt@grafana.com>

* Add missing key to logger

Co-authored-by: Emil Tullstedt <emil.tullstedt@grafana.com>

* Undo changes in markdown files

* Fix formatting

* Add context to secrets service

* Rename decryptSecureJsonData to decryptSecureJsonDataFn

* Name args in GetDecryptedValueFn

* Add template back to NewAlertmanagerNotifier

* Copy GetDecryptedValueFn to ngalert

* Add logging to pluginsettings

* Fix pluginsettings test

Co-authored-by: Tania B <yalyna.ts@gmail.com>
Co-authored-by: Emil Tullstedt <emil.tullstedt@grafana.com>
This commit is contained in:
Joan López de la Franca Beltran
2021-10-07 17:33:50 +03:00
committed by GitHub
co-authored by Emil Tullstedt Tania B
parent da813877fb
commit 722c414fef
141 changed files with 1968 additions and 1197 deletions
+9 -2
View File
@@ -43,11 +43,15 @@ func (dc *CacheServiceImpl) GetDatasource(
}
plog.Debug("Querying for data source via SQL store", "id", datasourceID, "orgId", user.OrgId)
ds, err := dc.SQLStore.GetDataSource("", datasourceID, "", user.OrgId)
query := &models.GetDataSourceQuery{Id: datasourceID, OrgId: user.OrgId}
err := dc.SQLStore.GetDataSource(query)
if err != nil {
return nil, err
}
ds := query.Result
if ds.Uid != "" {
dc.CacheService.Set(uidKey(ds.OrgId, ds.Uid), ds, time.Second*5)
}
@@ -78,11 +82,14 @@ func (dc *CacheServiceImpl) GetDatasourceByUID(
}
plog.Debug("Querying for data source via SQL store", "uid", datasourceUID, "orgId", user.OrgId)
ds, err := dc.SQLStore.GetDataSource(datasourceUID, 0, "", user.OrgId)
query := &models.GetDataSourceQuery{Uid: datasourceUID, OrgId: user.OrgId}
err := dc.SQLStore.GetDataSource(query)
if err != nil {
return nil, err
}
ds := query.Result
dc.CacheService.Set(uidCacheKey, ds, time.Second*5)
dc.CacheService.Set(idKey(ds.Id), ds, time.Second*5)
return ds, nil
+373
View File
@@ -0,0 +1,373 @@
package datasources
import (
"context"
"crypto/tls"
"fmt"
"net/http"
"strconv"
"sync"
"time"
sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/encryption"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb/azuremonitor/azcredentials"
)
type Service struct {
Bus bus.Bus
SQLStore *sqlstore.SQLStore
EncryptionService encryption.Service
ptc proxyTransportCache
dsDecryptionCache secureJSONDecryptionCache
}
type proxyTransportCache struct {
cache map[int64]cachedRoundTripper
sync.Mutex
}
type cachedRoundTripper struct {
updated time.Time
roundTripper http.RoundTripper
}
type secureJSONDecryptionCache struct {
cache map[int64]cachedDecryptedJSON
sync.Mutex
}
type cachedDecryptedJSON struct {
updated time.Time
json map[string]string
}
func ProvideService(bus bus.Bus, store *sqlstore.SQLStore, encryptionService encryption.Service) *Service {
s := &Service{
Bus: bus,
SQLStore: store,
EncryptionService: encryptionService,
ptc: proxyTransportCache{
cache: make(map[int64]cachedRoundTripper),
},
dsDecryptionCache: secureJSONDecryptionCache{
cache: make(map[int64]cachedDecryptedJSON),
},
}
s.Bus.AddHandler(s.GetDataSources)
s.Bus.AddHandler(s.GetDataSourcesByType)
s.Bus.AddHandler(s.GetDataSource)
s.Bus.AddHandlerCtx(s.AddDataSource)
s.Bus.AddHandler(s.DeleteDataSource)
s.Bus.AddHandlerCtx(s.UpdateDataSource)
s.Bus.AddHandler(s.GetDefaultDataSource)
return s
}
func (s *Service) GetDataSource(query *models.GetDataSourceQuery) error {
return s.SQLStore.GetDataSource(query)
}
func (s *Service) GetDataSources(query *models.GetDataSourcesQuery) error {
return s.SQLStore.GetDataSources(query)
}
func (s *Service) GetDataSourcesByType(query *models.GetDataSourcesByTypeQuery) error {
return s.SQLStore.GetDataSourcesByType(query)
}
func (s *Service) AddDataSource(ctx context.Context, cmd *models.AddDataSourceCommand) error {
var err error
cmd.EncryptedSecureJsonData, err = s.EncryptionService.EncryptJsonData(ctx, cmd.SecureJsonData, setting.SecretKey)
if err != nil {
return err
}
return s.SQLStore.AddDataSource(cmd)
}
func (s *Service) DeleteDataSource(cmd *models.DeleteDataSourceCommand) error {
return s.SQLStore.DeleteDataSource(cmd)
}
func (s *Service) UpdateDataSource(ctx context.Context, cmd *models.UpdateDataSourceCommand) error {
var err error
cmd.EncryptedSecureJsonData, err = s.EncryptionService.EncryptJsonData(ctx, cmd.SecureJsonData, setting.SecretKey)
if err != nil {
return err
}
return s.SQLStore.UpdateDataSource(cmd)
}
func (s *Service) GetDefaultDataSource(query *models.GetDefaultDataSourceQuery) error {
return s.SQLStore.GetDefaultDataSource(query)
}
func (s *Service) GetHTTPClient(ds *models.DataSource, provider httpclient.Provider) (*http.Client, error) {
transport, err := s.GetHTTPTransport(ds, provider)
if err != nil {
return nil, err
}
return &http.Client{
Timeout: s.getTimeout(ds),
Transport: transport,
}, nil
}
func (s *Service) GetHTTPTransport(ds *models.DataSource, provider httpclient.Provider,
customMiddlewares ...sdkhttpclient.Middleware) (http.RoundTripper, error) {
s.ptc.Lock()
defer s.ptc.Unlock()
if t, present := s.ptc.cache[ds.Id]; present && ds.Updated.Equal(t.updated) {
return t.roundTripper, nil
}
opts, err := s.httpClientOptions(ds)
if err != nil {
return nil, err
}
opts.Middlewares = customMiddlewares
rt, err := provider.GetTransport(*opts)
if err != nil {
return nil, err
}
s.ptc.cache[ds.Id] = cachedRoundTripper{
roundTripper: rt,
updated: ds.Updated,
}
return rt, nil
}
func (s *Service) GetTLSConfig(ds *models.DataSource, httpClientProvider httpclient.Provider) (*tls.Config, error) {
opts, err := s.httpClientOptions(ds)
if err != nil {
return nil, err
}
return httpClientProvider.GetTLSConfig(*opts)
}
func (s *Service) DecryptedValues(ds *models.DataSource) map[string]string {
s.dsDecryptionCache.Lock()
defer s.dsDecryptionCache.Unlock()
if item, present := s.dsDecryptionCache.cache[ds.Id]; present && ds.Updated.Equal(item.updated) {
return item.json
}
json, err := s.EncryptionService.DecryptJsonData(context.Background(), ds.SecureJsonData, setting.SecretKey)
if err != nil {
return map[string]string{}
}
s.dsDecryptionCache.cache[ds.Id] = cachedDecryptedJSON{
updated: ds.Updated,
json: json,
}
return json
}
func (s *Service) DecryptedValue(ds *models.DataSource, key string) (string, bool) {
value, exists := s.DecryptedValues(ds)[key]
return value, exists
}
func (s *Service) DecryptedBasicAuthPassword(ds *models.DataSource) string {
if value, ok := s.DecryptedValue(ds, "basicAuthPassword"); ok {
return value
}
return ds.BasicAuthPassword
}
func (s *Service) DecryptedPassword(ds *models.DataSource) string {
if value, ok := s.DecryptedValue(ds, "password"); ok {
return value
}
return ds.Password
}
func (s *Service) httpClientOptions(ds *models.DataSource) (*sdkhttpclient.Options, error) {
tlsOptions := s.dsTLSOptions(ds)
timeouts := &sdkhttpclient.TimeoutOptions{
Timeout: s.getTimeout(ds),
DialTimeout: sdkhttpclient.DefaultTimeoutOptions.DialTimeout,
KeepAlive: sdkhttpclient.DefaultTimeoutOptions.KeepAlive,
TLSHandshakeTimeout: sdkhttpclient.DefaultTimeoutOptions.TLSHandshakeTimeout,
ExpectContinueTimeout: sdkhttpclient.DefaultTimeoutOptions.ExpectContinueTimeout,
MaxConnsPerHost: sdkhttpclient.DefaultTimeoutOptions.MaxConnsPerHost,
MaxIdleConns: sdkhttpclient.DefaultTimeoutOptions.MaxIdleConns,
MaxIdleConnsPerHost: sdkhttpclient.DefaultTimeoutOptions.MaxIdleConnsPerHost,
IdleConnTimeout: sdkhttpclient.DefaultTimeoutOptions.IdleConnTimeout,
}
opts := &sdkhttpclient.Options{
Timeouts: timeouts,
Headers: s.getCustomHeaders(ds.JsonData, s.DecryptedValues(ds)),
Labels: map[string]string{
"datasource_name": ds.Name,
"datasource_uid": ds.Uid,
},
TLS: &tlsOptions,
}
if ds.JsonData != nil {
opts.CustomOptions = ds.JsonData.MustMap()
}
if ds.BasicAuth {
opts.BasicAuth = &sdkhttpclient.BasicAuthOptions{
User: ds.BasicAuthUser,
Password: s.DecryptedBasicAuthPassword(ds),
}
} else if ds.User != "" {
opts.BasicAuth = &sdkhttpclient.BasicAuthOptions{
User: ds.User,
Password: s.DecryptedPassword(ds),
}
}
if ds.JsonData != nil && ds.JsonData.Get("azureAuth").MustBool() {
credentials, err := azcredentials.FromDatasourceData(ds.JsonData.MustMap(), s.DecryptedValues(ds))
if err != nil {
err = fmt.Errorf("invalid Azure credentials: %s", err)
return nil, err
}
opts.CustomOptions["_azureAuth"] = true
if credentials != nil {
opts.CustomOptions["_azureCredentials"] = credentials
}
}
if ds.JsonData != nil && ds.JsonData.Get("sigV4Auth").MustBool(false) && setting.SigV4AuthEnabled {
opts.SigV4 = &sdkhttpclient.SigV4Config{
Service: awsServiceNamespace(ds.Type),
Region: ds.JsonData.Get("sigV4Region").MustString(),
AssumeRoleARN: ds.JsonData.Get("sigV4AssumeRoleArn").MustString(),
AuthType: ds.JsonData.Get("sigV4AuthType").MustString(),
ExternalID: ds.JsonData.Get("sigV4ExternalId").MustString(),
Profile: ds.JsonData.Get("sigV4Profile").MustString(),
}
if val, exists := s.DecryptedValue(ds, "sigV4AccessKey"); exists {
opts.SigV4.AccessKey = val
}
if val, exists := s.DecryptedValue(ds, "sigV4SecretKey"); exists {
opts.SigV4.SecretKey = val
}
}
return opts, nil
}
func (s *Service) dsTLSOptions(ds *models.DataSource) sdkhttpclient.TLSOptions {
var tlsSkipVerify, tlsClientAuth, tlsAuthWithCACert bool
var serverName string
if ds.JsonData != nil {
tlsClientAuth = ds.JsonData.Get("tlsAuth").MustBool(false)
tlsAuthWithCACert = ds.JsonData.Get("tlsAuthWithCACert").MustBool(false)
tlsSkipVerify = ds.JsonData.Get("tlsSkipVerify").MustBool(false)
serverName = ds.JsonData.Get("serverName").MustString()
}
opts := sdkhttpclient.TLSOptions{
InsecureSkipVerify: tlsSkipVerify,
ServerName: serverName,
}
if tlsClientAuth || tlsAuthWithCACert {
if tlsAuthWithCACert {
if val, exists := s.DecryptedValue(ds, "tlsCACert"); exists && len(val) > 0 {
opts.CACertificate = val
}
}
if tlsClientAuth {
if val, exists := s.DecryptedValue(ds, "tlsClientCert"); exists && len(val) > 0 {
opts.ClientCertificate = val
}
if val, exists := s.DecryptedValue(ds, "tlsClientKey"); exists && len(val) > 0 {
opts.ClientKey = val
}
}
}
return opts
}
func (s *Service) getTimeout(ds *models.DataSource) time.Duration {
timeout := 0
if ds.JsonData != nil {
timeout = ds.JsonData.Get("timeout").MustInt()
if timeout <= 0 {
if timeoutStr := ds.JsonData.Get("timeout").MustString(); timeoutStr != "" {
if t, err := strconv.Atoi(timeoutStr); err == nil {
timeout = t
}
}
}
}
if timeout <= 0 {
return sdkhttpclient.DefaultTimeoutOptions.Timeout
}
return time.Duration(timeout) * time.Second
}
// getCustomHeaders returns a map with all the to be set headers
// The map key represents the HeaderName and the value represents this header's value
func (s *Service) getCustomHeaders(jsonData *simplejson.Json, decryptedValues map[string]string) map[string]string {
headers := make(map[string]string)
if jsonData == nil {
return headers
}
index := 1
for {
headerNameSuffix := fmt.Sprintf("httpHeaderName%d", index)
headerValueSuffix := fmt.Sprintf("httpHeaderValue%d", index)
key := jsonData.Get(headerNameSuffix).MustString()
if key == "" {
// No (more) header values are available
break
}
if val, ok := decryptedValues[headerValueSuffix]; ok {
headers[key] = val
}
index++
}
return headers
}
func awsServiceNamespace(dsType string) string {
switch dsType {
case models.DS_ES, models.DS_ES_OPEN_DISTRO:
return "es"
case models.DS_PROMETHEUS:
return "aps"
default:
panic(fmt.Sprintf("Unsupported datasource %q", dsType))
}
}
+675
View File
@@ -0,0 +1,675 @@
package datasources
import (
"context"
"io/ioutil"
"net/http"
"net/http/httptest"
"testing"
"time"
sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/encryption/ossencryption"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb/azuremonitor/azcredentials"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestService(t *testing.T) {
sqlStore := sqlstore.InitTestDB(t)
s := ProvideService(bus.New(), sqlStore, ossencryption.ProvideService())
origSecret := setting.SecretKey
setting.SecretKey = "datasources_service_test"
t.Cleanup(func() {
setting.SecretKey = origSecret
})
var ds *models.DataSource
t.Run("create datasource should encrypt the secure json data", func(t *testing.T) {
ctx := context.Background()
sjd := map[string]string{"password": "12345"}
cmd := models.AddDataSourceCommand{SecureJsonData: sjd}
err := s.AddDataSource(ctx, &cmd)
require.NoError(t, err)
ds = cmd.Result
decrypted, err := s.EncryptionService.DecryptJsonData(ctx, ds.SecureJsonData, setting.SecretKey)
require.NoError(t, err)
require.Equal(t, sjd, decrypted)
})
t.Run("update datasource should encrypt the secure json data", func(t *testing.T) {
ctx := context.Background()
sjd := map[string]string{"password": "678910"}
cmd := models.UpdateDataSourceCommand{Id: ds.Id, OrgId: ds.OrgId, SecureJsonData: sjd}
err := s.UpdateDataSource(ctx, &cmd)
require.NoError(t, err)
decrypted, err := s.EncryptionService.DecryptJsonData(ctx, cmd.Result.SecureJsonData, setting.SecretKey)
require.NoError(t, err)
require.Equal(t, sjd, decrypted)
})
}
//nolint:goconst
func TestService_GetHttpTransport(t *testing.T) {
t.Run("Should use cached proxy", func(t *testing.T) {
var configuredTransport *http.Transport
provider := httpclient.NewProvider(sdkhttpclient.ProviderOptions{
ConfigureTransport: func(opts sdkhttpclient.Options, transport *http.Transport) {
configuredTransport = transport
},
})
ds := models.DataSource{
Id: 1,
Url: "http://k8s:8001",
Type: "Kubernetes",
}
dsService := ProvideService(bus.New(), nil, ossencryption.ProvideService())
rt1, err := dsService.GetHTTPTransport(&ds, provider)
require.NoError(t, err)
require.NotNil(t, rt1)
tr1 := configuredTransport
rt2, err := dsService.GetHTTPTransport(&ds, provider)
require.NoError(t, err)
require.NotNil(t, rt2)
tr2 := configuredTransport
require.Same(t, tr1, tr2)
require.False(t, tr1.TLSClientConfig.InsecureSkipVerify)
require.Empty(t, tr1.TLSClientConfig.Certificates)
require.Nil(t, tr1.TLSClientConfig.RootCAs)
})
t.Run("Should not use cached proxy when datasource updated", func(t *testing.T) {
var configuredTransport *http.Transport
provider := httpclient.NewProvider(sdkhttpclient.ProviderOptions{
ConfigureTransport: func(opts sdkhttpclient.Options, transport *http.Transport) {
configuredTransport = transport
},
})
setting.SecretKey = "password"
json := simplejson.New()
json.Set("tlsAuthWithCACert", true)
encryptionService := ossencryption.ProvideService()
dsService := ProvideService(bus.New(), nil, encryptionService)
tlsCaCert, err := encryptionService.Encrypt(context.Background(), []byte(caCert), "password")
require.NoError(t, err)
ds := models.DataSource{
Id: 1,
Url: "http://k8s:8001",
Type: "Kubernetes",
SecureJsonData: map[string][]byte{"tlsCACert": tlsCaCert},
Updated: time.Now().Add(-2 * time.Minute),
}
rt1, err := dsService.GetHTTPTransport(&ds, provider)
require.NotNil(t, rt1)
require.NoError(t, err)
tr1 := configuredTransport
require.False(t, tr1.TLSClientConfig.InsecureSkipVerify)
require.Empty(t, tr1.TLSClientConfig.Certificates)
require.Nil(t, tr1.TLSClientConfig.RootCAs)
ds.JsonData = nil
ds.SecureJsonData = map[string][]byte{}
ds.Updated = time.Now()
rt2, err := dsService.GetHTTPTransport(&ds, provider)
require.NoError(t, err)
require.NotNil(t, rt2)
tr2 := configuredTransport
require.NotSame(t, tr1, tr2)
require.Nil(t, tr2.TLSClientConfig.RootCAs)
})
t.Run("Should set TLS client authentication enabled if configured in JsonData", func(t *testing.T) {
var configuredTransport *http.Transport
provider := httpclient.NewProvider(sdkhttpclient.ProviderOptions{
ConfigureTransport: func(opts sdkhttpclient.Options, transport *http.Transport) {
configuredTransport = transport
},
})
setting.SecretKey = "password"
json := simplejson.New()
json.Set("tlsAuth", true)
encryptionService := ossencryption.ProvideService()
dsService := ProvideService(bus.New(), nil, encryptionService)
tlsClientCert, err := encryptionService.Encrypt(context.Background(), []byte(clientCert), "password")
require.NoError(t, err)
tlsClientKey, err := encryptionService.Encrypt(context.Background(), []byte(clientKey), "password")
require.NoError(t, err)
ds := models.DataSource{
Id: 1,
Url: "http://k8s:8001",
Type: "Kubernetes",
JsonData: json,
SecureJsonData: map[string][]byte{
"tlsClientCert": tlsClientCert,
"tlsClientKey": tlsClientKey,
},
}
rt, err := dsService.GetHTTPTransport(&ds, provider)
require.NoError(t, err)
require.NotNil(t, rt)
tr := configuredTransport
require.False(t, tr.TLSClientConfig.InsecureSkipVerify)
require.Len(t, tr.TLSClientConfig.Certificates, 1)
})
t.Run("Should set user-supplied TLS CA if configured in JsonData", func(t *testing.T) {
var configuredTransport *http.Transport
provider := httpclient.NewProvider(sdkhttpclient.ProviderOptions{
ConfigureTransport: func(opts sdkhttpclient.Options, transport *http.Transport) {
configuredTransport = transport
},
})
setting.SecretKey = "password"
json := simplejson.New()
json.Set("tlsAuthWithCACert", true)
json.Set("serverName", "server-name")
encryptionService := ossencryption.ProvideService()
dsService := ProvideService(bus.New(), nil, encryptionService)
tlsCaCert, err := encryptionService.Encrypt(context.Background(), []byte(caCert), "password")
require.NoError(t, err)
ds := models.DataSource{
Id: 1,
Url: "http://k8s:8001",
Type: "Kubernetes",
JsonData: json,
SecureJsonData: map[string][]byte{
"tlsCACert": tlsCaCert,
},
}
rt, err := dsService.GetHTTPTransport(&ds, provider)
require.NoError(t, err)
require.NotNil(t, rt)
tr := configuredTransport
require.False(t, tr.TLSClientConfig.InsecureSkipVerify)
require.Len(t, tr.TLSClientConfig.RootCAs.Subjects(), 1)
require.Equal(t, "server-name", tr.TLSClientConfig.ServerName)
})
t.Run("Should set skip TLS verification if configured in JsonData", func(t *testing.T) {
var configuredTransport *http.Transport
provider := httpclient.NewProvider(sdkhttpclient.ProviderOptions{
ConfigureTransport: func(opts sdkhttpclient.Options, transport *http.Transport) {
configuredTransport = transport
},
})
json := simplejson.New()
json.Set("tlsSkipVerify", true)
encryptionService := ossencryption.ProvideService()
dsService := ProvideService(bus.New(), nil, encryptionService)
ds := models.DataSource{
Id: 1,
Url: "http://k8s:8001",
Type: "Kubernetes",
JsonData: json,
}
rt1, err := dsService.GetHTTPTransport(&ds, provider)
require.NoError(t, err)
require.NotNil(t, rt1)
tr1 := configuredTransport
rt2, err := dsService.GetHTTPTransport(&ds, provider)
require.NoError(t, err)
require.NotNil(t, rt2)
tr2 := configuredTransport
require.Same(t, tr1, tr2)
require.True(t, tr1.TLSClientConfig.InsecureSkipVerify)
})
t.Run("Should set custom headers if configured in JsonData", func(t *testing.T) {
provider := httpclient.NewProvider()
json := simplejson.NewFromAny(map[string]interface{}{
"httpHeaderName1": "Authorization",
})
encryptionService := ossencryption.ProvideService()
dsService := ProvideService(bus.New(), nil, encryptionService)
encryptedData, err := encryptionService.Encrypt(context.Background(), []byte(`Bearer xf5yhfkpsnmgo`), setting.SecretKey)
require.NoError(t, err)
ds := models.DataSource{
Id: 1,
Url: "http://k8s:8001",
Type: "Kubernetes",
JsonData: json,
SecureJsonData: map[string][]byte{"httpHeaderValue1": encryptedData},
}
headers := dsService.getCustomHeaders(json, map[string]string{"httpHeaderValue1": "Bearer xf5yhfkpsnmgo"})
require.Equal(t, "Bearer xf5yhfkpsnmgo", headers["Authorization"])
// 1. Start HTTP test server which checks the request headers
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Authorization") == "Bearer xf5yhfkpsnmgo" {
w.WriteHeader(200)
_, err := w.Write([]byte("Ok"))
require.NoError(t, err)
return
}
w.WriteHeader(403)
_, err := w.Write([]byte("Invalid bearer token provided"))
require.NoError(t, err)
}))
defer backend.Close()
// 2. Get HTTP transport from datasource which uses the test server as backend
ds.Url = backend.URL
rt, err := dsService.GetHTTPTransport(&ds, provider)
require.NoError(t, err)
require.NotNil(t, rt)
// 3. Send test request which should have the Authorization header set
req := httptest.NewRequest("GET", backend.URL+"/test-headers", nil)
res, err := rt.RoundTrip(req)
require.NoError(t, err)
t.Cleanup(func() {
err := res.Body.Close()
require.NoError(t, err)
})
body, err := ioutil.ReadAll(res.Body)
require.NoError(t, err)
bodyStr := string(body)
require.Equal(t, "Ok", bodyStr)
})
t.Run("Should use request timeout if configured in JsonData", func(t *testing.T) {
provider := httpclient.NewProvider()
json := simplejson.NewFromAny(map[string]interface{}{
"timeout": 19,
})
encryptionService := ossencryption.ProvideService()
dsService := ProvideService(bus.New(), nil, encryptionService)
ds := models.DataSource{
Id: 1,
Url: "http://k8s:8001",
Type: "Kubernetes",
JsonData: json,
}
client, err := dsService.GetHTTPClient(&ds, provider)
require.NoError(t, err)
require.NotNil(t, client)
require.Equal(t, 19*time.Second, client.Timeout)
})
t.Run("Should populate SigV4 options if configured in JsonData", func(t *testing.T) {
var configuredOpts sdkhttpclient.Options
provider := httpclient.NewProvider(sdkhttpclient.ProviderOptions{
ConfigureTransport: func(opts sdkhttpclient.Options, transport *http.Transport) {
configuredOpts = opts
},
})
origSigV4Enabled := setting.SigV4AuthEnabled
setting.SigV4AuthEnabled = true
t.Cleanup(func() {
setting.SigV4AuthEnabled = origSigV4Enabled
})
json, err := simplejson.NewJson([]byte(`{ "sigV4Auth": true }`))
require.NoError(t, err)
encryptionService := ossencryption.ProvideService()
dsService := ProvideService(bus.New(), nil, encryptionService)
ds := models.DataSource{
Type: models.DS_ES,
JsonData: json,
}
_, err = dsService.GetHTTPTransport(&ds, provider)
require.NoError(t, err)
require.NotNil(t, configuredOpts)
require.NotNil(t, configuredOpts.SigV4)
require.Equal(t, "es", configuredOpts.SigV4.Service)
})
}
func TestService_getTimeout(t *testing.T) {
originalTimeout := sdkhttpclient.DefaultTimeoutOptions.Timeout
sdkhttpclient.DefaultTimeoutOptions.Timeout = 60 * time.Second
t.Cleanup(func() {
sdkhttpclient.DefaultTimeoutOptions.Timeout = originalTimeout
})
testCases := []struct {
jsonData *simplejson.Json
expectedTimeout time.Duration
}{
{jsonData: simplejson.New(), expectedTimeout: 60 * time.Second},
{jsonData: simplejson.NewFromAny(map[string]interface{}{"timeout": nil}), expectedTimeout: 60 * time.Second},
{jsonData: simplejson.NewFromAny(map[string]interface{}{"timeout": 0}), expectedTimeout: 60 * time.Second},
{jsonData: simplejson.NewFromAny(map[string]interface{}{"timeout": 1}), expectedTimeout: time.Second},
{jsonData: simplejson.NewFromAny(map[string]interface{}{"timeout": "2"}), expectedTimeout: 2 * time.Second},
}
encryptionService := ossencryption.ProvideService()
dsService := ProvideService(bus.New(), nil, encryptionService)
for _, tc := range testCases {
ds := &models.DataSource{
JsonData: tc.jsonData,
}
assert.Equal(t, tc.expectedTimeout, dsService.getTimeout(ds))
}
}
func TestService_DecryptedValue(t *testing.T) {
t.Run("When datasource hasn't been updated, encrypted JSON should be fetched from cache", func(t *testing.T) {
encryptionService := ossencryption.ProvideService()
dsService := ProvideService(bus.New(), nil, encryptionService)
encryptedJsonData, err := encryptionService.EncryptJsonData(
context.Background(),
map[string]string{
"password": "password",
}, setting.SecretKey)
require.NoError(t, err)
ds := models.DataSource{
Id: 1,
Type: models.DS_INFLUXDB_08,
JsonData: simplejson.New(),
User: "user",
SecureJsonData: encryptedJsonData,
}
// Populate cache
password, ok := dsService.DecryptedValue(&ds, "password")
require.True(t, ok)
require.Equal(t, "password", password)
encryptedJsonData, err = encryptionService.EncryptJsonData(
context.Background(),
map[string]string{
"password": "",
}, setting.SecretKey)
require.NoError(t, err)
ds.SecureJsonData = encryptedJsonData
password, ok = dsService.DecryptedValue(&ds, "password")
require.True(t, ok)
require.Equal(t, "password", password)
})
t.Run("When datasource is updated, encrypted JSON should not be fetched from cache", func(t *testing.T) {
encryptionService := ossencryption.ProvideService()
encryptedJsonData, err := encryptionService.EncryptJsonData(
context.Background(),
map[string]string{
"password": "password",
}, setting.SecretKey)
require.NoError(t, err)
ds := models.DataSource{
Id: 1,
Type: models.DS_INFLUXDB_08,
JsonData: simplejson.New(),
User: "user",
SecureJsonData: encryptedJsonData,
}
dsService := ProvideService(bus.New(), nil, encryptionService)
// Populate cache
password, ok := dsService.DecryptedValue(&ds, "password")
require.True(t, ok)
require.Equal(t, "password", password)
ds.SecureJsonData, err = encryptionService.EncryptJsonData(
context.Background(),
map[string]string{
"password": "",
}, setting.SecretKey)
ds.Updated = time.Now()
require.NoError(t, err)
password, ok = dsService.DecryptedValue(&ds, "password")
require.True(t, ok)
require.Empty(t, password)
})
}
func TestService_HTTPClientOptions(t *testing.T) {
emptyJsonData := simplejson.New()
emptySecureJsonData := map[string][]byte{}
ds := models.DataSource{
Id: 1,
Url: "https://api.example.com",
Type: "prometheus",
}
t.Run("Azure authentication", func(t *testing.T) {
t.Run("should be disabled if not enabled in JsonData", func(t *testing.T) {
t.Cleanup(func() { ds.JsonData = emptyJsonData; ds.SecureJsonData = emptySecureJsonData })
encryptionService := ossencryption.ProvideService()
dsService := ProvideService(bus.New(), nil, encryptionService)
opts, err := dsService.httpClientOptions(&ds)
require.NoError(t, err)
assert.NotEqual(t, true, opts.CustomOptions["_azureAuth"])
assert.NotContains(t, opts.CustomOptions, "_azureCredentials")
})
t.Run("should be enabled if enabled in JsonData without credentials configured", func(t *testing.T) {
t.Cleanup(func() { ds.JsonData = emptyJsonData; ds.SecureJsonData = emptySecureJsonData })
ds.JsonData = simplejson.NewFromAny(map[string]interface{}{
"azureAuth": true,
})
encryptionService := ossencryption.ProvideService()
dsService := ProvideService(bus.New(), nil, encryptionService)
opts, err := dsService.httpClientOptions(&ds)
require.NoError(t, err)
assert.Equal(t, true, opts.CustomOptions["_azureAuth"])
assert.NotContains(t, opts.CustomOptions, "_azureCredentials")
})
t.Run("should be enabled if enabled in JsonData with credentials configured", func(t *testing.T) {
t.Cleanup(func() { ds.JsonData = emptyJsonData; ds.SecureJsonData = emptySecureJsonData })
ds.JsonData = simplejson.NewFromAny(map[string]interface{}{
"azureAuth": true,
"azureCredentials": map[string]interface{}{
"authType": "msi",
},
})
encryptionService := ossencryption.ProvideService()
dsService := ProvideService(bus.New(), nil, encryptionService)
opts, err := dsService.httpClientOptions(&ds)
require.NoError(t, err)
assert.Equal(t, true, opts.CustomOptions["_azureAuth"])
require.Contains(t, opts.CustomOptions, "_azureCredentials")
credentials := opts.CustomOptions["_azureCredentials"]
assert.IsType(t, &azcredentials.AzureManagedIdentityCredentials{}, credentials)
})
t.Run("should be disabled if disabled in JsonData even with credentials configured", func(t *testing.T) {
t.Cleanup(func() { ds.JsonData = emptyJsonData; ds.SecureJsonData = emptySecureJsonData })
ds.JsonData = simplejson.NewFromAny(map[string]interface{}{
"azureAuth": false,
"azureCredentials": map[string]interface{}{
"authType": "msi",
},
})
encryptionService := ossencryption.ProvideService()
dsService := ProvideService(bus.New(), nil, encryptionService)
opts, err := dsService.httpClientOptions(&ds)
require.NoError(t, err)
assert.NotEqual(t, true, opts.CustomOptions["_azureAuth"])
assert.NotContains(t, opts.CustomOptions, "_azureCredentials")
})
t.Run("should fail if credentials are invalid", func(t *testing.T) {
t.Cleanup(func() { ds.JsonData = emptyJsonData; ds.SecureJsonData = emptySecureJsonData })
ds.JsonData = simplejson.NewFromAny(map[string]interface{}{
"azureAuth": true,
"azureCredentials": "invalid",
})
encryptionService := ossencryption.ProvideService()
dsService := ProvideService(bus.New(), nil, encryptionService)
_, err := dsService.httpClientOptions(&ds)
assert.Error(t, err)
})
t.Run("should pass resourceId from JsonData", func(t *testing.T) {
t.Cleanup(func() { ds.JsonData = emptyJsonData; ds.SecureJsonData = emptySecureJsonData })
ds.JsonData = simplejson.NewFromAny(map[string]interface{}{
"azureEndpointResourceId": "https://api.example.com/abd5c4ce-ca73-41e9-9cb2-bed39aa2adb5",
})
encryptionService := ossencryption.ProvideService()
dsService := ProvideService(bus.New(), nil, encryptionService)
opts, err := dsService.httpClientOptions(&ds)
require.NoError(t, err)
require.Contains(t, opts.CustomOptions, "azureEndpointResourceId")
azureEndpointResourceId := opts.CustomOptions["azureEndpointResourceId"]
assert.Equal(t, "https://api.example.com/abd5c4ce-ca73-41e9-9cb2-bed39aa2adb5", azureEndpointResourceId)
})
})
}
const caCert string = `-----BEGIN CERTIFICATE-----
MIIDATCCAemgAwIBAgIJAMQ5hC3CPDTeMA0GCSqGSIb3DQEBCwUAMBcxFTATBgNV
BAMMDGNhLWs4cy1zdGhsbTAeFw0xNjEwMjcwODQyMjdaFw00NDAzMTQwODQyMjda
MBcxFTATBgNVBAMMDGNhLWs4cy1zdGhsbTCCASIwDQYJKoZIhvcNAQEBBQADggEP
ADCCAQoCggEBAMLe2AmJ6IleeUt69vgNchOjjmxIIxz5sp1vFu94m1vUip7CqnOg
QkpUsHeBPrGYv8UGloARCL1xEWS+9FVZeXWQoDmbC0SxXhFwRIESNCET7Q8KMi/4
4YPvnMLGZi3Fjwxa8BdUBCN1cx4WEooMVTWXm7RFMtZgDfuOAn3TNXla732sfT/d
1HNFrh48b0wA+HhmA3nXoBnBEblA665hCeo7lIAdRr0zJxJpnFnWXkyTClsAUTMN
iL905LdBiiIRenojipfKXvMz88XSaWTI7JjZYU3BvhyXndkT6f12cef3I96NY3WJ
0uIK4k04WrbzdYXMU3rN6NqlvbHqnI+E7aMCAwEAAaNQME4wHQYDVR0OBBYEFHHx
2+vSPw9bECHj3O51KNo5VdWOMB8GA1UdIwQYMBaAFHHx2+vSPw9bECHj3O51KNo5
VdWOMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAH2eV5NcV3LBJHs9
I+adbiTPg2vyumrGWwy73T0X8Dtchgt8wU7Q9b9Ucg2fOTmSSyS0iMqEu1Yb2ORB
CknM9mixHC9PwEBbkGCom3VVkqdLwSP6gdILZgyLoH4i8sTUz+S1yGPepi+Vzhs7
adOXtryjcGnwft6HdfKPNklMOHFnjw6uqpho54oj/z55jUpicY/8glDHdrr1bh3k
MHuiWLGewHXPvxfG6UoUx1te65IhifVcJGFZDQwfEmhBflfCmtAJlZEsgTLlBBCh
FHoXIyGOdq1chmRVocdGBCF8fUoGIbuF14r53rpvcbEKtKnnP8+96luKAZLq0a4n
3lb92xM=
-----END CERTIFICATE-----`
const clientCert string = `
-----BEGIN CERTIFICATE-----
MIICsjCCAZoCCQCcd8sOfstQLzANBgkqhkiG9w0BAQsFADAXMRUwEwYDVQQDDAxj
YS1rOHMtc3RobG0wHhcNMTYxMTAyMDkyNTE1WhcNMTcxMTAyMDkyNTE1WjAfMR0w
GwYDVQQDDBRhZG0tZGFuaWVsLWs4cy1zdGhsbTCCASIwDQYJKoZIhvcNAQEBBQAD
ggEPADCCAQoCggEBAOMliaWyNEUJKM37vWCl5bGub3lMicyRAqGQyY/qxD9yKKM2
FbucVcmWmg5vvTqQVl5rlQ+c7GI8OD6ptmFl8a26coEki7bFr8bkpSyBSEc5p27b
Z0ORFSqBHWHQbr9PkxPLYW6T3gZYUtRYv3OQgGxLXlvUh85n/mQfuR3N1FgmShHo
GtAFi/ht6leXa0Ms+jNSDLCmXpJm1GIEqgyKX7K3+g3vzo9coYqXq4XTa8Efs2v8
SCwqWfBC3rHfgs/5DLB8WT4Kul8QzxkytzcaBQfRfzhSV6bkgm7oTzt2/1eRRsf4
YnXzLE9YkCC9sAn+Owzqf+TYC1KRluWDfqqBTJUCAwEAATANBgkqhkiG9w0BAQsF
AAOCAQEAdMsZg6edWGC+xngizn0uamrUg1ViaDqUsz0vpzY5NWLA4MsBc4EtxWRP
ueQvjUimZ3U3+AX0YWNLIrH1FCVos2jdij/xkTUmHcwzr8rQy+B17cFi+a8jtpgw
AU6WWoaAIEhhbWQfth/Diz3mivl1ARB+YqiWca2mjRPLTPcKJEURDVddQ423el0Q
4JNxS5icu7T2zYTYHAo/cT9zVdLZl0xuLxYm3asK1IONJ/evxyVZima3il6MPvhe
58Hwz+m+HdqHxi24b/1J/VKYbISG4huOQCdLzeNXgvwFlGPUmHSnnKo1/KbQDAR5
llG/Sw5+FquFuChaA6l5KWy7F3bQyA==
-----END CERTIFICATE-----`
const clientKey string = `-----BEGIN RSA PRIVATE KEY-----
MIIEpQIBAAKCAQEA4yWJpbI0RQkozfu9YKXlsa5veUyJzJECoZDJj+rEP3IoozYV
u5xVyZaaDm+9OpBWXmuVD5zsYjw4Pqm2YWXxrbpygSSLtsWvxuSlLIFIRzmnbttn
Q5EVKoEdYdBuv0+TE8thbpPeBlhS1Fi/c5CAbEteW9SHzmf+ZB+5Hc3UWCZKEega
0AWL+G3qV5drQyz6M1IMsKZekmbUYgSqDIpfsrf6De/Oj1yhiperhdNrwR+za/xI
LCpZ8ELesd+Cz/kMsHxZPgq6XxDPGTK3NxoFB9F/OFJXpuSCbuhPO3b/V5FGx/hi
dfMsT1iQIL2wCf47DOp/5NgLUpGW5YN+qoFMlQIDAQABAoIBAQCzy4u312XeW1Cs
Mx6EuOwmh59/ESFmBkZh4rxZKYgrfE5EWlQ7i5SwG4BX+wR6rbNfy6JSmHDXlTkk
CKvvToVNcW6fYHEivDnVojhIERFIJ4+rhQmpBtcNLOQ3/4cZ8X/GxE6b+3lb5l+x
64mnjPLKRaIr5/+TVuebEy0xNTJmjnJ7yiB2HRz7uXEQaVSk/P7KAkkyl/9J3/LM
8N9AX1w6qDaNQZ4/P0++1H4SQenosM/b/GqGTomarEk/GE0NcB9rzmR9VCXa7FRh
WV5jyt9vUrwIEiK/6nUnOkGO8Ei3kB7Y+e+2m6WdaNoU5RAfqXmXa0Q/a0lLRruf
vTMo2WrBAoGBAPRaK4cx76Q+3SJ/wfznaPsMM06OSR8A3ctKdV+ip/lyKtb1W8Pz
k8MYQDH7GwPtSu5QD8doL00pPjugZL/ba7X9nAsI+pinyEErfnB9y7ORNEjIYYzs
DiqDKup7ANgw1gZvznWvb9Ge0WUSXvWS0pFkgootQAf+RmnnbWGH6l6RAoGBAO35
aGUrLro5u9RD24uSXNU3NmojINIQFK5dHAT3yl0BBYstL43AEsye9lX95uMPTvOQ
Cqcn42Hjp/bSe3n0ObyOZeXVrWcDFAfE0wwB1BkvL1lpgnFO9+VQORlH4w3Ppnpo
jcPkR2TFeDaAYtvckhxe/Bk3OnuFmnsQ3VzM75fFAoGBAI6PvS2XeNU+yA3EtA01
hg5SQ+zlHswz2TMuMeSmJZJnhY78f5mHlwIQOAPxGQXlf/4iP9J7en1uPpzTK3S0
M9duK4hUqMA/w5oiIhbHjf0qDnMYVbG+V1V+SZ+cPBXmCDihKreGr5qBKnHpkfV8
v9WL6o1rcRw4wiQvnaV1gsvBAoGBALtzVTczr6gDKCAIn5wuWy+cQSGTsBunjRLX
xuVm5iEiV+KMYkPvAx/pKzMLP96lRVR3ptyKgAKwl7LFk3u50+zh4gQLr35QH2wL
Lw7rNc3srAhrItPsFzqrWX6/cGuFoKYVS239l/sZzRppQPXcpb7xVvTp2whHcir0
Wtnpl+TdAoGAGqKqo2KU3JoY3IuTDUk1dsNAm8jd9EWDh+s1x4aG4N79mwcss5GD
FF8MbFPneK7xQd8L6HisKUDAUi2NOyynM81LAftPkvN6ZuUVeFDfCL4vCA0HUXLD
+VrOhtUZkNNJlLMiVRJuQKUOGlg8PpObqYbstQAf/0/yFJMRHG82Tcg=
-----END RSA PRIVATE KEY-----`