public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Datasource: Shared HTTP client provider for core backend data sources and any data source using the data source proxy (#33439)

Browse Source 
Uses new httpclient package from grafana-plugin-sdk-go introduced 
via grafana/grafana-plugin-sdk-go#328. 
Replaces the GetHTTPClient, GetTransport, GetTLSConfig methods defined 
on DataSource model.
Longer-term the goal is to migrate core HTTP backend data sources to use the 
SDK contracts and using httpclient.Provider for creating HTTP clients and such.

Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>
This commit is contained in:
Marcus Efraimsson
2021-05-19 23:53:41 +02:00
committed by GitHub
co-authored by Arve Knudsen
parent 7a83d1f9ff
commit 348e76fc8e
46 changed files with 1082 additions and 467 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/models/datasource_cache_test.go
+101 -62
View File
@@ -8,17 +8,25 @@ import (
"testing"
"time"
sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
"github.com/grafana/grafana/pkg/components/securejsondata"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
//nolint:goconst
func TestDataSource_GetHttpTransport(t *testing.T) {
t.Run("Should use cached proxy", func(t *testing.T) {
var configuredTransport *http.Transport
provider := httpclient.NewProvider(sdkhttpclient.ProviderOptions{
ConfigureTransport: func(opts sdkhttpclient.Options, transport *http.Transport) {
configuredTransport = transport
},
})
clearDSProxyCache(t)
ds := DataSource{
Id: 1,
@@ -26,20 +34,30 @@ func TestDataSource_GetHttpTransport(t *testing.T) {
Type: "Kubernetes",
}
tr1, err := ds.GetHttpTransport()
rt1, err := ds.GetHTTPTransport(provider)
require.NoError(t, err)
require.NotNil(t, rt1)
tr1 := configuredTransport
tr2, err := ds.GetHttpTransport()
rt2, err := ds.GetHTTPTransport(provider)
require.NoError(t, err)
require.NotNil(t, rt2)
tr2 := configuredTransport
require.Same(t, tr1, tr2)
assert.False(t, tr1.transport.TLSClientConfig.InsecureSkipVerify)
assert.Empty(t, tr1.transport.TLSClientConfig.Certificates)
assert.Nil(t, tr1.transport.TLSClientConfig.RootCAs)
require.False(t, tr1.TLSClientConfig.InsecureSkipVerify)
require.Empty(t, tr1.TLSClientConfig.Certificates)
require.Nil(t, tr1.TLSClientConfig.RootCAs)
})
t.Run("Should not use cached proxy when datasource updated", func(t *testing.T) {
var configuredTransport *http.Transport
provider := httpclient.NewProvider(sdkhttpclient.ProviderOptions{
ConfigureTransport: func(opts sdkhttpclient.Options, transport *http.Transport) {
configuredTransport = transport
},
})
clearDSProxyCache(t)
setting.SecretKey = "password"
@@ -56,26 +74,36 @@ func TestDataSource_GetHttpTransport(t *testing.T) {
Updated: time.Now().Add(-2 * time.Minute),
}
tr1, err := ds.GetHttpTransport()
rt1, err := ds.GetHTTPTransport(provider)
require.NotNil(t, rt1)
require.NoError(t, err)
assert.False(t, tr1.transport.TLSClientConfig.InsecureSkipVerify)
assert.Empty(t, tr1.transport.TLSClientConfig.Certificates)
assert.Nil(t, tr1.transport.TLSClientConfig.RootCAs)
tr1 := configuredTransport
require.False(t, tr1.TLSClientConfig.InsecureSkipVerify)
require.Empty(t, tr1.TLSClientConfig.Certificates)
require.Nil(t, tr1.TLSClientConfig.RootCAs)
ds.JsonData = nil
ds.SecureJsonData = map[string][]byte{}
ds.Updated = time.Now()
tr2, err := ds.GetHttpTransport()
rt2, err := ds.GetHTTPTransport(provider)
require.NoError(t, err)
require.NotNil(t, rt2)
tr2 := configuredTransport
require.NotSame(t, tr1, tr2)
assert.Nil(t, tr2.transport.TLSClientConfig.RootCAs)
require.Nil(t, tr2.TLSClientConfig.RootCAs)
})
t.Run("Should set TLS client authentication enabled if configured in JsonData", func(t *testing.T) {
var configuredTransport *http.Transport
provider := httpclient.NewProvider(sdkhttpclient.ProviderOptions{
ConfigureTransport: func(opts sdkhttpclient.Options, transport *http.Transport) {
configuredTransport = transport
},
})
clearDSProxyCache(t)
setting.SecretKey = "password"
@@ -98,15 +126,24 @@ func TestDataSource_GetHttpTransport(t *testing.T) {
},
}
tr, err := ds.GetHttpTransport()
rt, err := ds.GetHTTPTransport(provider)
require.NoError(t, err)
require.NotNil(t, rt)
tr := configuredTransport
assert.False(t, tr.transport.TLSClientConfig.InsecureSkipVerify)
require.Len(t, tr.transport.TLSClientConfig.Certificates, 1)
require.False(t, tr.TLSClientConfig.InsecureSkipVerify)
require.Len(t, tr.TLSClientConfig.Certificates, 1)
})
t.Run("Should set user-supplied TLS CA if configured in JsonData", func(t *testing.T) {
var configuredTransport *http.Transport
provider := httpclient.NewProvider(sdkhttpclient.ProviderOptions{
ConfigureTransport: func(opts sdkhttpclient.Options, transport *http.Transport) {
configuredTransport = transport
},
})
clearDSProxyCache(t)
ClearDSDecryptionCache()
setting.SecretKey = "password"
json := simplejson.New()
@@ -126,15 +163,23 @@ func TestDataSource_GetHttpTransport(t *testing.T) {
},
}
tr, err := ds.GetHttpTransport()
rt, err := ds.GetHTTPTransport(provider)
require.NoError(t, err)
require.NotNil(t, rt)
tr := configuredTransport
assert.False(t, tr.transport.TLSClientConfig.InsecureSkipVerify)
require.Len(t, tr.transport.TLSClientConfig.RootCAs.Subjects(), 1)
assert.Equal(t, "server-name", tr.transport.TLSClientConfig.ServerName)
require.False(t, tr.TLSClientConfig.InsecureSkipVerify)
require.Len(t, tr.TLSClientConfig.RootCAs.Subjects(), 1)
require.Equal(t, "server-name", tr.TLSClientConfig.ServerName)
})
t.Run("Should set skip TLS verification if configured in JsonData", func(t *testing.T) {
var configuredTransport *http.Transport
provider := httpclient.NewProvider(sdkhttpclient.ProviderOptions{
ConfigureTransport: func(opts sdkhttpclient.Options, transport *http.Transport) {
configuredTransport = transport
},
})
clearDSProxyCache(t)
json := simplejson.New()
@@ -147,19 +192,24 @@ func TestDataSource_GetHttpTransport(t *testing.T) {
JsonData: json,
}
tr1, err := ds.GetHttpTransport()
rt1, err := ds.GetHTTPTransport(provider)
require.NoError(t, err)
require.NotNil(t, rt1)
tr1 := configuredTransport
tr2, err := ds.GetHttpTransport()
rt2, err := ds.GetHTTPTransport(provider)
require.NoError(t, err)
require.NotNil(t, rt2)
tr2 := configuredTransport
require.Same(t, tr1, tr2)
assert.True(t, tr1.transport.TLSClientConfig.InsecureSkipVerify)
require.True(t, tr1.TLSClientConfig.InsecureSkipVerify)
})
t.Run("Should set custom headers if configured in JsonData", func(t *testing.T) {
provider := httpclient.NewProvider()
clearDSProxyCache(t)
ClearDSDecryptionCache()
json := simplejson.NewFromAny(map[string]interface{}{
"httpHeaderName1": "Authorization",
@@ -177,8 +227,8 @@ func TestDataSource_GetHttpTransport(t *testing.T) {
SecureJsonData: map[string][]byte{"httpHeaderValue1": encryptedData},
}
headers := ds.getCustomHeaders()
assert.Equal(t, "Bearer xf5yhfkpsnmgo", headers["Authorization"])
headers := getCustomHeaders(json, map[string]string{"httpHeaderValue1": "Bearer xf5yhfkpsnmgo"})
require.Equal(t, "Bearer xf5yhfkpsnmgo", headers["Authorization"])
// 1. Start HTTP test server which checks the request headers
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -197,12 +247,13 @@ func TestDataSource_GetHttpTransport(t *testing.T) {
// 2. Get HTTP transport from datasource which uses the test server as backend
ds.Url = backend.URL
tr, err := ds.GetHttpTransport()
rt, err := ds.GetHTTPTransport(provider)
require.NoError(t, err)
require.NotNil(t, rt)
// 3. Send test request which should have the Authorization header set
req := httptest.NewRequest("GET", backend.URL+"/test-headers", nil)
res, err := tr.RoundTrip(req)
res, err := rt.RoundTrip(req)
require.NoError(t, err)
t.Cleanup(func() {
err := res.Body.Close()
@@ -211,10 +262,11 @@ func TestDataSource_GetHttpTransport(t *testing.T) {
body, err := ioutil.ReadAll(res.Body)
require.NoError(t, err)
bodyStr := string(body)
assert.Equal(t, "Ok", bodyStr)
require.Equal(t, "Ok", bodyStr)
})
t.Run("Should use request timeout if configured in JsonData", func(t *testing.T) {
provider := httpclient.NewProvider()
clearDSProxyCache(t)
json := simplejson.NewFromAny(map[string]interface{}{
@@ -227,47 +279,34 @@ func TestDataSource_GetHttpTransport(t *testing.T) {
JsonData: json,
}
client, err := ds.GetHttpClient()
client, err := ds.GetHTTPClient(provider)
require.NoError(t, err)
assert.Equal(t, 19*time.Second, client.Timeout)
require.NotNil(t, client)
require.Equal(t, 19*time.Second, client.Timeout)
})
t.Run("Should not include SigV4 middleware if not configured in JsonData", func(t *testing.T) {
t.Run("Should populate SigV4 options if configured in JsonData", func(t *testing.T) {
var configuredOpts sdkhttpclient.Options
provider := httpclient.NewProvider(sdkhttpclient.ProviderOptions{
ConfigureTransport: func(opts sdkhttpclient.Options, transport *http.Transport) {
configuredOpts = opts
},
})
clearDSProxyCache(t)
origEnabled := setting.SigV4AuthEnabled
setting.SigV4AuthEnabled = true
t.Cleanup(func() { setting.SigV4AuthEnabled = origEnabled })
ds := DataSource{}
tr, err := ds.GetHttpTransport()
require.NoError(t, err)
_, ok := tr.next.(*http.Transport)
require.True(t, ok)
})
t.Run("Should not include SigV4 middleware if not configured in app config", func(t *testing.T) {
clearDSProxyCache(t)
origEnabled := setting.SigV4AuthEnabled
setting.SigV4AuthEnabled = false
t.Cleanup(func() { setting.SigV4AuthEnabled = origEnabled })
json, err := simplejson.NewJson([]byte(`{ "sigV4Auth": true }`))
require.NoError(t, err)
ds := DataSource{
Type: DS_ES,
JsonData: json,
}
tr, err := ds.GetHttpTransport()
_, err = ds.GetHTTPTransport(provider)
require.NoError(t, err)
_, ok := tr.next.(*http.Transport)
require.True(t, ok)
require.NotNil(t, configuredOpts)
require.NotNil(t, configuredOpts.SigV4)
require.Equal(t, "es", configuredOpts.SigV4.Service)
})
}
@@ -288,7 +327,7 @@ func TestDataSource_DecryptedValue(t *testing.T) {
// Populate cache
password, ok := ds.DecryptedValue("password")
require.True(t, ok)
assert.Equal(t, "password", password)
require.Equal(t, "password", password)
ds.SecureJsonData = securejsondata.GetEncryptedJsonData(map[string]string{
"password": "",
@@ -296,7 +335,7 @@ func TestDataSource_DecryptedValue(t *testing.T) {
password, ok = ds.DecryptedValue("password")
require.True(t, ok)
assert.Equal(t, "password", password)
require.Equal(t, "password", password)
})
t.Run("When datasource is updated, encrypted JSON should not be fetched from cache", func(t *testing.T) {
@@ -315,7 +354,7 @@ func TestDataSource_DecryptedValue(t *testing.T) {
// Populate cache
password, ok := ds.DecryptedValue("password")
require.True(t, ok)
assert.Equal(t, "password", password)
require.Equal(t, "password", password)
ds.SecureJsonData = securejsondata.GetEncryptedJsonData(map[string]string{
"password": "",
@@ -324,7 +363,7 @@ func TestDataSource_DecryptedValue(t *testing.T) {
password, ok = ds.DecryptedValue("password")
require.True(t, ok)
assert.Empty(t, password)
require.Empty(t, password)
})
}
@@ -334,7 +373,7 @@ func clearDSProxyCache(t *testing.T) {
ptc.Lock()
defer ptc.Unlock()
ptc.cache = make(map[int64]cachedTransport)
ptc.cache = make(map[int64]cachedRoundTripper)
}
const caCert string = `-----BEGIN CERTIFICATE-----