public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use cache for http.client in tsdb package. (#6833)

Browse Source 
* datasource: move caching closer to datasource struct

* tsdb: use cached version of datasource http transport

closes #6825
This commit is contained in:
Carl Bergquist
2016-12-07 11:10:42 +01:00
committed by Torkel Ödegaard
parent 24172fca01
commit 2c7adccf12
20 changed files with 385 additions and 363 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/tsdb/batch.go
+4 -7
View File
@@ -1,9 +1,6 @@
package tsdb
import (
"context"
"errors"
)
import "context"
type Batch struct {
DataSourceId int64
@@ -24,12 +21,12 @@ func newBatch(dsId int64, queries QuerySlice) *Batch {
}
func (bg *Batch) process(ctx context.Context, queryContext *QueryContext) {
executor := getExecutorFor(bg.Queries[0].DataSource)
executor, err := getExecutorFor(bg.Queries[0].DataSource)
if executor == nil {
if err != nil {
bg.Done = true
result := &BatchResult{
Error: errors.New("Could not find executor for data source type: " + bg.Queries[0].DataSource.PluginId),
Error: err,
QueryResults: make(map[string]*QueryResult),
}
for _, query := range bg.Queries {
pkg/tsdb/executor.go
+16 -6
View File
@@ -1,6 +1,11 @@
package tsdb
import "context"
import (
"context"
"fmt"
"github.com/grafana/grafana/pkg/models"
)
type Executor interface {
Execute(ctx context.Context, queries QuerySlice, query *QueryContext) *BatchResult
@@ -8,17 +13,22 @@ type Executor interface {
var registry map[string]GetExecutorFn
type GetExecutorFn func(dsInfo *DataSourceInfo) Executor
type GetExecutorFn func(dsInfo *models.DataSource) (Executor, error)
func init() {
registry = make(map[string]GetExecutorFn)
}
func getExecutorFor(dsInfo *DataSourceInfo) Executor {
if fn, exists := registry[dsInfo.PluginId]; exists {
return fn(dsInfo)
func getExecutorFor(dsInfo *models.DataSource) (Executor, error) {
if fn, exists := registry[dsInfo.Type]; exists {
executor, err := fn(dsInfo)
if err != nil {
return nil, err
}
return executor, nil
}
return nil
return nil, fmt.Errorf("Could not find executor for data source type: %s", dsInfo.Type)
}
func RegisterExecutor(pluginId string, fn GetExecutorFn) {
pkg/tsdb/fake_test.go
+7 -3
View File
@@ -1,6 +1,10 @@
package tsdb
import "context"
import (
"context"
"github.com/grafana/grafana/pkg/models"
)
type FakeExecutor struct {
results map[string]*QueryResult
@@ -9,11 +13,11 @@ type FakeExecutor struct {
type ResultsFn func(context *QueryContext) *QueryResult
func NewFakeExecutor(dsInfo *DataSourceInfo) *FakeExecutor {
func NewFakeExecutor(dsInfo *models.DataSource) (*FakeExecutor, error) {
return &FakeExecutor{
results: make(map[string]*QueryResult),
resultsFn: make(map[string]ResultsFn),
}
}, nil
}
func (e *FakeExecutor) Execute(ctx context.Context, queries QuerySlice, context *QueryContext) *BatchResult {
pkg/tsdb/graphite/graphite.go
+16 -8
View File
@@ -14,28 +14,36 @@ import (
"golang.org/x/net/context/ctxhttp"
"github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb"
)
type GraphiteExecutor struct {
*tsdb.DataSourceInfo
*models.DataSource
HttpClient *http.Client
}
func NewGraphiteExecutor(dsInfo *tsdb.DataSourceInfo) tsdb.Executor {
return &GraphiteExecutor{dsInfo}
func NewGraphiteExecutor(datasource *models.DataSource) (tsdb.Executor, error) {
httpClient, err := datasource.GetHttpClient()
if err != nil {
return nil, err
}
return &GraphiteExecutor{
DataSource: datasource,
HttpClient: httpClient,
}, nil
}
var (
glog log.Logger
HttpClient *http.Client
glog log.Logger
)
func init() {
glog = log.New("tsdb.graphite")
tsdb.RegisterExecutor("graphite", NewGraphiteExecutor)
HttpClient = tsdb.GetDefaultClient()
}
func (e *GraphiteExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, context *tsdb.QueryContext) *tsdb.BatchResult {
@@ -66,7 +74,7 @@ func (e *GraphiteExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice,
return result
}
res, err := ctxhttp.Do(ctx, HttpClient, req)
res, err := ctxhttp.Do(ctx, e.HttpClient, req)
if err != nil {
result.Error = err
return result
pkg/tsdb/http.go
-29
View File
@@ -1,29 +0,0 @@
package tsdb
import (
"crypto/tls"
"net"
"net/http"
"time"
)
func GetDefaultClient() *http.Client {
tr := &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).DialContext,
MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
return &http.Client{
Timeout: time.Duration(30 * time.Second),
Transport: tr,
}
}
pkg/tsdb/influxdb/influxdb.go
+16 -10
View File
@@ -11,34 +11,40 @@ import (
"golang.org/x/net/context/ctxhttp"
"github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb"
)
type InfluxDBExecutor struct {
*tsdb.DataSourceInfo
*models.DataSource
QueryParser *InfluxdbQueryParser
ResponseParser *ResponseParser
HttpClient *http.Client
}
func NewInfluxDBExecutor(dsInfo *tsdb.DataSourceInfo) tsdb.Executor {
func NewInfluxDBExecutor(datasource *models.DataSource) (tsdb.Executor, error) {
httpClient, err := datasource.GetHttpClient()
if err != nil {
return nil, err
}
return &InfluxDBExecutor{
DataSourceInfo: dsInfo,
DataSource: datasource,
QueryParser: &InfluxdbQueryParser{},
ResponseParser: &ResponseParser{},
}
HttpClient: httpClient,
}, nil
}
var (
glog log.Logger
HttpClient *http.Client
glog log.Logger
)
func init() {
glog = log.New("tsdb.influxdb")
tsdb.RegisterExecutor("influxdb", NewInfluxDBExecutor)
HttpClient = tsdb.GetDefaultClient()
}
func (e *InfluxDBExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, context *tsdb.QueryContext) *tsdb.BatchResult {
@@ -63,7 +69,7 @@ func (e *InfluxDBExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice,
return result.WithError(err)
}
resp, err := ctxhttp.Do(ctx, HttpClient, req)
resp, err := ctxhttp.Do(ctx, e.HttpClient, req)
if err != nil {
return result.WithError(err)
}
@@ -95,7 +101,7 @@ func (e *InfluxDBExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice,
func (e *InfluxDBExecutor) getQuery(queries tsdb.QuerySlice, context *tsdb.QueryContext) (*Query, error) {
for _, v := range queries {
query, err := e.QueryParser.Parse(v.Model, e.DataSourceInfo)
query, err := e.QueryParser.Parse(v.Model, e.DataSource)
if err != nil {
return nil, err
}
pkg/tsdb/influxdb/model_parser.go
+2 -2
View File
@@ -4,12 +4,12 @@ import (
"strconv"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/grafana/grafana/pkg/models"
)
type InfluxdbQueryParser struct{}
func (qp *InfluxdbQueryParser) Parse(model *simplejson.Json, dsInfo *tsdb.DataSourceInfo) (*Query, error) {
func (qp *InfluxdbQueryParser) Parse(model *simplejson.Json, dsInfo *models.DataSource) (*Query, error) {
policy := model.Get("policy").MustString("default")
rawQuery := model.Get("query").MustString("")
useRawQuery := model.Get("rawQuery").MustBool(false)
pkg/tsdb/influxdb/model_parser_test.go
+2 -2
View File
@@ -4,7 +4,7 @@ import (
"testing"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/grafana/grafana/pkg/models"
. "github.com/smartystreets/goconvey/convey"
)
@@ -12,7 +12,7 @@ func TestInfluxdbQueryParser(t *testing.T) {
Convey("Influxdb query parser", t, func() {
parser := &InfluxdbQueryParser{}
dsInfo := &tsdb.DataSourceInfo{
dsInfo := &models.DataSource{
JsonData: simplejson.New(),
}
pkg/tsdb/models.go
+2 -15
View File
@@ -2,6 +2,7 @@ package tsdb
import (
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"gopkg.in/guregu/null.v3"
)
@@ -9,7 +10,7 @@ type Query struct {
RefId string
Model *simplejson.Json
Depends []string
DataSource *DataSourceInfo
DataSource *models.DataSource
Results []*TimeSeries
Exclude bool
MaxDataPoints int64
@@ -28,20 +29,6 @@ type Response struct {
Results map[string]*QueryResult `json:"results"`
}
type DataSourceInfo struct {
Id int64
Name string
PluginId string
Url string
Password string
User string
Database string
BasicAuth bool
BasicAuthUser string
BasicAuthPassword string
JsonData *simplejson.Json
}
type BatchTiming struct {
TimeElapsed int64
}
pkg/tsdb/opentsdb/opentsdb.go
+16 -8
View File
@@ -17,28 +17,36 @@ import (
"gopkg.in/guregu/null.v3"
"github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb"
)
type OpenTsdbExecutor struct {
*tsdb.DataSourceInfo
*models.DataSource
httpClient *http.Client
}
func NewOpenTsdbExecutor(dsInfo *tsdb.DataSourceInfo) tsdb.Executor {
return &OpenTsdbExecutor{dsInfo}
func NewOpenTsdbExecutor(datasource *models.DataSource) (tsdb.Executor, error) {
httpClient, err := datasource.GetHttpClient()
if err != nil {
return nil, err
}
return &OpenTsdbExecutor{
DataSource: datasource,
httpClient: httpClient,
}, nil
}
var (
plog log.Logger
HttpClient *http.Client
plog log.Logger
)
func init() {
plog = log.New("tsdb.opentsdb")
tsdb.RegisterExecutor("opentsdb", NewOpenTsdbExecutor)
HttpClient = tsdb.GetDefaultClient()
}
func (e *OpenTsdbExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, queryContext *tsdb.QueryContext) *tsdb.BatchResult {
@@ -64,7 +72,7 @@ func (e *OpenTsdbExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice,
return result
}
res, err := ctxhttp.Do(ctx, HttpClient, req)
res, err := ctxhttp.Do(ctx, e.httpClient, req)
if err != nil {
result.Error = err
return result
pkg/tsdb/prometheus/prometheus.go
+17 -4
View File
@@ -9,18 +9,30 @@ import (
"gopkg.in/guregu/null.v3"
"net/http"
"github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/prometheus/client_golang/api/prometheus"
pmodel "github.com/prometheus/common/model"
)
type PrometheusExecutor struct {
*tsdb.DataSourceInfo
*models.DataSource
Transport *http.Transport
}
func NewPrometheusExecutor(dsInfo *tsdb.DataSourceInfo) tsdb.Executor {
return &PrometheusExecutor{dsInfo}
func NewPrometheusExecutor(dsInfo *models.DataSource) (tsdb.Executor, error) {
transport, err := dsInfo.GetHttpTransport()
if err != nil {
return nil, err
}
return &PrometheusExecutor{
DataSource: dsInfo,
Transport: transport,
}, nil
}
var (
@@ -36,7 +48,8 @@ func init() {
func (e *PrometheusExecutor) getClient() (prometheus.QueryAPI, error) {
cfg := prometheus.Config{
Address: e.DataSourceInfo.Url,
Address: e.DataSource.Url,
Transport: e.Transport,
}
client, err := prometheus.New(cfg)
pkg/tsdb/testdata/testdata.go
Vendored
+6 -5
View File
@@ -4,19 +4,20 @@ import (
"context"
"github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/tsdb"
)
type TestDataExecutor struct {
*tsdb.DataSourceInfo
*models.DataSource
log log.Logger
}
func NewTestDataExecutor(dsInfo *tsdb.DataSourceInfo) tsdb.Executor {
func NewTestDataExecutor(dsInfo *models.DataSource) (tsdb.Executor, error) {
return &TestDataExecutor{
DataSourceInfo: dsInfo,
log: log.New("tsdb.testdata"),
}
DataSource: dsInfo,
log: log.New("tsdb.testdata"),
}, nil
}
func init() {
pkg/tsdb/tsdb_test.go
+19 -18
View File
@@ -5,6 +5,7 @@ import (
"testing"
"time"
"github.com/grafana/grafana/pkg/models"
. "github.com/smartystreets/goconvey/convey"
)
@@ -15,9 +16,9 @@ func TestMetricQuery(t *testing.T) {
Convey("Given 3 queries for 2 data sources", func() {
request := &Request{
Queries: QuerySlice{
{RefId: "A", DataSource: &DataSourceInfo{Id: 1}},
{RefId: "B", DataSource: &DataSourceInfo{Id: 1}},
{RefId: "C", DataSource: &DataSourceInfo{Id: 2}},
{RefId: "A", DataSource: &models.DataSource{Id: 1}},
{RefId: "B", DataSource: &models.DataSource{Id: 1}},
{RefId: "C", DataSource: &models.DataSource{Id: 2}},
},
}
@@ -32,9 +33,9 @@ func TestMetricQuery(t *testing.T) {
Convey("Given query 2 depends on query 1", func() {
request := &Request{
Queries: QuerySlice{
{RefId: "A", DataSource: &DataSourceInfo{Id: 1}},
{RefId: "B", DataSource: &DataSourceInfo{Id: 2}},
{RefId: "C", DataSource: &DataSourceInfo{Id: 3}, Depends: []string{"A", "B"}},
{RefId: "A", DataSource: &models.DataSource{Id: 1}},
{RefId: "B", DataSource: &models.DataSource{Id: 2}},
{RefId: "C", DataSource: &models.DataSource{Id: 3}, Depends: []string{"A", "B"}},
},
}
@@ -56,7 +57,7 @@ func TestMetricQuery(t *testing.T) {
Convey("When executing request with one query", t, func() {
req := &Request{
Queries: QuerySlice{
{RefId: "A", DataSource: &DataSourceInfo{Id: 1, PluginId: "test"}},
{RefId: "A", DataSource: &models.DataSource{Id: 1, Type: "test"}},
},
}
@@ -75,8 +76,8 @@ func TestMetricQuery(t *testing.T) {
Convey("When executing one request with two queries from same data source", t, func() {
req := &Request{
Queries: QuerySlice{
{RefId: "A", DataSource: &DataSourceInfo{Id: 1, PluginId: "test"}},
{RefId: "B", DataSource: &DataSourceInfo{Id: 1, PluginId: "test"}},
{RefId: "A", DataSource: &models.DataSource{Id: 1, Type: "test"}},
{RefId: "B", DataSource: &models.DataSource{Id: 1, Type: "test"}},
},
}
@@ -101,9 +102,9 @@ func TestMetricQuery(t *testing.T) {
Convey("When executing one request with three queries from different datasources", t, func() {
req := &Request{
Queries: QuerySlice{
{RefId: "A", DataSource: &DataSourceInfo{Id: 1, PluginId: "test"}},
{RefId: "B", DataSource: &DataSourceInfo{Id: 1, PluginId: "test"}},
{RefId: "C", DataSource: &DataSourceInfo{Id: 2, PluginId: "test"}},
{RefId: "A", DataSource: &models.DataSource{Id: 1, Type: "test"}},
{RefId: "B", DataSource: &models.DataSource{Id: 1, Type: "test"}},
{RefId: "C", DataSource: &models.DataSource{Id: 2, Type: "test"}},
},
}
@@ -118,7 +119,7 @@ func TestMetricQuery(t *testing.T) {
Convey("When query uses data source of unknown type", t, func() {
req := &Request{
Queries: QuerySlice{
{RefId: "A", DataSource: &DataSourceInfo{Id: 1, PluginId: "asdasdas"}},
{RefId: "A", DataSource: &models.DataSource{Id: 1, Type: "asdasdas"}},
},
}
@@ -130,10 +131,10 @@ func TestMetricQuery(t *testing.T) {
req := &Request{
Queries: QuerySlice{
{
RefId: "A", DataSource: &DataSourceInfo{Id: 1, PluginId: "test"},
RefId: "A", DataSource: &models.DataSource{Id: 1, Type: "test"},
},
{
RefId: "B", DataSource: &DataSourceInfo{Id: 2, PluginId: "test"}, Depends: []string{"A"},
RefId: "B", DataSource: &models.DataSource{Id: 2, Type: "test"}, Depends: []string{"A"},
},
},
}
@@ -167,9 +168,9 @@ func TestMetricQuery(t *testing.T) {
}
func registerFakeExecutor() *FakeExecutor {
executor := NewFakeExecutor(nil)
RegisterExecutor("test", func(dsInfo *DataSourceInfo) Executor {
return executor
executor, _ := NewFakeExecutor(nil)
RegisterExecutor("test", func(dsInfo *models.DataSource) (Executor, error) {
return executor, nil
})
return executor