public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Introduce TSDB service (#31520)

Browse Source 
* Introduce TSDB service

Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>

Co-authored-by: Erik Sundell <erik.sundell87@gmail.com>
Co-authored-by: Will Browne <will.browne@grafana.com>
Co-authored-by: Torkel Ödegaard <torkel@grafana.org>
Co-authored-by: Will Browne <wbrowne@users.noreply.github.com>
Co-authored-by: Zoltán Bedi <zoltan.bedi@gmail.com>
This commit is contained in:
Arve Knudsen
2021-03-08 07:02:49 +01:00
committed by GitHub
co-authored by Erik Sundell Will Browne Torkel Ödegaard Will Browne Zoltán Bedi
parent c899bf3592
commit b79e61656a
203 changed files with 5269 additions and 4776 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/tsdb/elasticsearch/client/client.go
+6 -5
View File
@@ -15,9 +15,10 @@ import (
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/grafana/grafana/pkg/tsdb/interval"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"golang.org/x/net/context/ctxhttp"
)
@@ -42,7 +43,7 @@ type Client interface {
}
// NewClient creates a new elasticsearch client
var NewClient = func(ctx context.Context, ds *models.DataSource, timeRange *tsdb.TimeRange) (Client, error) {
var NewClient = func(ctx context.Context, ds *models.DataSource, timeRange plugins.DataTimeRange) (Client, error) {
version, err := ds.JsonData.Get("esVersion").Int()
if err != nil {
return nil, fmt.Errorf("elasticsearch version is required, err=%v", err)
@@ -87,7 +88,7 @@ type baseClientImpl struct {
version int
timeField string
indices []string
timeRange *tsdb.TimeRange
timeRange plugins.DataTimeRange
debugEnabled bool
}
@@ -100,7 +101,7 @@ func (c *baseClientImpl) GetTimeField() string {
}
func (c *baseClientImpl) GetMinInterval(queryInterval string) (time.Duration, error) {
return tsdb.GetIntervalFrom(c.ds, simplejson.NewFromAny(map[string]interface{}{
return interval.GetIntervalFrom(c.ds, simplejson.NewFromAny(map[string]interface{}{
"interval": queryInterval,
}), 5*time.Second)
}
@@ -112,7 +113,7 @@ func (c *baseClientImpl) getSettings() *simplejson.Json {
type multiRequest struct {
header map[string]interface{}
body interface{}
interval tsdb.Interval
interval interval.Interval
}
func (c *baseClientImpl) executeBatchRequest(uriPath, uriQuery string, requests []*multiRequest) (*response, error) {
pkg/tsdb/elasticsearch/client/client_test.go
+12 -11
View File
@@ -12,7 +12,8 @@ import (
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/tsdb/interval"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -23,7 +24,7 @@ func TestNewClient(t *testing.T) {
JsonData: simplejson.NewFromAny(make(map[string]interface{})),
}
_, err := NewClient(context.Background(), ds, nil)
_, err := NewClient(context.Background(), ds, plugins.DataTimeRange{})
require.Error(t, err)
})
@@ -34,7 +35,7 @@ func TestNewClient(t *testing.T) {
}),
}
_, err := NewClient(context.Background(), ds, nil)
_, err := NewClient(context.Background(), ds, plugins.DataTimeRange{})
require.Error(t, err)
})
@@ -46,7 +47,7 @@ func TestNewClient(t *testing.T) {
}),
}
_, err := NewClient(context.Background(), ds, nil)
_, err := NewClient(context.Background(), ds, plugins.DataTimeRange{})
require.Error(t, err)
})
@@ -58,7 +59,7 @@ func TestNewClient(t *testing.T) {
}),
}
c, err := NewClient(context.Background(), ds, nil)
c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{})
require.NoError(t, err)
assert.Equal(t, 2, c.GetVersion())
})
@@ -71,7 +72,7 @@ func TestNewClient(t *testing.T) {
}),
}
c, err := NewClient(context.Background(), ds, nil)
c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{})
require.NoError(t, err)
assert.Equal(t, 5, c.GetVersion())
})
@@ -84,7 +85,7 @@ func TestNewClient(t *testing.T) {
}),
}
c, err := NewClient(context.Background(), ds, nil)
c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{})
require.NoError(t, err)
assert.Equal(t, 56, c.GetVersion())
})
@@ -97,7 +98,7 @@ func TestNewClient(t *testing.T) {
}),
}
c, err := NewClient(context.Background(), ds, nil)
c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{})
require.NoError(t, err)
assert.Equal(t, 60, c.GetVersion())
})
@@ -110,7 +111,7 @@ func TestNewClient(t *testing.T) {
}),
}
c, err := NewClient(context.Background(), ds, nil)
c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{})
require.NoError(t, err)
assert.Equal(t, 70, c.GetVersion())
})
@@ -329,7 +330,7 @@ func createMultisearchForTest(t *testing.T, c Client) (*MultiSearchRequest, erro
t.Helper()
msb := c.MultiSearch()
s := msb.Search(tsdb.Interval{Value: 15 * time.Second, Text: "15s"})
s := msb.Search(interval.Interval{Value: 15 * time.Second, Text: "15s"})
s.Agg().DateHistogram("2", "@timestamp", func(a *DateHistogramAgg, ab AggBuilder) {
a.Interval = "$__interval"
@@ -376,7 +377,7 @@ func httpClientScenario(t *testing.T, desc string, ds *models.DataSource, fn sce
to := time.Date(2018, 5, 15, 17, 55, 0, 0, time.UTC)
fromStr := fmt.Sprintf("%d", from.UnixNano()/int64(time.Millisecond))
toStr := fmt.Sprintf("%d", to.UnixNano()/int64(time.Millisecond))
timeRange := tsdb.NewTimeRange(fromStr, toStr)
timeRange := plugins.NewDataTimeRange(fromStr, toStr)
c, err := NewClient(context.Background(), ds, timeRange)
require.NoError(t, err)
pkg/tsdb/elasticsearch/client/index_pattern.go
+4 -4
View File
@@ -6,7 +6,7 @@ import (
"strings"
"time"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/grafana/grafana/pkg/plugins"
)
const (
@@ -19,7 +19,7 @@ const (
)
type indexPattern interface {
GetIndices(timeRange *tsdb.TimeRange) ([]string, error)
GetIndices(timeRange plugins.DataTimeRange) ([]string, error)
}
var newIndexPattern = func(interval string, pattern string) (indexPattern, error) {
@@ -34,7 +34,7 @@ type staticIndexPattern struct {
indexName string
}
func (ip *staticIndexPattern) GetIndices(timeRange *tsdb.TimeRange) ([]string, error) {
func (ip *staticIndexPattern) GetIndices(timeRange plugins.DataTimeRange) ([]string, error) {
return []string{ip.indexName}, nil
}
@@ -73,7 +73,7 @@ func newDynamicIndexPattern(interval, pattern string) (*dynamicIndexPattern, err
}, nil
}
func (ip *dynamicIndexPattern) GetIndices(timeRange *tsdb.TimeRange) ([]string, error) {
func (ip *dynamicIndexPattern) GetIndices(timeRange plugins.DataTimeRange) ([]string, error) {
from := timeRange.GetFromAsTimeUTC()
to := timeRange.GetToAsTimeUTC()
intervals := ip.intervalGenerator.Generate(from, to)
pkg/tsdb/elasticsearch/client/index_pattern_test.go
+17 -18
View File
@@ -5,19 +5,18 @@ import (
"testing"
"time"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/grafana/grafana/pkg/plugins"
. "github.com/smartystreets/goconvey/convey"
)
func TestIndexPattern(t *testing.T) {
Convey("Static index patterns", t, func() {
indexPatternScenario(noInterval, "data-*", nil, func(indices []string) {
indexPatternScenario(noInterval, "data-*", plugins.DataTimeRange{}, func(indices []string) {
So(indices, ShouldHaveLength, 1)
So(indices[0], ShouldEqual, "data-*")
})
indexPatternScenario(noInterval, "es-index-name", nil, func(indices []string) {
indexPatternScenario(noInterval, "es-index-name", plugins.DataTimeRange{}, func(indices []string) {
So(indices, ShouldHaveLength, 1)
So(indices[0], ShouldEqual, "es-index-name")
})
@@ -27,62 +26,62 @@ func TestIndexPattern(t *testing.T) {
from := fmt.Sprintf("%d", time.Date(2018, 5, 15, 17, 50, 0, 0, time.UTC).UnixNano()/int64(time.Millisecond))
to := fmt.Sprintf("%d", time.Date(2018, 5, 15, 17, 55, 0, 0, time.UTC).UnixNano()/int64(time.Millisecond))
indexPatternScenario(intervalHourly, "[data-]YYYY.MM.DD.HH", tsdb.NewTimeRange(from, to), func(indices []string) {
indexPatternScenario(intervalHourly, "[data-]YYYY.MM.DD.HH", plugins.NewDataTimeRange(from, to), func(indices []string) {
So(indices, ShouldHaveLength, 1)
So(indices[0], ShouldEqual, "data-2018.05.15.17")
})
indexPatternScenario(intervalHourly, "YYYY.MM.DD.HH[-data]", tsdb.NewTimeRange(from, to), func(indices []string) {
indexPatternScenario(intervalHourly, "YYYY.MM.DD.HH[-data]", plugins.NewDataTimeRange(from, to), func(indices []string) {
So(indices, ShouldHaveLength, 1)
So(indices[0], ShouldEqual, "2018.05.15.17-data")
})
indexPatternScenario(intervalDaily, "[data-]YYYY.MM.DD", tsdb.NewTimeRange(from, to), func(indices []string) {
indexPatternScenario(intervalDaily, "[data-]YYYY.MM.DD", plugins.NewDataTimeRange(from, to), func(indices []string) {
So(indices, ShouldHaveLength, 1)
So(indices[0], ShouldEqual, "data-2018.05.15")
})
indexPatternScenario(intervalDaily, "YYYY.MM.DD[-data]", tsdb.NewTimeRange(from, to), func(indices []string) {
indexPatternScenario(intervalDaily, "YYYY.MM.DD[-data]", plugins.NewDataTimeRange(from, to), func(indices []string) {
So(indices, ShouldHaveLength, 1)
So(indices[0], ShouldEqual, "2018.05.15-data")
})
indexPatternScenario(intervalWeekly, "[data-]GGGG.WW", tsdb.NewTimeRange(from, to), func(indices []string) {
indexPatternScenario(intervalWeekly, "[data-]GGGG.WW", plugins.NewDataTimeRange(from, to), func(indices []string) {
So(indices, ShouldHaveLength, 1)
So(indices[0], ShouldEqual, "data-2018.20")
})
indexPatternScenario(intervalWeekly, "GGGG.WW[-data]", tsdb.NewTimeRange(from, to), func(indices []string) {
indexPatternScenario(intervalWeekly, "GGGG.WW[-data]", plugins.NewDataTimeRange(from, to), func(indices []string) {
So(indices, ShouldHaveLength, 1)
So(indices[0], ShouldEqual, "2018.20-data")
})
indexPatternScenario(intervalMonthly, "[data-]YYYY.MM", tsdb.NewTimeRange(from, to), func(indices []string) {
indexPatternScenario(intervalMonthly, "[data-]YYYY.MM", plugins.NewDataTimeRange(from, to), func(indices []string) {
So(indices, ShouldHaveLength, 1)
So(indices[0], ShouldEqual, "data-2018.05")
})
indexPatternScenario(intervalMonthly, "YYYY.MM[-data]", tsdb.NewTimeRange(from, to), func(indices []string) {
indexPatternScenario(intervalMonthly, "YYYY.MM[-data]", plugins.NewDataTimeRange(from, to), func(indices []string) {
So(indices, ShouldHaveLength, 1)
So(indices[0], ShouldEqual, "2018.05-data")
})
indexPatternScenario(intervalYearly, "[data-]YYYY", tsdb.NewTimeRange(from, to), func(indices []string) {
indexPatternScenario(intervalYearly, "[data-]YYYY", plugins.NewDataTimeRange(from, to), func(indices []string) {
So(indices, ShouldHaveLength, 1)
So(indices[0], ShouldEqual, "data-2018")
})
indexPatternScenario(intervalYearly, "YYYY[-data]", tsdb.NewTimeRange(from, to), func(indices []string) {
indexPatternScenario(intervalYearly, "YYYY[-data]", plugins.NewDataTimeRange(from, to), func(indices []string) {
So(indices, ShouldHaveLength, 1)
So(indices[0], ShouldEqual, "2018-data")
})
indexPatternScenario(intervalDaily, "YYYY[-data-]MM.DD", tsdb.NewTimeRange(from, to), func(indices []string) {
indexPatternScenario(intervalDaily, "YYYY[-data-]MM.DD", plugins.NewDataTimeRange(from, to), func(indices []string) {
So(indices, ShouldHaveLength, 1)
So(indices[0], ShouldEqual, "2018-data-05.15")
})
indexPatternScenario(intervalDaily, "[data-]YYYY[-moredata-]MM.DD", tsdb.NewTimeRange(from, to), func(indices []string) {
indexPatternScenario(intervalDaily, "[data-]YYYY[-moredata-]MM.DD", plugins.NewDataTimeRange(from, to), func(indices []string) {
So(indices, ShouldHaveLength, 1)
So(indices[0], ShouldEqual, "data-2018-moredata-05.15")
})
@@ -90,7 +89,7 @@ func TestIndexPattern(t *testing.T) {
Convey("Should return 01 week", func() {
from = fmt.Sprintf("%d", time.Date(2018, 1, 15, 17, 50, 0, 0, time.UTC).UnixNano()/int64(time.Millisecond))
to = fmt.Sprintf("%d", time.Date(2018, 1, 15, 17, 55, 0, 0, time.UTC).UnixNano()/int64(time.Millisecond))
indexPatternScenario(intervalWeekly, "[data-]GGGG.WW", tsdb.NewTimeRange(from, to), func(indices []string) {
indexPatternScenario(intervalWeekly, "[data-]GGGG.WW", plugins.NewDataTimeRange(from, to), func(indices []string) {
So(indices, ShouldHaveLength, 1)
So(indices[0], ShouldEqual, "data-2018.03")
})
@@ -276,7 +275,7 @@ func TestIndexPattern(t *testing.T) {
})
}
func indexPatternScenario(interval string, pattern string, timeRange *tsdb.TimeRange, fn func(indices []string)) {
func indexPatternScenario(interval string, pattern string, timeRange plugins.DataTimeRange, fn func(indices []string)) {
Convey(fmt.Sprintf("Index pattern (interval=%s, index=%s", interval, pattern), func() {
ip, err := newIndexPattern(interval, pattern)
So(err, ShouldBeNil)
pkg/tsdb/elasticsearch/client/models.go
+2 -3
View File
@@ -5,8 +5,7 @@ import (
"net/http"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/grafana/grafana/pkg/tsdb/interval"
)
type response struct {
@@ -33,7 +32,7 @@ type SearchDebugInfo struct {
// SearchRequest represents a search request
type SearchRequest struct {
Index string
Interval tsdb.Interval
Interval interval.Interval
Size int
Sort map[string]interface{}
Query *Query
pkg/tsdb/elasticsearch/client/search_request.go
+4 -4
View File
@@ -3,13 +3,13 @@ package es
import (
"strings"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/grafana/grafana/pkg/tsdb/interval"
)
// SearchRequestBuilder represents a builder which can build a search request
type SearchRequestBuilder struct {
version int
interval tsdb.Interval
interval interval.Interval
index string
size int
sort map[string]interface{}
@@ -19,7 +19,7 @@ type SearchRequestBuilder struct {
}
// NewSearchRequestBuilder create a new search request builder
func NewSearchRequestBuilder(version int, interval tsdb.Interval) *SearchRequestBuilder {
func NewSearchRequestBuilder(version int, interval interval.Interval) *SearchRequestBuilder {
builder := &SearchRequestBuilder{
version: version,
interval: interval,
@@ -131,7 +131,7 @@ func NewMultiSearchRequestBuilder(version int) *MultiSearchRequestBuilder {
}
// Search initiates and returns a new search request builder
func (m *MultiSearchRequestBuilder) Search(interval tsdb.Interval) *SearchRequestBuilder {
func (m *MultiSearchRequestBuilder) Search(interval interval.Interval) *SearchRequestBuilder {
b := NewSearchRequestBuilder(m.version, interval)
m.requestBuilders = append(m.requestBuilders, b)
return b
pkg/tsdb/elasticsearch/client/search_request_test.go
+6 -6
View File
@@ -6,7 +6,7 @@ import (
"time"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/grafana/grafana/pkg/tsdb/interval"
. "github.com/smartystreets/goconvey/convey"
)
@@ -15,7 +15,7 @@ func TestSearchRequest(t *testing.T) {
Convey("Test elasticsearch search request", t, func() {
timeField := "@timestamp"
Convey("Given new search request builder for es version 5", func() {
b := NewSearchRequestBuilder(5, tsdb.Interval{Value: 15 * time.Second, Text: "15s"})
b := NewSearchRequestBuilder(5, interval.Interval{Value: 15 * time.Second, Text: "15s"})
Convey("When building search request", func() {
sr, err := b.Build()
@@ -390,7 +390,7 @@ func TestSearchRequest(t *testing.T) {
})
Convey("Given new search request builder for es version 2", func() {
b := NewSearchRequestBuilder(2, tsdb.Interval{Value: 15 * time.Second, Text: "15s"})
b := NewSearchRequestBuilder(2, interval.Interval{Value: 15 * time.Second, Text: "15s"})
Convey("When adding doc value field", func() {
b.AddDocValueField(timeField)
@@ -449,7 +449,7 @@ func TestMultiSearchRequest(t *testing.T) {
b := NewMultiSearchRequestBuilder(0)
Convey("When adding one search request", func() {
b.Search(tsdb.Interval{Value: 15 * time.Second, Text: "15s"})
b.Search(interval.Interval{Value: 15 * time.Second, Text: "15s"})
Convey("When building search request should contain one search request", func() {
mr, err := b.Build()
@@ -459,8 +459,8 @@ func TestMultiSearchRequest(t *testing.T) {
})
Convey("When adding two search requests", func() {
b.Search(tsdb.Interval{Value: 15 * time.Second, Text: "15s"})
b.Search(tsdb.Interval{Value: 15 * time.Second, Text: "15s"})
b.Search(interval.Interval{Value: 15 * time.Second, Text: "15s"})
b.Search(interval.Interval{Value: 15 * time.Second, Text: "15s"})
Convey("When building search request should contain two search requests", func() {
mr, err := b.Build()