public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(testdata): lots of work on new test data data source and scenarios

Browse Source
This commit is contained in:
Torkel Ödegaard
2016-09-27 18:17:39 +02:00
parent ade8aa5b92
commit 3ecd96e682
16 changed files with 257 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/tsdb/models.go
+8 -9
View File
@@ -4,7 +4,6 @@ import "github.com/grafana/grafana/pkg/components/simplejson"
type Query struct {
RefId string
Query string
Model *simplejson.Json
Depends []string
DataSource *DataSourceInfo
@@ -17,13 +16,13 @@ type Query struct {
type QuerySlice []*Query
type Request struct {
TimeRange TimeRange
TimeRange *TimeRange
Queries QuerySlice
}
type Response struct {
BatchTimings []*BatchTiming
Results map[string]*QueryResult
BatchTimings []*BatchTiming `json:"timings"`
Results map[string]*QueryResult `json:"results"`
}
type DataSourceInfo struct {
@@ -50,14 +49,14 @@ type BatchResult struct {
}
type QueryResult struct {
Error error
RefId string
Series TimeSeriesSlice
Error error `json:"error"`
RefId string `json:"refId"`
Series TimeSeriesSlice `json:"series"`
}
type TimeSeries struct {
Name string `json:"target"`
Points [][2]*float64 `json:"datapoints"`
Name string `json:"name"`
Points [][2]*float64 `json:"points"`
}
type TimeSeriesSlice []*TimeSeries
pkg/tsdb/prometheus/prometheus.go
+3 -3
View File
@@ -10,8 +10,8 @@ import (
"github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/prometheus/client_golang/api/prometheus"
"golang.org/x/net/context"
pmodel "github.com/prometheus/common/model"
"golang.org/x/net/context"
)
type PrometheusExecutor struct {
@@ -111,12 +111,12 @@ func parseQuery(queries tsdb.QuerySlice, queryContext *tsdb.QueryContext) (*Prom
return nil, err
}
start, err := queryContext.TimeRange.FromTime()
start, err := queryContext.TimeRange.ParseFrom()
if err != nil {
return nil, err
}
end, err := queryContext.TimeRange.ToTime()
end, err := queryContext.TimeRange.ParseTo()
if err != nil {
return nil, err
}
pkg/tsdb/query_context.go
+2 -2
View File
@@ -3,7 +3,7 @@ package tsdb
import "sync"
type QueryContext struct {
TimeRange TimeRange
TimeRange *TimeRange
Queries QuerySlice
Results map[string]*QueryResult
ResultsChan chan *BatchResult
@@ -11,7 +11,7 @@ type QueryContext struct {
BatchWaits sync.WaitGroup
}
func NewQueryContext(queries QuerySlice, timeRange TimeRange) *QueryContext {
func NewQueryContext(queries QuerySlice, timeRange *TimeRange) *QueryContext {
return &QueryContext{
TimeRange: timeRange,
Queries: queries,
pkg/tsdb/testdata/scenarios.go
Vendored
+98
View File
@@ -0,0 +1,98 @@
package testdata
import (
"math/rand"
"time"
"github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/tsdb"
)
type ScenarioHandler func(query *tsdb.Query, context *tsdb.QueryContext) *tsdb.QueryResult
type Scenario struct {
Id string `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
Handler ScenarioHandler `json:"-"`
}
var ScenarioRegistry map[string]*Scenario
func init() {
ScenarioRegistry = make(map[string]*Scenario)
logger := log.New("tsdb.testdata")
registerScenario(&Scenario{
Id: "random_walk",
Name: "Random Walk",
Handler: func(query *tsdb.Query, context *tsdb.QueryContext) *tsdb.QueryResult {
timeWalkerMs := context.TimeRange.MustGetFrom().Unix() * 1000
to := context.TimeRange.MustGetTo().Unix() * 1000
series := newSeriesForQuery(query)
points := make([][2]*float64, 0)
walker := rand.Float64() * 100
for i := int64(0); i < 10000 && timeWalkerMs < to; i++ {
timestamp := float64(timeWalkerMs)
val := float64(walker)
points = append(points, [2]*float64{&val, &timestamp})
walker += rand.Float64() - 0.5
timeWalkerMs += query.IntervalMs
}
series.Points = points
queryRes := &tsdb.QueryResult{}
queryRes.Series = append(queryRes.Series, series)
return queryRes
},
})
registerScenario(&Scenario{
Id: "no_data_points",
Name: "No Data Points",
Handler: func(query *tsdb.Query, context *tsdb.QueryContext) *tsdb.QueryResult {
return &tsdb.QueryResult{
Series: make(tsdb.TimeSeriesSlice, 0),
}
},
})
registerScenario(&Scenario{
Id: "datapoints_outside_range",
Name: "Datapoints Outside Range",
Handler: func(query *tsdb.Query, context *tsdb.QueryContext) *tsdb.QueryResult {
queryRes := &tsdb.QueryResult{}
series := newSeriesForQuery(query)
outsideTime := context.TimeRange.MustGetFrom().Add(-1*time.Hour).Unix() * 1000
timestamp := float64(outsideTime)
logger.Info("time", "from", timestamp)
val := float64(10)
series.Points = append(series.Points, [2]*float64{&val, &timestamp})
queryRes.Series = append(queryRes.Series, series)
return queryRes
},
})
}
func registerScenario(scenario *Scenario) {
ScenarioRegistry[scenario.Id] = scenario
}
func newSeriesForQuery(query *tsdb.Query) *tsdb.TimeSeries {
alias := query.Model.Get("alias").MustString("")
if alias == "" {
alias = query.RefId + "-series"
}
return &tsdb.TimeSeries{Name: alias}
}
pkg/tsdb/testdata/testdata.go
Vendored
+12 -27
View File
@@ -1,17 +1,20 @@
package testdata
import (
"math/rand"
"github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/tsdb"
)
type TestDataExecutor struct {
*tsdb.DataSourceInfo
log log.Logger
}
func NewTestDataExecutor(dsInfo *tsdb.DataSourceInfo) tsdb.Executor {
return &TestDataExecutor{dsInfo}
return &TestDataExecutor{
DataSourceInfo: dsInfo,
log: log.New("tsdb.testdata"),
}
}
func init() {
@@ -22,33 +25,15 @@ func (e *TestDataExecutor) Execute(queries tsdb.QuerySlice, context *tsdb.QueryC
result := &tsdb.BatchResult{}
result.QueryResults = make(map[string]*tsdb.QueryResult)
from, _ := context.TimeRange.FromTime()
to, _ := context.TimeRange.ToTime()
queryRes := &tsdb.QueryResult{}
for _, query := range queries {
// scenario := query.Model.Get("scenario").MustString("random_walk")
series := &tsdb.TimeSeries{Name: "test-series-0"}
stepInSeconds := (to.Unix() - from.Unix()) / query.MaxDataPoints
points := make([][2]*float64, 0)
walker := rand.Float64() * 100
time := from.Unix()
for i := int64(0); i < query.MaxDataPoints; i++ {
timestamp := float64(time)
val := float64(walker)
points = append(points, [2]*float64{&val, &timestamp})
walker += rand.Float64() - 0.5
time += stepInSeconds
scenarioId := query.Model.Get("scenarioId").MustString("random_walk")
if scenario, exist := ScenarioRegistry[scenarioId]; exist {
result.QueryResults[query.RefId] = scenario.Handler(query, context)
result.QueryResults[query.RefId].RefId = query.RefId
} else {
e.log.Error("Scenario not found", "scenarioId", scenarioId)
}
series.Points = points
queryRes.Series = append(queryRes.Series, series)
}
result.QueryResults["A"] = queryRes
return result
}
pkg/tsdb/time_range.go
+33 -9
View File
@@ -7,8 +7,8 @@ import (
"time"
)
func NewTimeRange(from, to string) TimeRange {
return TimeRange{
func NewTimeRange(from, to string) *TimeRange {
return &TimeRange{
From: from,
To: to,
Now: time.Now(),
@@ -21,13 +21,37 @@ type TimeRange struct {
Now time.Time
}
func (tr TimeRange) FromTime() (time.Time, error) {
if val, err := strconv.ParseInt(tr.From, 10, 64); err == nil {
return time.Unix(val, 0), nil
func (tr *TimeRange) MustGetFrom() time.Time {
if res, err := tr.ParseFrom(); err != nil {
return time.Unix(0, 0)
} else {
return res
}
}
func (tr *TimeRange) MustGetTo() time.Time {
if res, err := tr.ParseTo(); err != nil {
return time.Unix(0, 0)
} else {
return res
}
}
func tryParseUnixMsEpoch(val string) (time.Time, bool) {
if val, err := strconv.ParseInt(val, 10, 64); err == nil {
seconds := val / 1000
nano := (val - seconds*1000) * 1000000
return time.Unix(seconds, nano), true
}
return time.Time{}, false
}
func (tr *TimeRange) ParseFrom() (time.Time, error) {
if res, ok := tryParseUnixMsEpoch(tr.From); ok {
return res, nil
}
fromRaw := strings.Replace(tr.From, "now-", "", 1)
diff, err := time.ParseDuration("-" + fromRaw)
if err != nil {
return time.Time{}, err
@@ -36,7 +60,7 @@ func (tr TimeRange) FromTime() (time.Time, error) {
return tr.Now.Add(diff), nil
}
func (tr TimeRange) ToTime() (time.Time, error) {
func (tr *TimeRange) ParseTo() (time.Time, error) {
if tr.To == "now" {
return tr.Now, nil
} else if strings.HasPrefix(tr.To, "now-") {
@@ -50,8 +74,8 @@ func (tr TimeRange) ToTime() (time.Time, error) {
return tr.Now.Add(diff), nil
}
if val, err := strconv.ParseInt(tr.To, 10, 64); err == nil {
return time.Unix(val, 0), nil
if res, ok := tryParseUnixMsEpoch(tr.To); ok {
return res, nil
}
return time.Time{}, fmt.Errorf("cannot parse to value %s", tr.To)
pkg/tsdb/time_range_test.go
+10 -10
View File
@@ -23,13 +23,13 @@ func TestTimeRange(t *testing.T) {
fiveMinAgo, _ := time.ParseDuration("-5m")
expected := now.Add(fiveMinAgo)
res, err := tr.FromTime()
res, err := tr.ParseFrom()
So(err, ShouldBeNil)
So(res.Unix(), ShouldEqual, expected.Unix())
})
Convey("now ", func() {
res, err := tr.ToTime()
res, err := tr.ParseTo()
So(err, ShouldBeNil)
So(res.Unix(), ShouldEqual, now.Unix())
})
@@ -46,7 +46,7 @@ func TestTimeRange(t *testing.T) {
fiveHourAgo, _ := time.ParseDuration("-5h")
expected := now.Add(fiveHourAgo)
res, err := tr.FromTime()
res, err := tr.ParseFrom()
So(err, ShouldBeNil)
So(res.Unix(), ShouldEqual, expected.Unix())
})
@@ -54,7 +54,7 @@ func TestTimeRange(t *testing.T) {
Convey("now-10m ", func() {
fiveMinAgo, _ := time.ParseDuration("-10m")
expected := now.Add(fiveMinAgo)
res, err := tr.ToTime()
res, err := tr.ParseTo()
So(err, ShouldBeNil)
So(res.Unix(), ShouldEqual, expected.Unix())
})
@@ -68,13 +68,13 @@ func TestTimeRange(t *testing.T) {
Now: now,
}
res, err := tr.FromTime()
res, err := tr.ParseFrom()
So(err, ShouldBeNil)
So(res.Unix(), ShouldEqual, 1474973725473)
So(res.UnixNano()/int64(time.Millisecond), ShouldEqual, 1474973725473)
res, err = tr.ToTime()
res, err = tr.ParseTo()
So(err, ShouldBeNil)
So(res.Unix(), ShouldEqual, 1474975757930)
So(res.UnixNano()/int64(time.Millisecond), ShouldEqual, 1474975757930)
})
Convey("Cannot parse asdf", func() {
@@ -85,10 +85,10 @@ func TestTimeRange(t *testing.T) {
Now: now,
}
_, err = tr.FromTime()
_, err = tr.ParseFrom()
So(err, ShouldNotBeNil)
_, err = tr.ToTime()
_, err = tr.ParseTo()
So(err, ShouldNotBeNil)
})
})
pkg/tsdb/tsdb_test.go
+15 -15
View File
@@ -14,9 +14,9 @@ func TestMetricQuery(t *testing.T) {
Convey("Given 3 queries for 2 data sources", func() {
request := &Request{
Queries: QuerySlice{
{RefId: "A", Query: "asd", DataSource: &DataSourceInfo{Id: 1}},
{RefId: "B", Query: "asd", DataSource: &DataSourceInfo{Id: 1}},
{RefId: "C", Query: "asd", DataSource: &DataSourceInfo{Id: 2}},
{RefId: "A", DataSource: &DataSourceInfo{Id: 1}},
{RefId: "B", DataSource: &DataSourceInfo{Id: 1}},
{RefId: "C", DataSource: &DataSourceInfo{Id: 2}},
},
}
@@ -31,9 +31,9 @@ func TestMetricQuery(t *testing.T) {
Convey("Given query 2 depends on query 1", func() {
request := &Request{
Queries: QuerySlice{
{RefId: "A", Query: "asd", DataSource: &DataSourceInfo{Id: 1}},
{RefId: "B", Query: "asd", DataSource: &DataSourceInfo{Id: 2}},
{RefId: "C", Query: "#A / #B", DataSource: &DataSourceInfo{Id: 3}, Depends: []string{"A", "B"}},
{RefId: "A", DataSource: &DataSourceInfo{Id: 1}},
{RefId: "B", DataSource: &DataSourceInfo{Id: 2}},
{RefId: "C", DataSource: &DataSourceInfo{Id: 3}, Depends: []string{"A", "B"}},
},
}
@@ -55,7 +55,7 @@ func TestMetricQuery(t *testing.T) {
Convey("When executing request with one query", t, func() {
req := &Request{
Queries: QuerySlice{
{RefId: "A", Query: "asd", DataSource: &DataSourceInfo{Id: 1, PluginId: "test"}},
{RefId: "A", DataSource: &DataSourceInfo{Id: 1, PluginId: "test"}},
},
}
@@ -74,8 +74,8 @@ func TestMetricQuery(t *testing.T) {
Convey("When executing one request with two queries from same data source", t, func() {
req := &Request{
Queries: QuerySlice{
{RefId: "A", Query: "asd", DataSource: &DataSourceInfo{Id: 1, PluginId: "test"}},
{RefId: "B", Query: "asd", DataSource: &DataSourceInfo{Id: 1, PluginId: "test"}},
{RefId: "A", DataSource: &DataSourceInfo{Id: 1, PluginId: "test"}},
{RefId: "B", DataSource: &DataSourceInfo{Id: 1, PluginId: "test"}},
},
}
@@ -100,9 +100,9 @@ func TestMetricQuery(t *testing.T) {
Convey("When executing one request with three queries from different datasources", t, func() {
req := &Request{
Queries: QuerySlice{
{RefId: "A", Query: "asd", DataSource: &DataSourceInfo{Id: 1, PluginId: "test"}},
{RefId: "B", Query: "asd", DataSource: &DataSourceInfo{Id: 1, PluginId: "test"}},
{RefId: "C", Query: "asd", DataSource: &DataSourceInfo{Id: 2, PluginId: "test"}},
{RefId: "A", DataSource: &DataSourceInfo{Id: 1, PluginId: "test"}},
{RefId: "B", DataSource: &DataSourceInfo{Id: 1, PluginId: "test"}},
{RefId: "C", DataSource: &DataSourceInfo{Id: 2, PluginId: "test"}},
},
}
@@ -117,7 +117,7 @@ func TestMetricQuery(t *testing.T) {
Convey("When query uses data source of unknown type", t, func() {
req := &Request{
Queries: QuerySlice{
{RefId: "A", Query: "asd", DataSource: &DataSourceInfo{Id: 1, PluginId: "asdasdas"}},
{RefId: "A", DataSource: &DataSourceInfo{Id: 1, PluginId: "asdasdas"}},
},
}
@@ -129,10 +129,10 @@ func TestMetricQuery(t *testing.T) {
req := &Request{
Queries: QuerySlice{
{
RefId: "A", Query: "asd", DataSource: &DataSourceInfo{Id: 1, PluginId: "test"},
RefId: "A", DataSource: &DataSourceInfo{Id: 1, PluginId: "test"},
},
{
RefId: "B", Query: "#A / 2", DataSource: &DataSourceInfo{Id: 2, PluginId: "test"}, Depends: []string{"A"},
RefId: "B", DataSource: &DataSourceInfo{Id: 2, PluginId: "test"}, Depends: []string{"A"},
},
},
}