Merge remote-tracking branch 'upstream/master' into update-xorm

This commit is contained in:
Daniel Lee
2018-03-20 19:21:58 +01:00
139 changed files with 80643 additions and 943 deletions
+9
View File
@@ -1,6 +1,7 @@
package api
import (
"regexp"
"strings"
"github.com/grafana/grafana/pkg/bus"
@@ -21,6 +22,14 @@ func AdminGetSettings(c *m.ReqContext) {
if strings.Contains(keyName, "secret") || strings.Contains(keyName, "password") || (strings.Contains(keyName, "provider_config")) {
value = "************"
}
if strings.Contains(keyName, "url") {
var rgx = regexp.MustCompile(`.*:\/\/([^:]*):([^@]*)@.*?$`)
var subs = rgx.FindAllSubmatch([]byte(value), -1)
if subs != nil && len(subs[0]) == 3 {
value = strings.Replace(value, string(subs[0][1]), "******", 1)
value = strings.Replace(value, string(subs[0][2]), "******", 1)
}
}
jsonSec[keyName] = value
}
+1
View File
@@ -66,6 +66,7 @@ func (hs *HttpServer) registerRoutes() {
r.Get("/plugins/:id/page/:page", reqSignedIn, Index)
r.Get("/d/:uid/:slug", reqSignedIn, Index)
r.Get("/d/:uid", reqSignedIn, Index)
r.Get("/dashboard/db/:slug", reqSignedIn, redirectFromLegacyDashboardUrl, Index)
r.Get("/dashboard/script/*", reqSignedIn, Index)
r.Get("/dashboard-solo/snapshot/*", Index)
+3 -1
View File
@@ -72,7 +72,9 @@ func RenderToPng(params *RenderOpts) (string, error) {
localDomain = setting.HttpAddr
}
url := fmt.Sprintf("%s://%s:%s/%s", setting.Protocol, localDomain, setting.HttpPort, params.Path)
// &render=1 signals to the legacy redirect layer to
// avoid redirect these requests.
url := fmt.Sprintf("%s://%s:%s/%s&render=1", setting.Protocol, localDomain, setting.HttpPort, params.Path)
binPath, _ := filepath.Abs(filepath.Join(setting.PhantomDir, executable))
scriptPath, _ := filepath.Abs(filepath.Join(setting.PhantomDir, "render.js"))
+6
View File
@@ -6,6 +6,7 @@ import (
"github.com/grafana/grafana/pkg/bus"
m "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/setting"
"gopkg.in/macaron.v1"
)
@@ -36,9 +37,14 @@ func RedirectFromLegacyDashboardUrl() macaron.Handler {
func RedirectFromLegacyDashboardSoloUrl() macaron.Handler {
return func(c *m.ReqContext) {
slug := c.Params("slug")
renderRequest := c.QueryBool("render")
if slug != "" {
if url, err := getDashboardUrlBySlug(c.OrgId, slug); err == nil {
if renderRequest && strings.Contains(url, setting.AppSubUrl) {
url = strings.Replace(url, setting.AppSubUrl, "", 1)
}
url = strings.Replace(url, "/d/", "/d-solo/", 1)
url = fmt.Sprintf("%s?%s", url, c.Req.URL.RawQuery)
c.Redirect(url, 301)
+2
View File
@@ -19,6 +19,7 @@ const (
DS_PROMETHEUS = "prometheus"
DS_POSTGRES = "postgres"
DS_MYSQL = "mysql"
DS_MSSQL = "mssql"
DS_ACCESS_DIRECT = "direct"
DS_ACCESS_PROXY = "proxy"
)
@@ -68,6 +69,7 @@ var knownDatasourcePlugins map[string]bool = map[string]bool{
DS_OPENTSDB: true,
DS_POSTGRES: true,
DS_MYSQL: true,
DS_MSSQL: true,
"opennms": true,
"abhisant-druid-datasource": true,
"dalmatinerdb-datasource": true,
+1 -1
View File
@@ -80,7 +80,7 @@ func ImportDashboard(cmd *ImportDashboardCommand) error {
User: cmd.User,
}
savedDash, err := dashboards.NewService().SaveDashboard(dto)
savedDash, err := dashboards.NewService().ImportDashboard(dto)
if err != nil {
return err
+15
View File
@@ -74,6 +74,21 @@ func (e *DashAlertExtractor) GetAlertFromPanels(jsonWithPanels *simplejson.Json)
for _, panelObj := range jsonWithPanels.Get("panels").MustArray() {
panel := simplejson.NewFromAny(panelObj)
collapsedJson, collapsed := panel.CheckGet("collapsed")
// check if the panel is collapsed
if collapsed && collapsedJson.MustBool() {
// extract alerts from sub panels for collapsed panels
als, err := e.GetAlertFromPanels(panel)
if err != nil {
return nil, err
}
alerts = append(alerts, als...)
continue
}
jsonAlert, hasAlert := panel.CheckGet("alert")
if !hasAlert {
+26
View File
@@ -22,6 +22,7 @@ func TestAlertRuleExtraction(t *testing.T) {
defaultDs := &m.DataSource{Id: 12, OrgId: 1, Name: "I am default", IsDefault: true}
graphite2Ds := &m.DataSource{Id: 15, OrgId: 1, Name: "graphite2"}
influxDBDs := &m.DataSource{Id: 16, OrgId: 1, Name: "InfluxDB"}
prom := &m.DataSource{Id: 17, OrgId: 1, Name: "Prometheus"}
bus.AddHandler("test", func(query *m.GetDataSourcesQuery) error {
query.Result = []*m.DataSource{defaultDs, graphite2Ds}
@@ -38,6 +39,10 @@ func TestAlertRuleExtraction(t *testing.T) {
if query.Name == influxDBDs.Name {
query.Result = influxDBDs
}
if query.Name == prom.Name {
query.Result = prom
}
return nil
})
@@ -214,5 +219,26 @@ func TestAlertRuleExtraction(t *testing.T) {
}
})
})
Convey("Should be able to extract collapsed panels", func() {
json, err := ioutil.ReadFile("./test-data/collapsed-panels.json")
So(err, ShouldBeNil)
dashJson, err := simplejson.NewJson(json)
So(err, ShouldBeNil)
dash := m.NewDashboardFromJson(dashJson)
extractor := NewDashAlertExtractor(dash, 1)
alerts, err := extractor.GetAlerts()
Convey("Get rules without error", func() {
So(err, ShouldBeNil)
})
Convey("should be able to extract collapsed alerts", func() {
So(len(alerts), ShouldEqual, 4)
})
})
})
}
+62 -16
View File
@@ -46,25 +46,49 @@ type AlertmanagerNotifier struct {
}
func (this *AlertmanagerNotifier) ShouldNotify(evalContext *alerting.EvalContext) bool {
this.log.Debug("Should notify", "ruleId", evalContext.Rule.Id, "state", evalContext.Rule.State, "previousState", evalContext.PrevAlertState)
// Do not notify when we become OK for the first time.
if (evalContext.PrevAlertState == m.AlertStatePending) && (evalContext.Rule.State == m.AlertStateOK) {
return false
}
// Notify on Alerting -> OK to resolve before alertmanager timeout.
if (evalContext.PrevAlertState == m.AlertStateAlerting) && (evalContext.Rule.State == m.AlertStateOK) {
return true
}
return evalContext.Rule.State == m.AlertStateAlerting
}
func (this *AlertmanagerNotifier) Notify(evalContext *alerting.EvalContext) error {
func (this *AlertmanagerNotifier) createAlert(evalContext *alerting.EvalContext, match *alerting.EvalMatch, ruleUrl string) *simplejson.Json {
alertJSON := simplejson.New()
alertJSON.Set("startsAt", evalContext.StartTime.UTC().Format(time.RFC3339))
if evalContext.Rule.State == m.AlertStateOK {
alertJSON.Set("endsAt", time.Now().UTC().Format(time.RFC3339))
}
alertJSON.Set("generatorURL", ruleUrl)
alerts := make([]interface{}, 0)
for _, match := range evalContext.EvalMatches {
alertJSON := simplejson.New()
alertJSON.Set("startsAt", evalContext.StartTime.UTC().Format(time.RFC3339))
if ruleUrl, err := evalContext.GetRuleUrl(); err == nil {
alertJSON.Set("generatorURL", ruleUrl)
// Annotations (summary and description are very commonly used).
alertJSON.SetPath([]string{"annotations", "summary"}, evalContext.Rule.Name)
description := ""
if evalContext.Rule.Message != "" {
description += evalContext.Rule.Message
}
if evalContext.Error != nil {
if description != "" {
description += "\n"
}
description += "Error: " + evalContext.Error.Error()
}
if description != "" {
alertJSON.SetPath([]string{"annotations", "description"}, description)
}
if evalContext.ImagePublicUrl != "" {
alertJSON.SetPath([]string{"annotations", "image"}, evalContext.ImagePublicUrl)
}
if evalContext.Rule.Message != "" {
alertJSON.SetPath([]string{"annotations", "description"}, evalContext.Rule.Message)
}
tags := make(map[string]string)
// Labels (from metrics tags + mandatory alertname).
tags := make(map[string]string)
if match != nil {
if len(match.Tags) == 0 {
tags["metric"] = match.Metric
} else {
@@ -72,10 +96,32 @@ func (this *AlertmanagerNotifier) Notify(evalContext *alerting.EvalContext) erro
tags[k] = v
}
}
tags["alertname"] = evalContext.Rule.Name
alertJSON.Set("labels", tags)
}
tags["alertname"] = evalContext.Rule.Name
alertJSON.Set("labels", tags)
return alertJSON
}
alerts = append(alerts, alertJSON)
func (this *AlertmanagerNotifier) Notify(evalContext *alerting.EvalContext) error {
this.log.Info("Sending Alertmanager alert", "ruleId", evalContext.Rule.Id, "notification", this.Name)
ruleUrl, err := evalContext.GetRuleUrl()
if err != nil {
this.log.Error("Failed get rule link", "error", err)
return err
}
// Send one alert per matching series.
alerts := make([]interface{}, 0)
for _, match := range evalContext.EvalMatches {
alert := this.createAlert(evalContext, match, ruleUrl)
alerts = append(alerts, alert)
}
// This happens on ExecutionError or NoData
if len(alerts) == 0 {
alert := this.createAlert(evalContext, nil, ruleUrl)
alerts = append(alerts, alert)
}
bodyJSON := simplejson.NewFromAny(alerts)
+6
View File
@@ -27,15 +27,21 @@ func NewNotifierBase(id int64, isDefault bool, name, notifierType string, model
}
func defaultShouldNotify(context *alerting.EvalContext) bool {
// Only notify on state change.
if context.PrevAlertState == context.Rule.State {
return false
}
// Do not notify when we become OK for the first time.
if (context.PrevAlertState == m.AlertStatePending) && (context.Rule.State == m.AlertStateOK) {
return false
}
return true
}
func (n *NotifierBase) ShouldNotify(context *alerting.EvalContext) bool {
return defaultShouldNotify(context)
}
func (n *NotifierBase) GetType() string {
return n.Type
}
@@ -38,10 +38,6 @@ func NewDingDingNotifier(model *m.AlertNotification) (alerting.Notifier, error)
}, nil
}
func (this *DingDingNotifier) ShouldNotify(context *alerting.EvalContext) bool {
return defaultShouldNotify(context)
}
type DingDingNotifier struct {
NotifierBase
Url string
-4
View File
@@ -58,10 +58,6 @@ func NewEmailNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
}, nil
}
func (this *EmailNotifier) ShouldNotify(context *alerting.EvalContext) bool {
return defaultShouldNotify(context)
}
func (this *EmailNotifier) Notify(evalContext *alerting.EvalContext) error {
this.log.Info("Sending alert notification to", "addresses", this.Addresses)
@@ -75,10 +75,6 @@ type HipChatNotifier struct {
log log.Logger
}
func (this *HipChatNotifier) ShouldNotify(context *alerting.EvalContext) bool {
return defaultShouldNotify(context)
}
func (this *HipChatNotifier) Notify(evalContext *alerting.EvalContext) error {
this.log.Info("Executing hipchat notification", "ruleId", evalContext.Rule.Id, "notification", this.Name)
-4
View File
@@ -57,10 +57,6 @@ type KafkaNotifier struct {
log log.Logger
}
func (this *KafkaNotifier) ShouldNotify(context *alerting.EvalContext) bool {
return defaultShouldNotify(context)
}
func (this *KafkaNotifier) Notify(evalContext *alerting.EvalContext) error {
state := evalContext.Rule.State
-4
View File
@@ -51,10 +51,6 @@ type LineNotifier struct {
log log.Logger
}
func (this *LineNotifier) ShouldNotify(context *alerting.EvalContext) bool {
return defaultShouldNotify(context)
}
func (this *LineNotifier) Notify(evalContext *alerting.EvalContext) error {
this.log.Info("Executing line notification", "ruleId", evalContext.Rule.Id, "notification", this.Name)
@@ -72,10 +72,6 @@ type OpsGenieNotifier struct {
log log.Logger
}
func (this *OpsGenieNotifier) ShouldNotify(context *alerting.EvalContext) bool {
return defaultShouldNotify(context)
}
func (this *OpsGenieNotifier) Notify(evalContext *alerting.EvalContext) error {
var err error
@@ -65,10 +65,6 @@ type PagerdutyNotifier struct {
log log.Logger
}
func (this *PagerdutyNotifier) ShouldNotify(context *alerting.EvalContext) bool {
return defaultShouldNotify(context)
}
func (this *PagerdutyNotifier) Notify(evalContext *alerting.EvalContext) error {
if evalContext.Rule.State == m.AlertStateOK && !this.AutoResolve {
@@ -123,10 +123,6 @@ type PushoverNotifier struct {
log log.Logger
}
func (this *PushoverNotifier) ShouldNotify(context *alerting.EvalContext) bool {
return defaultShouldNotify(context)
}
func (this *PushoverNotifier) Notify(evalContext *alerting.EvalContext) error {
ruleUrl, err := evalContext.GetRuleUrl()
if err != nil {
-4
View File
@@ -71,10 +71,6 @@ type SensuNotifier struct {
log log.Logger
}
func (this *SensuNotifier) ShouldNotify(context *alerting.EvalContext) bool {
return defaultShouldNotify(context)
}
func (this *SensuNotifier) Notify(evalContext *alerting.EvalContext) error {
this.log.Info("Sending sensu result")
-4
View File
@@ -98,10 +98,6 @@ type SlackNotifier struct {
log log.Logger
}
func (this *SlackNotifier) ShouldNotify(context *alerting.EvalContext) bool {
return defaultShouldNotify(context)
}
func (this *SlackNotifier) Notify(evalContext *alerting.EvalContext) error {
this.log.Info("Executing slack notification", "ruleId", evalContext.Rule.Id, "notification", this.Name)
-4
View File
@@ -47,10 +47,6 @@ type TeamsNotifier struct {
log log.Logger
}
func (this *TeamsNotifier) ShouldNotify(context *alerting.EvalContext) bool {
return defaultShouldNotify(context)
}
func (this *TeamsNotifier) Notify(evalContext *alerting.EvalContext) error {
this.log.Info("Executing teams notification", "ruleId", evalContext.Rule.Id, "notification", this.Name)
+1 -4
View File
@@ -208,6 +208,7 @@ func generateImageCaption(evalContext *alerting.EvalContext, ruleUrl string, met
return message
}
func appendIfPossible(message string, extra string, sizeLimit int) string {
if len(extra)+len(message) <= sizeLimit {
return message + extra
@@ -216,10 +217,6 @@ func appendIfPossible(message string, extra string, sizeLimit int) string {
return message
}
func (this *TelegramNotifier) ShouldNotify(context *alerting.EvalContext) bool {
return defaultShouldNotify(context)
}
func (this *TelegramNotifier) Notify(evalContext *alerting.EvalContext) error {
var cmd *m.SendWebhookSync
if evalContext.ImagePublicUrl == "" && this.UploadImage == true {
@@ -114,10 +114,6 @@ func NewThreemaNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
}, nil
}
func (this *ThreemaNotifier) ShouldNotify(context *alerting.EvalContext) bool {
return defaultShouldNotify(context)
}
func (notifier *ThreemaNotifier) Notify(evalContext *alerting.EvalContext) error {
notifier.log.Info("Sending alert notification from", "threema_id", notifier.GatewayID)
notifier.log.Info("Sending alert notification to", "threema_id", notifier.RecipientID)
@@ -68,10 +68,6 @@ type VictoropsNotifier struct {
log log.Logger
}
func (this *VictoropsNotifier) ShouldNotify(context *alerting.EvalContext) bool {
return defaultShouldNotify(context)
}
// Notify sends notification to Victorops via POST to URL endpoint
func (this *VictoropsNotifier) Notify(evalContext *alerting.EvalContext) error {
this.log.Info("Executing victorops notification", "ruleId", evalContext.Rule.Id, "notification", this.Name)
@@ -65,10 +65,6 @@ type WebhookNotifier struct {
log log.Logger
}
func (this *WebhookNotifier) ShouldNotify(context *alerting.EvalContext) bool {
return defaultShouldNotify(context)
}
func (this *WebhookNotifier) Notify(evalContext *alerting.EvalContext) error {
this.log.Info("Sending webhook")
@@ -0,0 +1,597 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"id": 127,
"links": [],
"panels": [
{
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 9,
"title": "Row title",
"type": "row"
},
{
"alert": {
"conditions": [
{
"evaluator": {
"params": [
200
],
"type": "gt"
},
"operator": {
"type": "and"
},
"query": {
"params": [
"A",
"5m",
"now"
]
},
"reducer": {
"params": [],
"type": "avg"
},
"type": "query"
}
],
"executionErrorState": "alerting",
"frequency": "10s",
"handler": 1,
"name": "Panel Title alert",
"noDataState": "no_data",
"notifications": []
},
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"fill": 1,
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 1
},
"id": 10,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "go_goroutines",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{job}}",
"refId": "A"
}
],
"thresholds": [
{
"colorMode": "critical",
"fill": true,
"line": true,
"op": "gt",
"value": 200
}
],
"timeFrom": null,
"timeShift": null,
"title": "Panel Title",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
},
{
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 1
},
"id": 14,
"limit": 10,
"links": [],
"onlyAlertsOnDashboard": true,
"show": "current",
"sortOrder": 1,
"stateFilter": [],
"title": "Panel Title",
"type": "alertlist"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 10
},
"id": 6,
"panels": [
{
"alert": {
"conditions": [
{
"evaluator": {
"params": [
200
],
"type": "gt"
},
"operator": {
"type": "and"
},
"query": {
"params": [
"A",
"5m",
"now"
]
},
"reducer": {
"params": [],
"type": "avg"
},
"type": "query"
}
],
"executionErrorState": "alerting",
"frequency": "10s",
"handler": 1,
"name": "Panel 2 alert",
"noDataState": "no_data",
"notifications": []
},
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"fill": 1,
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 11
},
"id": 11,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "go_goroutines",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{job}}",
"refId": "A"
}
],
"thresholds": [
{
"colorMode": "critical",
"fill": true,
"line": true,
"op": "gt",
"value": 200
}
],
"timeFrom": null,
"timeShift": null,
"title": "Panel 2",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
},
{
"alert": {
"conditions": [
{
"evaluator": {
"params": [
200
],
"type": "gt"
},
"operator": {
"type": "and"
},
"query": {
"params": [
"A",
"5m",
"now"
]
},
"reducer": {
"params": [],
"type": "avg"
},
"type": "query"
}
],
"executionErrorState": "alerting",
"frequency": "10s",
"handler": 1,
"name": "Panel 4 alert",
"noDataState": "no_data",
"notifications": []
},
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"fill": 1,
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 11
},
"id": 15,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "go_goroutines",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{job}}",
"refId": "A"
}
],
"thresholds": [
{
"colorMode": "critical",
"fill": true,
"line": true,
"op": "gt",
"value": 200
}
],
"timeFrom": null,
"timeShift": null,
"title": "Panel 4",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
}
],
"title": "Row title",
"type": "row"
},
{
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 11
},
"id": 4,
"title": "Row title",
"type": "row"
},
{
"alert": {
"conditions": [
{
"evaluator": {
"params": [
200
],
"type": "gt"
},
"operator": {
"type": "and"
},
"query": {
"params": [
"A",
"5m",
"now"
]
},
"reducer": {
"params": [],
"type": "avg"
},
"type": "query"
}
],
"executionErrorState": "alerting",
"frequency": "10s",
"handler": 1,
"name": "Panel 3 alert",
"noDataState": "no_data",
"notifications": []
},
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"fill": 1,
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 12
},
"id": 12,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "go_goroutines",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{job}}",
"refId": "A"
}
],
"thresholds": [
{
"colorMode": "critical",
"fill": true,
"line": true,
"op": "gt",
"value": 200
}
],
"timeFrom": null,
"timeShift": null,
"title": "Panel 3",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
}
],
"schemaVersion": 16,
"style": "dark",
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
]
},
"timezone": "",
"title": "New dashboard Copy",
"uid": "6v5pg36zk",
"version": 17
}
@@ -13,6 +13,7 @@ import (
// DashboardService service for operating on dashboards
type DashboardService interface {
SaveDashboard(dto *SaveDashboardDTO) (*models.Dashboard, error)
ImportDashboard(dto *SaveDashboardDTO) (*models.Dashboard, error)
}
// DashboardProvisioningService service for operating on provisioned dashboards
@@ -214,6 +215,20 @@ func (dr *dashboardServiceImpl) SaveDashboard(dto *SaveDashboardDTO) (*models.Da
return cmd.Result, nil
}
func (dr *dashboardServiceImpl) ImportDashboard(dto *SaveDashboardDTO) (*models.Dashboard, error) {
cmd, err := dr.buildSaveDashboardCommand(dto, false)
if err != nil {
return nil, err
}
err = bus.Dispatch(cmd)
if err != nil {
return nil, err
}
return cmd.Result, nil
}
type FakeDashboardService struct {
SaveDashboardResult *models.Dashboard
SaveDashboardError error
@@ -230,6 +245,10 @@ func (s *FakeDashboardService) SaveDashboard(dto *SaveDashboardDTO) (*models.Das
return s.SaveDashboardResult, s.SaveDashboardError
}
func (s *FakeDashboardService) ImportDashboard(dto *SaveDashboardDTO) (*models.Dashboard, error) {
return s.SaveDashboard(dto)
}
func MockDashboardService(mock *FakeDashboardService) {
NewService = func() DashboardService {
return mock
+12
View File
@@ -106,6 +106,18 @@ type SessionWrapper struct {
}
func (s *SessionWrapper) Start(c *macaron.Context) error {
// See https://github.com/grafana/grafana/issues/11155 for details on why
// a recover and retry is needed
defer func() error {
if err := recover(); err != nil {
var retryErr error
s.session, retryErr = s.manager.Start(c)
return retryErr
}
return nil
}()
var err error
s.session, err = s.manager.Start(c)
return err
+2
View File
@@ -24,6 +24,8 @@ import (
"github.com/go-xorm/xorm"
_ "github.com/lib/pq"
_ "github.com/mattn/go-sqlite3"
_ "github.com/grafana/grafana/pkg/tsdb/mssql"
)
type DatabaseConfig struct {
+1
View File
@@ -14,6 +14,7 @@ type TestDB struct {
var TestDB_Sqlite3 = TestDB{DriverName: "sqlite3", ConnStr: ":memory:"}
var TestDB_Mysql = TestDB{DriverName: "mysql", ConnStr: "grafana:password@tcp(localhost:3306)/grafana_tests?collation=utf8mb4_unicode_ci"}
var TestDB_Postgres = TestDB{DriverName: "postgres", ConnStr: "user=grafanatest password=grafanatest host=localhost port=5432 dbname=grafanatest sslmode=disable"}
var TestDB_Mssql = TestDB{DriverName: "mssql", ConnStr: "server=localhost;port=1433;database=grafanatest;user id=grafana;password=Password!"}
func CleanDB(x *xorm.Engine) {
if x.DriverName() == "postgres" {
+129
View File
@@ -0,0 +1,129 @@
package mssql
import (
"fmt"
"regexp"
"strings"
"time"
"strconv"
"github.com/grafana/grafana/pkg/tsdb"
)
const rsIdentifier = `([_a-zA-Z0-9]+)`
const sExpr = `\$` + rsIdentifier + `\(([^\)]*)\)`
type MsSqlMacroEngine struct {
TimeRange *tsdb.TimeRange
Query *tsdb.Query
}
func NewMssqlMacroEngine() tsdb.SqlMacroEngine {
return &MsSqlMacroEngine{}
}
func (m *MsSqlMacroEngine) Interpolate(query *tsdb.Query, timeRange *tsdb.TimeRange, sql string) (string, error) {
m.TimeRange = timeRange
m.Query = query
rExp, _ := regexp.Compile(sExpr)
var macroError error
sql = replaceAllStringSubmatchFunc(rExp, sql, func(groups []string) string {
args := strings.Split(groups[2], ",")
for i, arg := range args {
args[i] = strings.Trim(arg, " ")
}
res, err := m.evaluateMacro(groups[1], args)
if err != nil && macroError == nil {
macroError = err
return "macro_error()"
}
return res
})
if macroError != nil {
return "", macroError
}
return sql, nil
}
func replaceAllStringSubmatchFunc(re *regexp.Regexp, str string, repl func([]string) string) string {
result := ""
lastIndex := 0
for _, v := range re.FindAllSubmatchIndex([]byte(str), -1) {
groups := []string{}
for i := 0; i < len(v); i += 2 {
groups = append(groups, str[v[i]:v[i+1]])
}
result += str[lastIndex:v[0]] + repl(groups)
lastIndex = v[1]
}
return result + str[lastIndex:]
}
func (m *MsSqlMacroEngine) evaluateMacro(name string, args []string) (string, error) {
switch name {
case "__time":
if len(args) == 0 {
return "", fmt.Errorf("missing time column argument for macro %v", name)
}
return fmt.Sprintf("%s AS time", args[0]), nil
case "__utcTime":
if len(args) == 0 {
return "", fmt.Errorf("missing time column argument for macro %v", name)
}
return fmt.Sprintf("DATEADD(second, DATEDIFF(second,GETDATE(),GETUTCDATE()), %s) AS time", args[0]), nil
case "__timeEpoch":
if len(args) == 0 {
return "", fmt.Errorf("missing time column argument for macro %v", name)
}
return fmt.Sprintf("DATEDIFF(second, {d '1970-01-01'}, DATEADD(second, DATEDIFF(second,GETDATE(),GETUTCDATE()), %s) ) AS time", args[0]), nil
case "__timeFilter":
if len(args) == 0 {
return "", fmt.Errorf("missing time column argument for macro %v", name)
}
return fmt.Sprintf("%s >= DATEADD(s, %d+DATEDIFF(second,GETUTCDATE(),GETDATE()), '1970-01-01') AND %s <= DATEADD(s, %d+DATEDIFF(second,GETUTCDATE(),GETDATE()), '1970-01-01')", args[0], uint64(m.TimeRange.GetFromAsMsEpoch()/1000), args[0], uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil
case "__timeFrom":
return fmt.Sprintf("DATEADD(second, %d+DATEDIFF(second,GETUTCDATE(),GETDATE()), '1970-01-01')", uint64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil
case "__timeTo":
return fmt.Sprintf("DATEADD(second, %d+DATEDIFF(second,GETUTCDATE(),GETDATE()), '1970-01-01')", uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil
case "__timeGroup":
if len(args) < 2 {
return "", fmt.Errorf("macro %v needs time column and interval", name)
}
interval, err := time.ParseDuration(strings.Trim(args[1], `'"`))
if err != nil {
return "", fmt.Errorf("error parsing interval %v", args[1])
}
if len(args) == 3 {
m.Query.Model.Set("fill", true)
m.Query.Model.Set("fillInterval", interval.Seconds())
if args[2] == "NULL" {
m.Query.Model.Set("fillNull", true)
} else {
floatVal, err := strconv.ParseFloat(args[2], 64)
if err != nil {
return "", fmt.Errorf("error parsing fill value %v", args[2])
}
m.Query.Model.Set("fillValue", floatVal)
}
}
return fmt.Sprintf("cast(cast(DATEDIFF(second, {d '1970-01-01'}, DATEADD(second, DATEDIFF(second,GETDATE(),GETUTCDATE()), %s))/%.0f as int)*%.0f as int)", args[0], interval.Seconds(), interval.Seconds()), nil
case "__unixEpochFilter":
if len(args) == 0 {
return "", fmt.Errorf("missing time column argument for macro %v", name)
}
return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], uint64(m.TimeRange.GetFromAsMsEpoch()/1000), args[0], uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil
case "__unixEpochFrom":
return fmt.Sprintf("%d", uint64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil
case "__unixEpochTo":
return fmt.Sprintf("%d", uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil
default:
return "", fmt.Errorf("Unknown macro %v", name)
}
}
+131
View File
@@ -0,0 +1,131 @@
package mssql
import (
"testing"
"time"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/tsdb"
. "github.com/smartystreets/goconvey/convey"
)
func TestMacroEngine(t *testing.T) {
Convey("MacroEngine", t, func() {
engine := &MsSqlMacroEngine{}
timeRange := &tsdb.TimeRange{From: "5m", To: "now"}
query := &tsdb.Query{
Model: simplejson.New(),
}
Convey("interpolate __time function", func() {
sql, err := engine.Interpolate(query, nil, "select $__time(time_column)")
So(err, ShouldBeNil)
So(sql, ShouldEqual, "select time_column AS time")
})
Convey("interpolate __utcTime function", func() {
sql, err := engine.Interpolate(query, nil, "select $__utcTime(time_column)")
So(err, ShouldBeNil)
So(sql, ShouldEqual, "select DATEADD(second, DATEDIFF(second,GETDATE(),GETUTCDATE()), time_column) AS time")
})
Convey("interpolate __timeEpoch function", func() {
sql, err := engine.Interpolate(query, nil, "select $__timeEpoch(time_column)")
So(err, ShouldBeNil)
So(sql, ShouldEqual, "select DATEDIFF(second, {d '1970-01-01'}, DATEADD(second, DATEDIFF(second,GETDATE(),GETUTCDATE()), time_column) ) AS time")
})
Convey("interpolate __timeEpoch function wrapped in aggregation", func() {
sql, err := engine.Interpolate(query, nil, "select min($__timeEpoch(time_column))")
So(err, ShouldBeNil)
So(sql, ShouldEqual, "select min(DATEDIFF(second, {d '1970-01-01'}, DATEADD(second, DATEDIFF(second,GETDATE(),GETUTCDATE()), time_column) ) AS time)")
})
Convey("interpolate __timeFilter function", func() {
sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)")
So(err, ShouldBeNil)
So(sql, ShouldEqual, "WHERE time_column >= DATEADD(s, 18446744066914186738+DATEDIFF(second,GETUTCDATE(),GETDATE()), '1970-01-01') AND time_column <= DATEADD(s, 18446744066914187038+DATEDIFF(second,GETUTCDATE(),GETDATE()), '1970-01-01')")
})
Convey("interpolate __timeGroup function", func() {
sql, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column,'5m')")
So(err, ShouldBeNil)
So(sql, ShouldEqual, "GROUP BY cast(cast(DATEDIFF(second, {d '1970-01-01'}, DATEADD(second, DATEDIFF(second,GETDATE(),GETUTCDATE()), time_column))/300 as int)*300 as int)")
})
Convey("interpolate __timeGroup function with spaces around arguments", func() {
sql, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column , '5m')")
So(err, ShouldBeNil)
So(sql, ShouldEqual, "GROUP BY cast(cast(DATEDIFF(second, {d '1970-01-01'}, DATEADD(second, DATEDIFF(second,GETDATE(),GETUTCDATE()), time_column))/300 as int)*300 as int)")
})
Convey("interpolate __timeGroup function with fill (value = NULL)", func() {
_, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column,'5m', NULL)")
fill := query.Model.Get("fill").MustBool()
fillNull := query.Model.Get("fillNull").MustBool()
fillInterval := query.Model.Get("fillInterval").MustInt()
So(err, ShouldBeNil)
So(fill, ShouldBeTrue)
So(fillNull, ShouldBeTrue)
So(fillInterval, ShouldEqual, 5*time.Minute.Seconds())
})
Convey("interpolate __timeGroup function with fill (value = float)", func() {
_, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column,'5m', 1.5)")
fill := query.Model.Get("fill").MustBool()
fillValue := query.Model.Get("fillValue").MustFloat64()
fillInterval := query.Model.Get("fillInterval").MustInt()
So(err, ShouldBeNil)
So(fill, ShouldBeTrue)
So(fillValue, ShouldEqual, 1.5)
So(fillInterval, ShouldEqual, 5*time.Minute.Seconds())
})
Convey("interpolate __timeFrom function", func() {
sql, err := engine.Interpolate(query, timeRange, "select $__timeFrom(time_column)")
So(err, ShouldBeNil)
So(sql, ShouldEqual, "select DATEADD(second, 18446744066914186738+DATEDIFF(second,GETUTCDATE(),GETDATE()), '1970-01-01')")
})
Convey("interpolate __timeTo function", func() {
sql, err := engine.Interpolate(query, timeRange, "select $__timeTo(time_column)")
So(err, ShouldBeNil)
So(sql, ShouldEqual, "select DATEADD(second, 18446744066914187038+DATEDIFF(second,GETUTCDATE(),GETDATE()), '1970-01-01')")
})
Convey("interpolate __unixEpochFilter function", func() {
sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFilter(18446744066914186738)")
So(err, ShouldBeNil)
So(sql, ShouldEqual, "select 18446744066914186738 >= 18446744066914186738 AND 18446744066914186738 <= 18446744066914187038")
})
Convey("interpolate __unixEpochFrom function", func() {
sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFrom()")
So(err, ShouldBeNil)
So(sql, ShouldEqual, "select 18446744066914186738")
})
Convey("interpolate __unixEpochTo function", func() {
sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochTo()")
So(err, ShouldBeNil)
So(sql, ShouldEqual, "select 18446744066914187038")
})
})
}
+328
View File
@@ -0,0 +1,328 @@
package mssql
import (
"container/list"
"context"
"database/sql"
"fmt"
"strconv"
"strings"
"time"
"math"
_ "github.com/denisenkom/go-mssqldb"
"github.com/go-xorm/core"
"github.com/grafana/grafana/pkg/components/null"
"github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/tsdb"
)
type MssqlQueryEndpoint struct {
sqlEngine tsdb.SqlEngine
log log.Logger
}
func init() {
tsdb.RegisterTsdbQueryEndpoint("mssql", NewMssqlQueryEndpoint)
}
func NewMssqlQueryEndpoint(datasource *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
endpoint := &MssqlQueryEndpoint{
log: log.New("tsdb.mssql"),
}
endpoint.sqlEngine = &tsdb.DefaultSqlEngine{
MacroEngine: NewMssqlMacroEngine(),
}
cnnstr := generateConnectionString(datasource)
endpoint.log.Debug("getEngine", "connection", cnnstr)
if err := endpoint.sqlEngine.InitEngine("mssql", datasource, cnnstr); err != nil {
return nil, err
}
return endpoint, nil
}
func generateConnectionString(datasource *models.DataSource) string {
password := ""
for key, value := range datasource.SecureJsonData.Decrypt() {
if key == "password" {
password = value
break
}
}
hostParts := strings.Split(datasource.Url, ":")
if len(hostParts) < 2 {
hostParts = append(hostParts, "1433")
}
server, port := hostParts[0], hostParts[1]
return fmt.Sprintf("server=%s;port=%s;database=%s;user id=%s;password=%s;",
server,
port,
datasource.Database,
datasource.User,
password,
)
}
// Query is the main function for the MssqlQueryEndpoint
func (e *MssqlQueryEndpoint) Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) {
return e.sqlEngine.Query(ctx, dsInfo, tsdbQuery, e.transformToTimeSeries, e.transformToTable)
}
func (e MssqlQueryEndpoint) transformToTable(query *tsdb.Query, rows *core.Rows, result *tsdb.QueryResult, tsdbQuery *tsdb.TsdbQuery) error {
columnNames, err := rows.Columns()
columnCount := len(columnNames)
if err != nil {
return err
}
rowLimit := 1000000
rowCount := 0
timeIndex := -1
table := &tsdb.Table{
Columns: make([]tsdb.TableColumn, columnCount),
Rows: make([]tsdb.RowValues, 0),
}
for i, name := range columnNames {
table.Columns[i].Text = name
// check if there is a column named time
switch name {
case "time":
timeIndex = i
}
}
columnTypes, err := rows.ColumnTypes()
if err != nil {
return err
}
for ; rows.Next(); rowCount++ {
if rowCount > rowLimit {
return fmt.Errorf("MsSQL query row limit exceeded, limit %d", rowLimit)
}
values, err := e.getTypedRowData(columnTypes, rows)
if err != nil {
return err
}
// convert column named time to unix timestamp to make
// native datetime mssql types work in annotation queries
if timeIndex != -1 {
switch value := values[timeIndex].(type) {
case time.Time:
values[timeIndex] = float64(value.Unix())
}
}
table.Rows = append(table.Rows, values)
}
result.Tables = append(result.Tables, table)
result.Meta.Set("rowCount", rowCount)
return nil
}
func (e MssqlQueryEndpoint) getTypedRowData(types []*sql.ColumnType, rows *core.Rows) (tsdb.RowValues, error) {
values := make([]interface{}, len(types))
valuePtrs := make([]interface{}, len(types))
for i, stype := range types {
e.log.Debug("type", "type", stype)
valuePtrs[i] = &values[i]
}
if err := rows.Scan(valuePtrs...); err != nil {
return nil, err
}
// convert types not handled by denisenkom/go-mssqldb
// unhandled types are returned as []byte
for i := 0; i < len(types); i++ {
if value, ok := values[i].([]byte); ok == true {
switch types[i].DatabaseTypeName() {
case "MONEY", "SMALLMONEY", "DECIMAL":
if v, err := strconv.ParseFloat(string(value), 64); err == nil {
values[i] = v
} else {
e.log.Debug("Rows", "Error converting numeric to float", value)
}
default:
e.log.Debug("Rows", "Unknown database type", types[i].DatabaseTypeName(), "value", value)
values[i] = string(value)
}
}
}
return values, nil
}
func (e MssqlQueryEndpoint) transformToTimeSeries(query *tsdb.Query, rows *core.Rows, result *tsdb.QueryResult, tsdbQuery *tsdb.TsdbQuery) error {
pointsBySeries := make(map[string]*tsdb.TimeSeries)
seriesByQueryOrder := list.New()
columnNames, err := rows.Columns()
if err != nil {
return err
}
columnTypes, err := rows.ColumnTypes()
if err != nil {
return err
}
rowLimit := 1000000
rowCount := 0
timeIndex := -1
metricIndex := -1
// check columns of resultset: a column named time is mandatory
// the first text column is treated as metric name unless a column named metric is present
for i, col := range columnNames {
switch col {
case "time":
timeIndex = i
case "metric":
metricIndex = i
default:
if metricIndex == -1 {
switch columnTypes[i].DatabaseTypeName() {
case "VARCHAR", "CHAR", "NVARCHAR", "NCHAR":
metricIndex = i
}
}
}
}
if timeIndex == -1 {
return fmt.Errorf("Found no column named time")
}
fillMissing := query.Model.Get("fill").MustBool(false)
var fillInterval float64
fillValue := null.Float{}
if fillMissing {
fillInterval = query.Model.Get("fillInterval").MustFloat64() * 1000
if query.Model.Get("fillNull").MustBool(false) == false {
fillValue.Float64 = query.Model.Get("fillValue").MustFloat64()
fillValue.Valid = true
}
}
for rows.Next() {
var timestamp float64
var value null.Float
var metric string
if rowCount > rowLimit {
return fmt.Errorf("MSSQL query row limit exceeded, limit %d", rowLimit)
}
values, err := e.getTypedRowData(columnTypes, rows)
if err != nil {
return err
}
switch columnValue := values[timeIndex].(type) {
case int64:
timestamp = float64(columnValue * 1000)
case float64:
timestamp = columnValue * 1000
case time.Time:
timestamp = (float64(columnValue.Unix()) * 1000) + float64(columnValue.Nanosecond()/1e6) // in case someone is trying to map times beyond 2262 :D
default:
return fmt.Errorf("Invalid type for column time, must be of type timestamp or unix timestamp")
}
if metricIndex >= 0 {
if columnValue, ok := values[metricIndex].(string); ok == true {
metric = columnValue
} else {
return fmt.Errorf("Column metric must be of type CHAR, VARCHAR, NCHAR or NVARCHAR. metric column name: %s type: %s but datatype is %T", columnNames[metricIndex], columnTypes[metricIndex].DatabaseTypeName(), values[metricIndex])
}
}
for i, col := range columnNames {
if i == timeIndex || i == metricIndex {
continue
}
switch columnValue := values[i].(type) {
case int64:
value = null.FloatFrom(float64(columnValue))
case float64:
value = null.FloatFrom(columnValue)
case nil:
value.Valid = false
default:
return fmt.Errorf("Value column must have numeric datatype, column: %s type: %T value: %v", col, columnValue, columnValue)
}
if metricIndex == -1 {
metric = col
}
series, exist := pointsBySeries[metric]
if exist == false {
series = &tsdb.TimeSeries{Name: metric}
pointsBySeries[metric] = series
seriesByQueryOrder.PushBack(metric)
}
if fillMissing {
var intervalStart float64
if exist == false {
intervalStart = float64(tsdbQuery.TimeRange.MustGetFrom().UnixNano() / 1e6)
} else {
intervalStart = series.Points[len(series.Points)-1][1].Float64 + fillInterval
}
// align interval start
intervalStart = math.Floor(intervalStart/fillInterval) * fillInterval
for i := intervalStart; i < timestamp; i += fillInterval {
series.Points = append(series.Points, tsdb.TimePoint{fillValue, null.FloatFrom(i)})
rowCount++
}
}
series.Points = append(series.Points, tsdb.TimePoint{value, null.FloatFrom(timestamp)})
e.log.Debug("Rows", "metric", metric, "time", timestamp, "value", value)
}
}
for elem := seriesByQueryOrder.Front(); elem != nil; elem = elem.Next() {
key := elem.Value.(string)
result.Series = append(result.Series, pointsBySeries[key])
if fillMissing {
series := pointsBySeries[key]
// fill in values from last fetched value till interval end
intervalStart := series.Points[len(series.Points)-1][1].Float64
intervalEnd := float64(tsdbQuery.TimeRange.MustGetTo().UnixNano() / 1e6)
// align interval start
intervalStart = math.Floor(intervalStart/fillInterval) * fillInterval
for i := intervalStart + fillInterval; i < intervalEnd; i += fillInterval {
series.Points = append(series.Points, tsdb.TimePoint{fillValue, null.FloatFrom(i)})
rowCount++
}
}
}
result.Meta.Set("rowCount", rowCount)
return nil
}
+686
View File
@@ -0,0 +1,686 @@
package mssql
import (
"fmt"
"math/rand"
"strings"
"testing"
"time"
"github.com/go-xorm/xorm"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/services/sqlstore/sqlutil"
"github.com/grafana/grafana/pkg/tsdb"
. "github.com/smartystreets/goconvey/convey"
)
// To run this test, remove the Skip from SkipConvey
// and set up a MSSQL db named grafanatest and a user/password grafana/Password!
// Use the docker/blocks/mssql_tests/docker-compose.yaml to spin up a
// preconfigured MSSQL server suitable for running these tests.
// If needed, change the variable below to the IP address of the database.
var serverIP string = "localhost"
func TestMSSQL(t *testing.T) {
SkipConvey("MSSQL", t, func() {
x := InitMSSQLTestDB(t)
endpoint := &MssqlQueryEndpoint{
sqlEngine: &tsdb.DefaultSqlEngine{
MacroEngine: NewMssqlMacroEngine(),
XormEngine: x,
},
log: log.New("tsdb.mssql"),
}
sess := x.NewSession()
defer sess.Close()
fromStart := time.Date(2018, 3, 15, 13, 0, 0, 0, time.UTC)
Convey("Given a table with different native data types", func() {
sql := `
IF OBJECT_ID('dbo.[mssql_types]', 'U') IS NOT NULL
DROP TABLE dbo.[mssql_types]
CREATE TABLE [mssql_types] (
c_bit bit,
c_tinyint tinyint,
c_smallint smallint,
c_int int,
c_bigint bigint,
c_money money,
c_smallmoney smallmoney,
c_numeric numeric(10,5),
c_real real,
c_decimal decimal(10,2),
c_float float,
c_char char(10),
c_varchar varchar(10),
c_text text,
c_nchar nchar(12),
c_nvarchar nvarchar(12),
c_ntext ntext,
c_datetime datetime,
c_datetime2 datetime2,
c_smalldatetime smalldatetime,
c_date date,
c_time time,
c_datetimeoffset datetimeoffset
)
`
_, err := sess.Exec(sql)
So(err, ShouldBeNil)
dt := time.Date(2018, 3, 14, 21, 20, 6, 527e6, time.UTC)
dtFormat := "2006-01-02 15:04:05.999999999"
d := dt.Format(dtFormat)
dt2 := time.Date(2018, 3, 14, 21, 20, 6, 8896406e2, time.UTC)
dt2Format := "2006-01-02 15:04:05.999999999 -07:00"
d2 := dt2.Format(dt2Format)
sql = fmt.Sprintf(`
INSERT INTO [mssql_types]
SELECT
1, 5, 20020, 980300, 1420070400, '$20000.15', '£2.15', 12345.12,
1.11, 2.22, 3.33,
'char10', 'varchar10', 'text',
N'☺nchar12☺', N'☺nvarchar12☺', N'☺text☺',
CAST('%s' AS DATETIME), CAST('%s' AS DATETIME2), CAST('%s' AS SMALLDATETIME), CAST('%s' AS DATE), CAST('%s' AS TIME), SWITCHOFFSET(CAST('%s' AS DATETIMEOFFSET), '-07:00')
`, d, d2, d, d, d, d2)
_, err = sess.Exec(sql)
So(err, ShouldBeNil)
Convey("When doing a table query should map MSSQL column types to Go types", func() {
query := &tsdb.TsdbQuery{
Queries: []*tsdb.Query{
{
Model: simplejson.NewFromAny(map[string]interface{}{
"rawSql": "SELECT * FROM mssql_types",
"format": "table",
}),
RefId: "A",
},
},
}
resp, err := endpoint.Query(nil, nil, query)
queryResult := resp.Results["A"]
So(err, ShouldBeNil)
column := queryResult.Tables[0].Rows[0]
So(column[0].(bool), ShouldEqual, true)
So(column[1].(int64), ShouldEqual, 5)
So(column[2].(int64), ShouldEqual, 20020)
So(column[3].(int64), ShouldEqual, 980300)
So(column[4].(int64), ShouldEqual, 1420070400)
So(column[5].(float64), ShouldEqual, 20000.15)
So(column[6].(float64), ShouldEqual, 2.15)
So(column[7].(float64), ShouldEqual, 12345.12)
So(column[8].(float64), ShouldEqual, 1.1100000143051147)
So(column[9].(float64), ShouldEqual, 2.22)
So(column[10].(float64), ShouldEqual, 3.33)
So(column[11].(string), ShouldEqual, "char10 ")
So(column[12].(string), ShouldEqual, "varchar10")
So(column[13].(string), ShouldEqual, "text")
So(column[14].(string), ShouldEqual, "☺nchar12☺ ")
So(column[15].(string), ShouldEqual, "☺nvarchar12☺")
So(column[16].(string), ShouldEqual, "☺text☺")
So(column[17].(time.Time), ShouldEqual, dt)
So(column[18].(time.Time), ShouldEqual, dt2)
So(column[19].(time.Time), ShouldEqual, dt.Truncate(time.Minute))
So(column[20].(time.Time), ShouldEqual, dt.Truncate(24*time.Hour))
So(column[21].(time.Time), ShouldEqual, time.Date(1, 1, 1, dt.Hour(), dt.Minute(), dt.Second(), dt.Nanosecond(), time.UTC))
So(column[22].(time.Time), ShouldEqual, dt2.In(time.FixedZone("UTC", int(-7*time.Hour))))
})
})
Convey("Given a table with metrics that lacks data for some series ", func() {
sql := `
IF OBJECT_ID('dbo.[metric]', 'U') IS NOT NULL
DROP TABLE dbo.[metric]
CREATE TABLE [metric] (
time datetime,
value int
)
`
_, err := sess.Exec(sql)
So(err, ShouldBeNil)
type metric struct {
Time time.Time
Value int64
}
series := []*metric{}
firstRange := genTimeRangeByInterval(fromStart, 10*time.Minute, 10*time.Second)
secondRange := genTimeRangeByInterval(fromStart.Add(20*time.Minute), 10*time.Minute, 10*time.Second)
for _, t := range firstRange {
series = append(series, &metric{
Time: t,
Value: 15,
})
}
for _, t := range secondRange {
series = append(series, &metric{
Time: t,
Value: 20,
})
}
dtFormat := "2006-01-02 15:04:05.999999999"
for _, s := range series {
sql = fmt.Sprintf(`
INSERT INTO metric (time, value)
VALUES(CAST('%s' AS DATETIME), %d)
`, s.Time.Format(dtFormat), s.Value)
_, err = sess.Exec(sql)
So(err, ShouldBeNil)
}
Convey("When doing a metric query using timeGroup", func() {
query := &tsdb.TsdbQuery{
Queries: []*tsdb.Query{
{
Model: simplejson.NewFromAny(map[string]interface{}{
"rawSql": "SELECT $__timeGroup(time, '5m') AS time, avg(value) as value FROM metric GROUP BY $__timeGroup(time, '5m') ORDER BY 1",
"format": "time_series",
}),
RefId: "A",
},
},
}
resp, err := endpoint.Query(nil, nil, query)
queryResult := resp.Results["A"]
So(err, ShouldBeNil)
So(queryResult.Error, ShouldBeNil)
points := queryResult.Series[0].Points
So(len(points), ShouldEqual, 4)
actualValueFirst := points[0][0].Float64
actualTimeFirst := time.Unix(int64(points[0][1].Float64)/1000, 0)
So(actualValueFirst, ShouldEqual, 15)
So(actualTimeFirst, ShouldEqual, fromStart)
actualValueLast := points[3][0].Float64
actualTimeLast := time.Unix(int64(points[3][1].Float64)/1000, 0)
So(actualValueLast, ShouldEqual, 20)
So(actualTimeLast, ShouldEqual, fromStart.Add(25*time.Minute))
})
Convey("When doing a metric query using timeGroup with NULL fill enabled", func() {
query := &tsdb.TsdbQuery{
Queries: []*tsdb.Query{
{
Model: simplejson.NewFromAny(map[string]interface{}{
"rawSql": "SELECT $__timeGroup(time, '5m', NULL) AS time, avg(value) as value FROM metric GROUP BY $__timeGroup(time, '5m') ORDER BY 1",
"format": "time_series",
}),
RefId: "A",
},
},
TimeRange: &tsdb.TimeRange{
From: fmt.Sprintf("%v", fromStart.Unix()*1000),
To: fmt.Sprintf("%v", fromStart.Add(34*time.Minute).Unix()*1000),
},
}
resp, err := endpoint.Query(nil, nil, query)
queryResult := resp.Results["A"]
So(err, ShouldBeNil)
So(queryResult.Error, ShouldBeNil)
points := queryResult.Series[0].Points
So(len(points), ShouldEqual, 7)
actualValueFirst := points[0][0].Float64
actualTimeFirst := time.Unix(int64(points[0][1].Float64)/1000, 0)
So(actualValueFirst, ShouldEqual, 15)
So(actualTimeFirst, ShouldEqual, fromStart)
actualNullPoint := points[3][0]
actualNullTime := time.Unix(int64(points[3][1].Float64)/1000, 0)
So(actualNullPoint.Valid, ShouldBeFalse)
So(actualNullTime, ShouldEqual, fromStart.Add(15*time.Minute))
actualValueLast := points[5][0].Float64
actualTimeLast := time.Unix(int64(points[5][1].Float64)/1000, 0)
So(actualValueLast, ShouldEqual, 20)
So(actualTimeLast, ShouldEqual, fromStart.Add(25*time.Minute))
actualLastNullPoint := points[6][0]
actualLastNullTime := time.Unix(int64(points[6][1].Float64)/1000, 0)
So(actualLastNullPoint.Valid, ShouldBeFalse)
So(actualLastNullTime, ShouldEqual, fromStart.Add(30*time.Minute))
})
Convey("When doing a metric query using timeGroup with float fill enabled", func() {
query := &tsdb.TsdbQuery{
Queries: []*tsdb.Query{
{
Model: simplejson.NewFromAny(map[string]interface{}{
"rawSql": "SELECT $__timeGroup(time, '5m', 1.5) AS time, avg(value) as value FROM metric GROUP BY $__timeGroup(time, '5m') ORDER BY 1",
"format": "time_series",
}),
RefId: "A",
},
},
TimeRange: &tsdb.TimeRange{
From: fmt.Sprintf("%v", fromStart.Unix()*1000),
To: fmt.Sprintf("%v", fromStart.Add(34*time.Minute).Unix()*1000),
},
}
resp, err := endpoint.Query(nil, nil, query)
queryResult := resp.Results["A"]
So(err, ShouldBeNil)
So(queryResult.Error, ShouldBeNil)
points := queryResult.Series[0].Points
So(points[6][0].Float64, ShouldEqual, 1.5)
})
})
Convey("Given a table with metrics having multiple values and measurements", func() {
sql := `
IF OBJECT_ID('dbo.[metric_values]', 'U') IS NOT NULL
DROP TABLE dbo.[metric_values]
CREATE TABLE [metric_values] (
time datetime,
measurement nvarchar(100),
valueOne int,
valueTwo int,
)
`
_, err := sess.Exec(sql)
So(err, ShouldBeNil)
type metricValues struct {
Time time.Time
Measurement string
ValueOne int64
ValueTwo int64
}
rand.Seed(time.Now().Unix())
rnd := func(min, max int64) int64 {
return rand.Int63n(max-min) + min
}
series := []*metricValues{}
for _, t := range genTimeRangeByInterval(fromStart.Add(-30*time.Minute), 90*time.Minute, 5*time.Minute) {
series = append(series, &metricValues{
Time: t,
Measurement: "Metric A",
ValueOne: rnd(0, 100),
ValueTwo: rnd(0, 100),
})
series = append(series, &metricValues{
Time: t,
Measurement: "Metric B",
ValueOne: rnd(0, 100),
ValueTwo: rnd(0, 100),
})
}
dtFormat := "2006-01-02 15:04:05"
for _, s := range series {
sql = fmt.Sprintf(`
INSERT metric_values (time, measurement, valueOne, valueTwo)
VALUES(CAST('%s' AS DATETIME), '%s', %d, %d)
`, s.Time.Format(dtFormat), s.Measurement, s.ValueOne, s.ValueTwo)
_, err = sess.Exec(sql)
So(err, ShouldBeNil)
}
Convey("When doing a metric query grouping by time and select metric column should return correct series", func() {
query := &tsdb.TsdbQuery{
Queries: []*tsdb.Query{
{
Model: simplejson.NewFromAny(map[string]interface{}{
"rawSql": "SELECT $__timeEpoch(time), measurement + ' - value one' as metric, valueOne FROM metric_values ORDER BY 1",
"format": "time_series",
}),
RefId: "A",
},
},
}
resp, err := endpoint.Query(nil, nil, query)
queryResult := resp.Results["A"]
So(err, ShouldBeNil)
So(queryResult.Error, ShouldBeNil)
So(len(queryResult.Series), ShouldEqual, 2)
So(queryResult.Series[0].Name, ShouldEqual, "Metric A - value one")
So(queryResult.Series[1].Name, ShouldEqual, "Metric B - value one")
})
Convey("When doing a metric query grouping by time should return correct series", func() {
query := &tsdb.TsdbQuery{
Queries: []*tsdb.Query{
{
Model: simplejson.NewFromAny(map[string]interface{}{
"rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values ORDER BY 1",
"format": "time_series",
}),
RefId: "A",
},
},
}
resp, err := endpoint.Query(nil, nil, query)
queryResult := resp.Results["A"]
So(err, ShouldBeNil)
So(queryResult.Error, ShouldBeNil)
So(len(queryResult.Series), ShouldEqual, 2)
So(queryResult.Series[0].Name, ShouldEqual, "valueOne")
So(queryResult.Series[1].Name, ShouldEqual, "valueTwo")
})
Convey("Given a stored procedure that takes @from and @to in epoch time", func() {
sql := `
IF object_id('sp_test_epoch') IS NOT NULL
DROP PROCEDURE sp_test_epoch
`
_, err := sess.Exec(sql)
So(err, ShouldBeNil)
sql = `
CREATE PROCEDURE sp_test_epoch(
@from int,
@to int
) AS
BEGIN
SELECT
cast(cast(DATEDIFF(second, {d '1970-01-01'}, DATEADD(second, DATEDIFF(second,GETDATE(),GETUTCDATE()), time))/600 as int)*600 as int) as time,
measurement + ' - value one' as metric,
avg(valueOne) as value
FROM
metric_values
WHERE
time >= DATEADD(s, @from, '1970-01-01') AND time <= DATEADD(s, @to, '1970-01-01')
GROUP BY
cast(cast(DATEDIFF(second, {d '1970-01-01'}, DATEADD(second, DATEDIFF(second,GETDATE(),GETUTCDATE()), time))/600 as int)*600 as int),
measurement
UNION ALL
SELECT
cast(cast(DATEDIFF(second, {d '1970-01-01'}, DATEADD(second, DATEDIFF(second,GETDATE(),GETUTCDATE()), time))/600 as int)*600 as int) as time,
measurement + ' - value two' as metric,
avg(valueTwo) as value
FROM
metric_values
WHERE
time >= DATEADD(s, @from, '1970-01-01') AND time <= DATEADD(s, @to, '1970-01-01')
GROUP BY
cast(cast(DATEDIFF(second, {d '1970-01-01'}, DATEADD(second, DATEDIFF(second,GETDATE(),GETUTCDATE()), time))/600 as int)*600 as int),
measurement
ORDER BY 1
END
`
_, err = sess.Exec(sql)
So(err, ShouldBeNil)
Convey("When doing a metric query using stored procedure should return correct result", func() {
query := &tsdb.TsdbQuery{
Queries: []*tsdb.Query{
{
Model: simplejson.NewFromAny(map[string]interface{}{
"rawSql": `DECLARE
@from int = $__unixEpochFrom(),
@to int = $__unixEpochTo()
EXEC dbo.sp_test_epoch @from, @to`,
"format": "time_series",
}),
RefId: "A",
},
},
TimeRange: &tsdb.TimeRange{
From: "1521117000000",
To: "1521122100000",
},
}
resp, err := endpoint.Query(nil, nil, query)
queryResult := resp.Results["A"]
So(err, ShouldBeNil)
So(queryResult.Error, ShouldBeNil)
So(len(queryResult.Series), ShouldEqual, 4)
So(queryResult.Series[0].Name, ShouldEqual, "Metric A - value one")
So(queryResult.Series[1].Name, ShouldEqual, "Metric B - value one")
So(queryResult.Series[2].Name, ShouldEqual, "Metric A - value two")
So(queryResult.Series[3].Name, ShouldEqual, "Metric B - value two")
})
})
Convey("Given a stored procedure that takes @from and @to in datetime", func() {
sql := `
IF object_id('sp_test_datetime') IS NOT NULL
DROP PROCEDURE sp_test_datetime
`
_, err := sess.Exec(sql)
So(err, ShouldBeNil)
sql = `
CREATE PROCEDURE sp_test_datetime(
@from datetime,
@to datetime
) AS
BEGIN
SELECT
cast(cast(DATEDIFF(second, {d '1970-01-01'}, time)/600 as int)*600 as int) as time,
measurement + ' - value one' as metric,
avg(valueOne) as value
FROM
metric_values
WHERE
time >= @from AND time <= @to
GROUP BY
cast(cast(DATEDIFF(second, {d '1970-01-01'}, time)/600 as int)*600 as int),
measurement
UNION ALL
SELECT
cast(cast(DATEDIFF(second, {d '1970-01-01'}, time)/600 as int)*600 as int) as time,
measurement + ' - value two' as metric,
avg(valueTwo) as value
FROM
metric_values
WHERE
time >= @from AND time <= @to
GROUP BY
cast(cast(DATEDIFF(second, {d '1970-01-01'}, time)/600 as int)*600 as int),
measurement
ORDER BY 1
END
`
_, err = sess.Exec(sql)
So(err, ShouldBeNil)
Convey("When doing a metric query using stored procedure should return correct result", func() {
query := &tsdb.TsdbQuery{
Queries: []*tsdb.Query{
{
Model: simplejson.NewFromAny(map[string]interface{}{
"rawSql": `DECLARE
@from int = $__unixEpochFrom(),
@to int = $__unixEpochTo()
EXEC dbo.sp_test_epoch @from, @to`,
"format": "time_series",
}),
RefId: "A",
},
},
TimeRange: &tsdb.TimeRange{
From: "1521117000000",
To: "1521122100000",
},
}
resp, err := endpoint.Query(nil, nil, query)
queryResult := resp.Results["A"]
So(err, ShouldBeNil)
So(queryResult.Error, ShouldBeNil)
So(len(queryResult.Series), ShouldEqual, 4)
So(queryResult.Series[0].Name, ShouldEqual, "Metric A - value one")
So(queryResult.Series[1].Name, ShouldEqual, "Metric B - value one")
So(queryResult.Series[2].Name, ShouldEqual, "Metric A - value two")
So(queryResult.Series[3].Name, ShouldEqual, "Metric B - value two")
})
})
})
Convey("Given a table with event data", func() {
sql := `
IF OBJECT_ID('dbo.[event]', 'U') IS NOT NULL
DROP TABLE dbo.[event]
CREATE TABLE [event] (
time_sec bigint,
description nvarchar(100),
tags nvarchar(100),
)
`
_, err := sess.Exec(sql)
So(err, ShouldBeNil)
type event struct {
TimeSec int64
Description string
Tags string
}
events := []*event{}
for _, t := range genTimeRangeByInterval(fromStart.Add(-20*time.Minute), 60*time.Minute, 25*time.Minute) {
events = append(events, &event{
TimeSec: t.Unix(),
Description: "Someone deployed something",
Tags: "deploy",
})
events = append(events, &event{
TimeSec: t.Add(5 * time.Minute).Unix(),
Description: "New support ticket registered",
Tags: "ticket",
})
}
for _, e := range events {
sql = fmt.Sprintf(`
INSERT [event] (time_sec, description, tags)
VALUES(%d, '%s', '%s')
`, e.TimeSec, e.Description, e.Tags)
_, err = sess.Exec(sql)
So(err, ShouldBeNil)
}
Convey("When doing an annotation query of deploy events should return expected result", func() {
query := &tsdb.TsdbQuery{
Queries: []*tsdb.Query{
{
Model: simplejson.NewFromAny(map[string]interface{}{
"rawSql": "SELECT time_sec as time, description as [text], tags FROM [event] WHERE $__unixEpochFilter(time_sec) AND tags='deploy' ORDER BY 1 ASC",
"format": "table",
}),
RefId: "Deploys",
},
},
TimeRange: &tsdb.TimeRange{
From: fmt.Sprintf("%v", fromStart.Add(-20*time.Minute).Unix()*1000),
To: fmt.Sprintf("%v", fromStart.Add(40*time.Minute).Unix()*1000),
},
}
resp, err := endpoint.Query(nil, nil, query)
queryResult := resp.Results["Deploys"]
So(err, ShouldBeNil)
So(len(queryResult.Tables[0].Rows), ShouldEqual, 3)
})
Convey("When doing an annotation query of ticket events should return expected result", func() {
query := &tsdb.TsdbQuery{
Queries: []*tsdb.Query{
{
Model: simplejson.NewFromAny(map[string]interface{}{
"rawSql": "SELECT time_sec as time, description as [text], tags FROM [event] WHERE $__unixEpochFilter(time_sec) AND tags='ticket' ORDER BY 1 ASC",
"format": "table",
}),
RefId: "Tickets",
},
},
TimeRange: &tsdb.TimeRange{
From: fmt.Sprintf("%v", fromStart.Add(-20*time.Minute).Unix()*1000),
To: fmt.Sprintf("%v", fromStart.Add(40*time.Minute).Unix()*1000),
},
}
resp, err := endpoint.Query(nil, nil, query)
queryResult := resp.Results["Tickets"]
So(err, ShouldBeNil)
So(len(queryResult.Tables[0].Rows), ShouldEqual, 3)
})
})
})
}
func InitMSSQLTestDB(t *testing.T) *xorm.Engine {
x, err := xorm.NewEngine(sqlutil.TestDB_Mssql.DriverName, strings.Replace(sqlutil.TestDB_Mssql.ConnStr, "localhost", serverIP, 1))
// x.ShowSQL()
if err != nil {
t.Fatalf("Failed to init mssql db %v", err)
}
sqlutil.CleanDB(x)
return x
}
func genTimeRangeByInterval(from time.Time, duration time.Duration, interval time.Duration) []time.Time {
durationSec := int64(duration.Seconds())
intervalSec := int64(interval.Seconds())
timeRange := []time.Time{}
for i := int64(0); i < durationSec; i += intervalSec {
timeRange = append(timeRange, from)
from = from.Add(time.Duration(int64(time.Second) * intervalSec))
}
return timeRange
}