diff --git a/.gitignore b/.gitignore index 721a2a71ad4..1ab7068c96a 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ public/css/*.min.css *.swp .idea/ *.iml +*.tmp .vscode/ /data/* diff --git a/CHANGELOG.md b/CHANGELOG.md index 128abbb1662..2c98ec2c091 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ * **Units**: New number format: Scientific notation [#7781](https://github.com/grafana/grafana/issues/7781) thx [@cadnce](https://github.com/cadnce) * **Oauth**: Add common type for oauth authorization errors [#6428](https://github.com/grafana/grafana/issues/6428) thx [@amenzhinsky](https://github.com/amenzhinsky) * **Templating**: Data source variable now supports multi value and panel repeats [#7030](https://github.com/grafana/grafana/issues/7030) thx [@mtanda](https://github.com/mtanda) +* **Telegram**: Telegram alert is not sending metric and legend. [#8110](https://github.com/grafana/grafana/issues/8110), thx [@bashgeek](https://github.com/bashgeek) ## Fixes * **Table Panel**: Fixed annotation display in table panel, [#8023](https://github.com/grafana/grafana/issues/8023) diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 00000000000..260c4151442 --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,31 @@ +# Roadmap (2017-04-23) + +This roadmap is a tentative plan for the core development team. Things change constantly as PRs come in and priorities change. +But it will give you an idea of our current vision and plan. + +### Short term (1-4 months) + + - New Heatmap Panel (Implemented and available in master) + - Support for MySQL & Postgres as data sources (Work started and a alpha version for MySQL is available in master) + - User Groups & Dashboard folders with ACLs (work started, not yet completed, https://github.com/grafana/grafana/issues/1611#issuecomment-287742633) + - Improve new user UX + - Improve docs + - Support for alerting for Elasticsearch (can be tested in [branch](https://github.com/grafana/grafana/tree/alerting-elasticsearch) but needs more work) + - Graph annotations (create from grafana, region annotations, better annotation viz) + - Improve alerting (clustering, silence rules) + +### Long term + +- Improved dashboard panel layout engine (to make it easier and enable more flexible layouts) +- Backend plugins to support more Auth options, Alerting data sources & notifications +- Universial time series transformations for any data source (meta queries) +- Reporting +- Web socket & live data streams +- Migrate to Angular2 + + +### Outside contributions +We know this is being worked on right now by contributors (and we hope to merge it when it's ready). + +- Dashboard revisions (be able to revert dashboard changes) +- Clustering for alert engine (load distribution) diff --git a/docker/blocks/mysql/fig b/docker/blocks/mysql/fig index 731d0fbbdc5..24cb47b61a7 100644 --- a/docker/blocks/mysql/fig +++ b/docker/blocks/mysql/fig @@ -10,3 +10,5 @@ mysql: volumes: - /etc/localtime:/etc/localtime:ro - /etc/timezone:/etc/timezone:ro + command: [mysqld, --character-set-server=utf8mb4, --collation-server=utf8mb4_unicode_ci, --innodb_monitor_enable=all] + diff --git a/docker/blocks/mysql_opendata/Dockerfile b/docker/blocks/mysql_opendata/Dockerfile new file mode 100644 index 00000000000..c1086d19b82 --- /dev/null +++ b/docker/blocks/mysql_opendata/Dockerfile @@ -0,0 +1,20 @@ +## MySQL with Open Data Set from NYC Open Data (https://data.cityofnewyork.us) + +FROM mysql:latest + +ENV MYSQL_DATABASE="testdata" \ + MYSQL_ROOT_PASSWORD="rootpass" \ + MYSQL_USER="grafana" \ + MYSQL_PASSWORD="password" + +# Install requirement (wget) +RUN apt-get update && apt-get install -y wget && apt-get install unzip + +# Fetch NYC Data Set +RUN wget https://data.cityofnewyork.us/download/57g5-etyj/application%2Fzip -O /tmp/data.zip && \ + unzip -j /tmp/data.zip 311_Service_Requests_from_2015.csv -d /var/lib/mysql-files && \ + rm /tmp/data.zip + +ADD import_csv.sql /docker-entrypoint-initdb.d/ + +EXPOSE 3306 diff --git a/docker/blocks/mysql_opendata/fig b/docker/blocks/mysql_opendata/fig new file mode 100644 index 00000000000..a374fbd0931 --- /dev/null +++ b/docker/blocks/mysql_opendata/fig @@ -0,0 +1,9 @@ +mysql_opendata: + build: blocks/mysql_opendata + environment: + MYSQL_ROOT_PASSWORD: rootpass + MYSQL_DATABASE: testdata + MYSQL_USER: grafana + MYSQL_PASSWORD: password + ports: + - "3307:3306" diff --git a/docker/blocks/mysql_opendata/import_csv.sql b/docker/blocks/mysql_opendata/import_csv.sql new file mode 100644 index 00000000000..d77361f3b9d --- /dev/null +++ b/docker/blocks/mysql_opendata/import_csv.sql @@ -0,0 +1,80 @@ +use testdata; +DROP TABLE IF EXISTS `nyc_open_data`; +CREATE TABLE IF NOT EXISTS `nyc_open_data` ( + UniqueKey bigint(255), + `CreatedDate` varchar(255), + `ClosedDate` varchar(255), + Agency varchar(255), + AgencyName varchar(255), + ComplaintType varchar(255), + Descriptor varchar(255), + LocationType varchar(255), + IncidentZip varchar(255), + IncidentAddress varchar(255), + StreetName varchar(255), + CrossStreet1 varchar(255), + CrossStreet2 varchar(255), + IntersectionStreet1 varchar(255), + IntersectionStreet2 varchar(255), + AddressType varchar(255), + City varchar(255), + Landmark varchar(255), + FacilityType varchar(255), + Status varchar(255), + `DueDate` varchar(255), + ResolutionDescription varchar(2048), + `ResolutionActionUpdatedDate` varchar(255), + CommunityBoard varchar(255), + Borough varchar(255), + XCoordinateStatePlane varchar(255), + YCoordinateStatePlane varchar(255), + ParkFacilityName varchar(255), + ParkBorough varchar(255), + SchoolName varchar(255), + SchoolNumber varchar(255), + SchoolRegion varchar(255), + SchoolCode varchar(255), + SchoolPhoneNumber varchar(255), + SchoolAddress varchar(255), + SchoolCity varchar(255), + SchoolState varchar(255), + SchoolZip varchar(255), + SchoolNotFound varchar(255), + SchoolOrCitywideComplaint varchar(255), + VehicleType varchar(255), + TaxiCompanyBorough varchar(255), + TaxiPickUpLocation varchar(255), + BridgeHighwayName varchar(255), + BridgeHighwayDirection varchar(255), + RoadRamp varchar(255), + BridgeHighwaySegment varchar(255), + GarageLotName varchar(255), + FerryDirection varchar(255), + FerryTerminalName varchar(255), + Latitude varchar(255), + Longitude varchar(255), + Location varchar(255) +); +LOAD DATA INFILE '/var/lib/mysql-files/311_Service_Requests_from_2015.csv' INTO TABLE nyc_open_data FIELDS OPTIONALLY ENCLOSED BY '"' TERMINATED BY ',' IGNORE 1 LINES; +UPDATE nyc_open_data SET CreatedDate = STR_TO_DATE(CreatedDate, '%m/%d/%Y %r') WHERE CreatedDate <> ''; +UPDATE nyc_open_data SET ClosedDate = STR_TO_DATE(ClosedDate, '%m/%d/%Y %r') WHERE ClosedDate <> ''; +UPDATE nyc_open_data SET DueDate = STR_TO_DATE(DueDate, '%m/%d/%Y %r') WHERE DueDate <> ''; +UPDATE nyc_open_data SET ResolutionActionUpdatedDate = STR_TO_DATE(ResolutionActionUpdatedDate, '%m/%d/%Y %r') WHERE ResolutionActionUpdatedDate <> ''; + +UPDATE nyc_open_data SET CreatedDate=null WHERE CreatedDate = ''; +UPDATE nyc_open_data SET ClosedDate=null WHERE ClosedDate = ''; +UPDATE nyc_open_data SET DueDate=null WHERE DueDate = ''; +UPDATE nyc_open_data SET ResolutionActionUpdatedDate=null WHERE ResolutionActionUpdatedDate = ''; + +ALTER TABLE nyc_open_data modify CreatedDate datetime NULL; +ALTER TABLE nyc_open_data modify ClosedDate datetime NULL; +ALTER TABLE nyc_open_data modify DueDate datetime NULL; +ALTER TABLE nyc_open_data modify ResolutionActionUpdatedDate datetime NULL; + +ALTER TABLE `nyc_open_data` ADD INDEX `IX_ComplaintType` (`ComplaintType`); +ALTER TABLE `nyc_open_data` ADD INDEX `IX_CreatedDate` (`CreatedDate`); +ALTER TABLE `nyc_open_data` ADD INDEX `IX_LocationType` (`LocationType`); +ALTER TABLE `nyc_open_data` ADD INDEX `IX_AgencyName` (`AgencyName`); +ALTER TABLE `nyc_open_data` ADD INDEX `IX_City` (`City`); + +SYSTEM rm /var/lib/mysql-files/311_Service_Requests_from_2015.csv diff --git a/docs/sources/administration/cli.md b/docs/sources/administration/cli.md index 8c7755506e8..645f75ab412 100644 --- a/docs/sources/administration/cli.md +++ b/docs/sources/administration/cli.md @@ -27,6 +27,24 @@ To show all admin commands: ### Reset admin password -You can reset the password for the admin user using the CLI. +You can reset the password for the admin user using the CLI. The use case for this command is when you have lost the admin password. `grafana-cli admin reset-admin-password ...` + +If running the command returns this error: + +> Could not find config defaults, make sure homepath command line parameter is set or working directory is homepath + +then there are two flags that can be used to set homepath and the config file path. + +`grafana-cli admin reset-admin-password --homepath "/usr/share/grafana" newpass` + +If you have not lost the admin password then it is better to set in the Grafana UI. If you need to set the password in a script then the [Grafana API](http://docs.grafana.org/http_api/user/#change-password) can be used. Here is an example with curl using basic auth: + +``` +curl -X PUT -H "Content-Type: application/json" -d '{ + "oldPassword": "admin", + "newPassword": "newpass", + "confirmNew": "newpass" +}' http://admin:admin@:3000/api/user/password +``` diff --git a/docs/sources/http_api/alerting.md b/docs/sources/http_api/alerting.md index 22aadb80f58..0a422e10f4d 100644 --- a/docs/sources/http_api/alerting.md +++ b/docs/sources/http_api/alerting.md @@ -202,7 +202,7 @@ This API can also be used to create, update and delete alert notifications. **Example Request**: - DELETE /api/alerts-notifications/1 HTTP/1.1 + DELETE /api/alert-notifications/1 HTTP/1.1 Accept: application/json Content-Type: application/json Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk diff --git a/docs/sources/http_api/auth.md b/docs/sources/http_api/auth.md index ef62f271715..d8ded124ac5 100644 --- a/docs/sources/http_api/auth.md +++ b/docs/sources/http_api/auth.md @@ -41,3 +41,80 @@ You use the token in all requests in the `Authorization` header, like this: Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk The `Authorization` header value should be `Bearer `. + +# Auth HTTP resources / actions + +## Api Keys + +`GET /api/auth/keys` + +**Example Request**: + + GET /api/auth/keys HTTP/1.1 + Accept: application/json + Content-Type: application/json + Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk + +**Example Response**: + + HTTP/1.1 200 + Content-Type: application/json + + [ + { + "id": 3, + "name": "API", + "role": "Admin" + }, + { + "id": 1, + "name": "TestAdmin", + "role": "Admin" + } + ] + +## Create API Key + +`POST /api/auth/keys` + +**Example Request**: + + POST /api/auth/keys HTTP/1.1 + Accept: application/json + Content-Type: application/json + Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk + + { + "name": "mykey", + "role": "Admin" + } + +JSON Body schema: + +- **name** – The key name +- **role** – Sets the access level/Grafana Role for the key. Can be one of the following values: `Viewer`, `Editor`, `Read Only Editor` or `Admin`. + +**Example Response**: + + HTTP/1.1 200 + Content-Type: application/json + + {"name":"mykey","key":"eyJrIjoiWHZiSWd3NzdCYUZnNUtibE9obUpESmE3bzJYNDRIc0UiLCJuIjoibXlrZXkiLCJpZCI6MX1="} + +## Delete API Key + +`DELETE /api/auth/keys/:id` + +**Example Request**: + + DELETE /api/auth/keys/3 HTTP/1.1 + Accept: application/json + Content-Type: application/json + Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk + +**Example Response**: + + HTTP/1.1 200 + Content-Type: application/json + + {"message":"API key deleted"} diff --git a/package.json b/package.json index 326c813dc93..c72b93ddd1a 100644 --- a/package.json +++ b/package.json @@ -76,8 +76,8 @@ "systemjs-builder": "^0.15.34", "tether": "^1.4.0", "tether-drop": "https://github.com/torkelo/drop", - "tslint": "^4.5.1", - "typescript": "^2.1.4", + "tslint": "^5.1.0", + "typescript": "^2.2.2", "virtual-scroll": "^1.1.1" } } diff --git a/pkg/api/metrics.go b/pkg/api/metrics.go index d10491950ef..abd6527431a 100644 --- a/pkg/api/metrics.go +++ b/pkg/api/metrics.go @@ -50,13 +50,16 @@ func QueryMetrics(c *middleware.Context, reqDto dtos.MetricRequest) Response { return ApiError(500, "Metric request error", err) } + statusCode := 200 for _, res := range resp.Results { if res.Error != nil { res.ErrorString = res.Error.Error() + resp.Message = res.ErrorString + statusCode = 500 } } - return Json(200, &resp) + return Json(statusCode, &resp) } // GET /api/tsdb/testdata/scenarios diff --git a/pkg/cmd/grafana-cli/commands/commands.go b/pkg/cmd/grafana-cli/commands/commands.go index 8b2ecfcf7f5..d8f01bbdcab 100644 --- a/pkg/cmd/grafana-cli/commands/commands.go +++ b/pkg/cmd/grafana-cli/commands/commands.go @@ -11,22 +11,18 @@ import ( "github.com/grafana/grafana/pkg/setting" ) -var configFile = flag.String("config", "", "path to config file") -var homePath = flag.String("homepath", "", "path to grafana install/home path, defaults to working directory") - func runDbCommand(command func(commandLine CommandLine) error) func(context *cli.Context) { return func(context *cli.Context) { + cmd := &contextCommandLine{context} - flag.Parse() setting.NewConfigContext(&setting.CommandLineArgs{ - Config: *configFile, - HomePath: *homePath, + Config: cmd.String("config"), + HomePath: cmd.String("homepath"), Args: flag.Args(), }) sqlstore.NewEngine() - cmd := &contextCommandLine{context} if err := command(cmd); err != nil { logger.Errorf("\n%s: ", color.RedString("Error")) logger.Errorf("%s\n\n", err) @@ -95,6 +91,16 @@ var adminCommands = []cli.Command{ Name: "reset-admin-password", Usage: "reset-admin-password ", Action: runDbCommand(resetPasswordCommand), + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "homepath", + Usage: "path to grafana install/home path, defaults to working directory", + }, + cli.StringFlag{ + Name: "config", + Usage: "path to config file", + }, + }, }, } diff --git a/pkg/services/alerting/notifiers/telegram.go b/pkg/services/alerting/notifiers/telegram.go index f70d299bc6c..71169c15599 100644 --- a/pkg/services/alerting/notifiers/telegram.go +++ b/pkg/services/alerting/notifiers/telegram.go @@ -87,7 +87,7 @@ func (this *TelegramNotifier) Notify(evalContext *alerting.EvalContext) error { bodyJSON.Set("chat_id", this.ChatID) bodyJSON.Set("parse_mode", "html") - message := fmt.Sprintf("%s\nState: %s\nMessage: %s\n", evalContext.GetNotificationTitle(), evalContext.Rule.Name, evalContext.Rule.Message) + message := fmt.Sprintf("%s\nState: %s\nMessage: %s\n", evalContext.GetNotificationTitle(), evalContext.Rule.Name, evalContext.Rule.Message) ruleUrl, err := evalContext.GetRuleUrl() if err == nil { @@ -96,6 +96,19 @@ func (this *TelegramNotifier) Notify(evalContext *alerting.EvalContext) error { if evalContext.ImagePublicUrl != "" { message = message + fmt.Sprintf("Image: %s\n", evalContext.ImagePublicUrl) } + + metrics := "" + fieldLimitCount := 4 + for index, evt := range evalContext.EvalMatches { + metrics += fmt.Sprintf("\n%s: %s", evt.Metric, evt.Value) + if index > fieldLimitCount { + break + } + } + if metrics != "" { + message = message + fmt.Sprintf("\nMetrics:%s", metrics) + } + bodyJSON.Set("text", message) url := fmt.Sprintf(telegeramApiUrl, this.BotToken, "sendMessage") diff --git a/pkg/services/sqlstore/migrations/migrations.go b/pkg/services/sqlstore/migrations/migrations.go index 163c6d762a8..bf334d57bb0 100644 --- a/pkg/services/sqlstore/migrations/migrations.go +++ b/pkg/services/sqlstore/migrations/migrations.go @@ -24,7 +24,6 @@ func AddMigrations(mg *Migrator) { addPreferencesMigrations(mg) addAlertMigrations(mg) addAnnotationMig(mg) - addStatsMigrations(mg) addTestDataMigrations(mg) } diff --git a/pkg/services/sqlstore/sql_test_data.go b/pkg/services/sqlstore/sql_test_data.go index a83ab76ecc0..ffb3f0fc997 100644 --- a/pkg/services/sqlstore/sql_test_data.go +++ b/pkg/services/sqlstore/sql_test_data.go @@ -14,7 +14,7 @@ func init() { func sqlRandomWalk(m1 string, m2 string, intWalker int64, floatWalker float64, sess *session) error { - timeWalker := time.Now().UTC().Add(time.Hour * -1) + timeWalker := time.Now().UTC().Add(time.Hour * -200) now := time.Now().UTC() step := time.Minute @@ -29,7 +29,7 @@ func sqlRandomWalk(m1 string, m2 string, intWalker int64, floatWalker float64, s timeWalker = timeWalker.Add(step) row.Id = 0 - row.ValueBigInt += rand.Int63n(100) - 100 + row.ValueBigInt += rand.Int63n(200) - 100 row.ValueDouble += rand.Float64() - 0.5 row.ValueFloat += rand.Float32() - 0.5 row.TimeEpoch = timeWalker.Unix() @@ -61,11 +61,6 @@ func InsertSqlTestData(cmd *m.InsertSqlTestDataCommand) error { sqlRandomWalk("server2", "frontend", 100, 1.123, sess) sqlRandomWalk("server3", "frontend", 100, 1.123, sess) - sqlRandomWalk("server1", "backend", 100, 1.123, sess) - sqlRandomWalk("server2", "backend", 100, 1.123, sess) - sqlRandomWalk("server3", "backend", 100, 1.123, sess) - sqlRandomWalk("db-server1", "backend", 100, 1.123, sess) - return err }) } diff --git a/pkg/tsdb/models.go b/pkg/tsdb/models.go index 838767dd5d9..5ae27867c44 100644 --- a/pkg/tsdb/models.go +++ b/pkg/tsdb/models.go @@ -27,6 +27,7 @@ type Request struct { type Response struct { BatchTimings []*BatchTiming `json:"timings"` Results map[string]*QueryResult `json:"results"` + Message string `json:"message,omitempty"` } type BatchTiming struct { @@ -45,18 +46,30 @@ func (br *BatchResult) WithError(err error) *BatchResult { } type QueryResult struct { - Error error `json:"-"` - ErrorString string `json:"error"` - RefId string `json:"refId"` - Series TimeSeriesSlice `json:"series"` + Error error `json:"-"` + ErrorString string `json:"error,omitempty"` + RefId string `json:"refId"` + Meta *simplejson.Json `json:"meta,omitempty"` + Series TimeSeriesSlice `json:"series"` + Tables []*Table `json:"tables"` } type TimeSeries struct { Name string `json:"name"` Points TimeSeriesPoints `json:"points"` - Tags map[string]string `json:"tags"` + Tags map[string]string `json:"tags,omitempty"` } +type Table struct { + Columns []TableColumn `json:"columns"` + Rows []RowValues `json:"rows"` +} + +type TableColumn struct { + Text string `json:"text"` +} + +type RowValues []interface{} type TimePoint [2]null.Float type TimeSeriesPoints []TimePoint type TimeSeriesSlice []*TimeSeries diff --git a/pkg/tsdb/mysql/macros.go b/pkg/tsdb/mysql/macros.go new file mode 100644 index 00000000000..def2fde9fcc --- /dev/null +++ b/pkg/tsdb/mysql/macros.go @@ -0,0 +1,80 @@ +package mysql + +import ( + "fmt" + "regexp" + + "github.com/grafana/grafana/pkg/tsdb" +) + +//const rsString = `(?:"([^"]*)")`; +const rsIdentifier = `([_a-zA-Z0-9]+)` +const sExpr = `\$` + rsIdentifier + `\(([^\)]*)\)` + +type SqlMacroEngine interface { + Interpolate(sql string) (string, error) +} + +type MySqlMacroEngine struct { + TimeRange *tsdb.TimeRange +} + +func NewMysqlMacroEngine(timeRange *tsdb.TimeRange) SqlMacroEngine { + return &MySqlMacroEngine{ + TimeRange: timeRange, + } +} + +func (m *MySqlMacroEngine) Interpolate(sql string) (string, error) { + rExp, _ := regexp.Compile(sExpr) + var macroError error + + sql = ReplaceAllStringSubmatchFunc(rExp, sql, func(groups []string) string { + res, err := m.EvaluateMacro(groups[1], groups[2:]) + if err != nil && macroError == nil { + macroError = err + return "macro_error()" + } + return res + }) + + if macroError != nil { + return "", macroError + } + + return sql, nil +} + +func ReplaceAllStringSubmatchFunc(re *regexp.Regexp, str string, repl func([]string) string) string { + result := "" + lastIndex := 0 + + for _, v := range re.FindAllSubmatchIndex([]byte(str), -1) { + groups := []string{} + for i := 0; i < len(v); i += 2 { + groups = append(groups, str[v[i]:v[i+1]]) + } + + result += str[lastIndex:v[0]] + repl(groups) + lastIndex = v[1] + } + + return result + str[lastIndex:] +} + +func (m *MySqlMacroEngine) EvaluateMacro(name string, args []string) (string, error) { + switch name { + case "__time": + if len(args) == 0 { + return "", fmt.Errorf("missing time column argument for macro %v", name) + } + return fmt.Sprintf("UNIX_TIMESTAMP(%s) as time_sec", args[0]), nil + case "__timeFilter": + if len(args) == 0 { + return "", fmt.Errorf("missing time column argument for macro %v", name) + } + return fmt.Sprintf("%s > FROM_UNIXTIME(%d) AND %s < FROM_UNIXTIME(%d)", args[0], uint64(m.TimeRange.GetFromAsMsEpoch()/1000), args[0], uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + default: + return "", fmt.Errorf("Unknown macro %v", name) + } +} diff --git a/pkg/tsdb/mysql/macros_test.go b/pkg/tsdb/mysql/macros_test.go new file mode 100644 index 00000000000..5b6b885ff0e --- /dev/null +++ b/pkg/tsdb/mysql/macros_test.go @@ -0,0 +1,43 @@ +package mysql + +import ( + "testing" + + "github.com/grafana/grafana/pkg/tsdb" + . "github.com/smartystreets/goconvey/convey" +) + +func TestMacroEngine(t *testing.T) { + Convey("MacroEngine", t, func() { + + Convey("interpolate __time function", func() { + engine := &MySqlMacroEngine{} + + sql, err := engine.Interpolate("select $__time(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "select UNIX_TIMESTAMP(time_column) as time_sec") + }) + + Convey("interpolate __time function wrapped in aggregation", func() { + engine := &MySqlMacroEngine{} + + sql, err := engine.Interpolate("select min($__time(time_column))") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "select min(UNIX_TIMESTAMP(time_column) as time_sec)") + }) + + Convey("interpolate __timeFilter function", func() { + engine := &MySqlMacroEngine{ + TimeRange: &tsdb.TimeRange{From: "5m", To: "now"}, + } + + sql, err := engine.Interpolate("WHERE $__timeFilter(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "WHERE time_column > FROM_UNIXTIME(18446744066914186738) AND time_column < FROM_UNIXTIME(18446744066914187038)") + }) + + }) +} diff --git a/pkg/tsdb/mysql/mysql.go b/pkg/tsdb/mysql/mysql.go index be59079c65c..705db1e50da 100644 --- a/pkg/tsdb/mysql/mysql.go +++ b/pkg/tsdb/mysql/mysql.go @@ -7,9 +7,13 @@ import ( "strconv" "sync" + "time" + + "github.com/go-sql-driver/mysql" "github.com/go-xorm/core" "github.com/go-xorm/xorm" "github.com/grafana/grafana/pkg/components/null" + "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/tsdb" @@ -81,6 +85,7 @@ func (e *MysqlExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, co QueryResults: make(map[string]*tsdb.QueryResult), } + macroEngine := NewMysqlMacroEngine(context.TimeRange) session := e.engine.NewSession() defer session.Close() db := session.DB() @@ -91,48 +96,145 @@ func (e *MysqlExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, co continue } + queryResult := &tsdb.QueryResult{Meta: simplejson.New(), RefId: query.RefId} + result.QueryResults[query.RefId] = queryResult + + rawSql, err := macroEngine.Interpolate(rawSql) + if err != nil { + queryResult.Error = err + continue + } + + queryResult.Meta.Set("sql", rawSql) + rows, err := db.Query(rawSql) if err != nil { - result.QueryResults[query.RefId] = &tsdb.QueryResult{Error: err} + queryResult.Error = err continue } defer rows.Close() - result.QueryResults[query.RefId] = e.TransformToTimeSeries(query, rows) + format := query.Model.Get("format").MustString("time_series") + + switch format { + case "time_series": + err := e.TransformToTimeSeries(query, rows, queryResult) + if err != nil { + queryResult.Error = err + continue + } + case "table": + err := e.TransformToTable(query, rows, queryResult) + if err != nil { + queryResult.Error = err + continue + } + } } return result } -func (e MysqlExecutor) TransformToTimeSeries(query *tsdb.Query, rows *core.Rows) *tsdb.QueryResult { - result := &tsdb.QueryResult{RefId: query.RefId} +func (e MysqlExecutor) TransformToTable(query *tsdb.Query, rows *core.Rows, result *tsdb.QueryResult) error { + columnNames, err := rows.Columns() + columnCount := len(columnNames) + + if err != nil { + return err + } + + table := &tsdb.Table{ + Columns: make([]tsdb.TableColumn, columnCount), + Rows: make([]tsdb.RowValues, 0), + } + + for i, name := range columnNames { + table.Columns[i].Text = name + } + + columnTypes, err := rows.ColumnTypes() + if err != nil { + return err + } + + rowLimit := 1000000 + rowCount := 0 + + for ; rows.Next(); rowCount += 1 { + if rowCount > rowLimit { + return fmt.Errorf("MySQL query row limit exceeded, limit %d", rowLimit) + } + + values, err := e.getTypedRowData(columnTypes, rows) + if err != nil { + return err + } + + table.Rows = append(table.Rows, values) + } + + result.Tables = append(result.Tables, table) + result.Meta.Set("rowCount", rowCount) + return nil +} + +func (e MysqlExecutor) getTypedRowData(types []*sql.ColumnType, rows *core.Rows) (tsdb.RowValues, error) { + values := make([]interface{}, len(types)) + + for i, stype := range types { + switch stype.DatabaseTypeName() { + case mysql.FieldTypeNameVarString: + values[i] = new(string) + case mysql.FieldTypeNameLongLong: + values[i] = new(int64) + case mysql.FieldTypeNameDouble: + values[i] = new(float64) + case mysql.FieldTypeNameDateTime: + values[i] = new(time.Time) + default: + return nil, fmt.Errorf("Database type %s not supported", stype.DatabaseTypeName()) + } + } + + if err := rows.Scan(values...); err != nil { + return nil, err + } + + return values, nil +} + +func (e MysqlExecutor) TransformToTimeSeries(query *tsdb.Query, rows *core.Rows, result *tsdb.QueryResult) error { pointsBySeries := make(map[string]*tsdb.TimeSeries) columnNames, err := rows.Columns() if err != nil { - result.Error = err - return result + return err } rowData := NewStringStringScan(columnNames) - for rows.Next() { + rowLimit := 1000000 + rowCount := 0 + + for ; rows.Next(); rowCount += 1 { + if rowCount > rowLimit { + return fmt.Errorf("MySQL query row limit exceeded, limit %d", rowLimit) + } + err := rowData.Update(rows.Rows) if err != nil { - e.log.Error("Mysql response parsing", "error", err) - result.Error = err - return result + e.log.Error("MySQL response parsing", "error", err) + return fmt.Errorf("MySQL response parsing error %v", err) } if rowData.metric == "" { rowData.metric = "Unknown" } - e.log.Info("Rows", "metric", rowData.metric, "time", rowData.time, "value", rowData.value) + //e.log.Debug("Rows", "metric", rowData.metric, "time", rowData.time, "value", rowData.value) if !rowData.time.Valid { - result.Error = fmt.Errorf("Found row with no time value") - return result + return fmt.Errorf("Found row with no time value") } if series, exist := pointsBySeries[rowData.metric]; exist { @@ -148,7 +250,8 @@ func (e MysqlExecutor) TransformToTimeSeries(query *tsdb.Query, rows *core.Rows) result.Series = append(result.Series, value) } - return result + result.Meta.Set("rowCount", rowCount) + return nil } type stringStringScan struct { diff --git a/public/app/core/components/search/search.ts b/public/app/core/components/search/search.ts index bff6d3149f2..ec58a5c9f5c 100644 --- a/public/app/core/components/search/search.ts +++ b/public/app/core/components/search/search.ts @@ -117,7 +117,7 @@ export class SearchCtrl { queryHasNoFilters() { var query = this.query; return query.query === '' && query.starred === false && query.tag.length === 0; - }; + } filterByTag(tag, evt) { this.query.tag.push(tag); @@ -127,7 +127,7 @@ export class SearchCtrl { evt.stopPropagation(); evt.preventDefault(); } - }; + } removeTag(tag, evt) { this.query.tag = _.without(this.query.tag, tag); @@ -135,7 +135,7 @@ export class SearchCtrl { this.giveSearchFocus = this.giveSearchFocus + 1; evt.stopPropagation(); evt.preventDefault(); - }; + } getTags() { return this.backendSrv.get('/api/dashboards/tags').then((results) => { @@ -146,19 +146,19 @@ export class SearchCtrl { this.search(); } }); - }; + } showStarred() { this.query.starred = !this.query.starred; this.giveSearchFocus = this.giveSearchFocus + 1; this.search(); - }; + } search() { this.showImport = false; this.selectedIndex = 0; this.searchDashboards(); - }; + } } diff --git a/public/app/core/components/sidemenu/sidemenu.ts b/public/app/core/components/sidemenu/sidemenu.ts index a6bc170fddd..62258a25e7d 100644 --- a/public/app/core/components/sidemenu/sidemenu.ts +++ b/public/app/core/components/sidemenu/sidemenu.ts @@ -84,7 +84,11 @@ export class SideMenuCtrl { return; } - if (this.orgItems.length < this.maxShownOrgs && (this.orgFilter === '' || org.name.indexOf(this.orgFilter) !== -1)){ + if (this.orgItems.length === this.maxShownOrgs) { + return; + } + + if (this.orgFilter === '' || (org.name.toLowerCase().indexOf(this.orgFilter.toLowerCase()) !== -1)) { this.orgItems.push({ text: "Switch to " + org.name, icon: "fa fa-fw fa-random", diff --git a/public/app/core/controllers/signup_ctrl.ts b/public/app/core/controllers/signup_ctrl.ts index 36544586a31..459f215a1fc 100644 --- a/public/app/core/controllers/signup_ctrl.ts +++ b/public/app/core/controllers/signup_ctrl.ts @@ -44,7 +44,7 @@ export class SignUpCtrl { window.location.href = config.appSubUrl + '/'; } }); - }; + } } coreModule.controller('SignUpCtrl', SignUpCtrl); diff --git a/public/app/core/directives/plugin_component.ts b/public/app/core/directives/plugin_component.ts index 83c030cb317..3e446b14b9e 100644 --- a/public/app/core/directives/plugin_component.ts +++ b/public/app/core/directives/plugin_component.ts @@ -75,7 +75,7 @@ function pluginDirectiveLoader($compile, datasourceSrv, $rootScope, $q, $http, $ if (!PanelCtrl || PanelCtrl.registered) { return componentInfo; - }; + } if (PanelCtrl.templatePromise) { return PanelCtrl.templatePromise.then(res => { diff --git a/public/app/core/services/backend_srv.ts b/public/app/core/services/backend_srv.ts index 041cd1ab1db..16edc364340 100644 --- a/public/app/core/services/backend_srv.ts +++ b/public/app/core/services/backend_srv.ts @@ -23,7 +23,7 @@ export class BackendSrv { post(url, data) { return this.request({ method: 'POST', url: url, data: data }); - }; + } patch(url, data) { return this.request({ method: 'PATCH', url: url, data: data }); @@ -98,7 +98,7 @@ export class BackendSrv { this.$timeout(this.requestErrorHandler.bind(this, err), 50); throw err; }); - }; + } addCanceler(requestId, canceler) { if (requestId in this.inFlightRequests) { @@ -186,7 +186,7 @@ export class BackendSrv { this.inFlightRequests[options.requestId].shift(); } }); - }; + } loginPing() { return this.request({url: '/api/login/ping', method: 'GET', retry: 1 }); diff --git a/public/app/core/time_series2.ts b/public/app/core/time_series2.ts index e5d6c342fd0..daf03f8f827 100644 --- a/public/app/core/time_series2.ts +++ b/public/app/core/time_series2.ts @@ -92,7 +92,7 @@ export default class TimeSeries { this.yaxis = override.yaxis; } } - }; + } getFlotPairs(fillStyle) { var result = []; diff --git a/public/app/core/utils/file_export.ts b/public/app/core/utils/file_export.ts index 27f1064286f..f2f0192e034 100644 --- a/public/app/core/utils/file_export.ts +++ b/public/app/core/utils/file_export.ts @@ -12,7 +12,7 @@ export function exportSeriesListToCsv(seriesList) { }); }); saveSaveBlob(text, 'grafana_data_export.csv'); -}; +} export function exportSeriesListToCsvColumns(seriesList) { var text = 'sep=;\nTime;'; @@ -47,7 +47,7 @@ export function exportSeriesListToCsvColumns(seriesList) { text += '\n'; } saveSaveBlob(text, 'grafana_data_export.csv'); -}; +} export function exportTableDataToCsv(table) { var text = 'sep=;\n'; @@ -64,9 +64,9 @@ export function exportTableDataToCsv(table) { text += '\n'; }); saveSaveBlob(text, 'grafana_data_export.csv'); -}; +} export function saveSaveBlob(payload, fname) { var blob = new Blob([payload], { type: "text/csv;charset=utf-8" }); window.saveAs(blob, fname); -}; +} diff --git a/public/app/features/annotations/editor_ctrl.ts b/public/app/features/annotations/editor_ctrl.ts index deb90691d91..74c4768b5ad 100644 --- a/public/app/features/annotations/editor_ctrl.ts +++ b/public/app/features/annotations/editor_ctrl.ts @@ -69,7 +69,7 @@ export class AnnotationsEditorCtrl { this.reset(); this.mode = 'list'; this.$scope.broadcastRefresh(); - }; + } add() { this.annotations.push(this.currentAnnotation); @@ -77,7 +77,7 @@ export class AnnotationsEditorCtrl { this.mode = 'list'; this.$scope.broadcastRefresh(); this.$scope.dashboard.updateSubmenuVisibility(); - }; + } removeAnnotation(annotation) { var index = _.indexOf(this.annotations, annotation); diff --git a/public/app/features/dashboard/row/row_ctrl.ts b/public/app/features/dashboard/row/row_ctrl.ts index ce92821dfef..34b03b3c3be 100644 --- a/public/app/features/dashboard/row/row_ctrl.ts +++ b/public/app/features/dashboard/row/row_ctrl.ts @@ -216,7 +216,6 @@ coreModule.directive('panelDropZone', function($timeout) { } if (indrag === true) { - var dropZoneSpan = 12 - row.span; if (dropZoneSpan > 1) { return showPanel(dropZoneSpan, 'Drop Here'); } diff --git a/public/app/features/dashboard/time_srv.ts b/public/app/features/dashboard/time_srv.ts index 7891232d53b..1385751decc 100644 --- a/public/app/features/dashboard/time_srv.ts +++ b/public/app/features/dashboard/time_srv.ts @@ -60,7 +60,7 @@ class TimeSrv { if (_.isString(this.time.to) && this.time.to.indexOf('Z') >= 0) { this.time.to = moment(this.time.to).utc(); } - }; + } private parseUrlParam(value) { if (value.indexOf('now') !== -1) { @@ -92,7 +92,7 @@ class TimeSrv { if (params.refresh) { this.refresh = params.refresh || this.refresh; } - }; + } private routeUpdated() { var params = this.$location.search(); @@ -154,7 +154,7 @@ class TimeSrv { private cancelNextRefresh() { this.timer.cancel(this.refreshTimer); - }; + } setTime(time, fromRouteUpdate?) { _.extend(this.time, time); @@ -184,8 +184,8 @@ class TimeSrv { timeRangeForUrl() { var range = this.timeRange().raw; - if (moment.isMoment(range.from)) { range.from = range.from.valueOf(); } - if (moment.isMoment(range.to)) { range.to = range.to.valueOf(); } + if (moment.isMoment(range.from)) { range.from = range.from.valueOf().toString(); } + if (moment.isMoment(range.to)) { range.to = range.to.valueOf().toString(); } return range; } diff --git a/public/app/features/panel/metrics_panel_ctrl.ts b/public/app/features/panel/metrics_panel_ctrl.ts index b42d4a8b02a..af9d06d8742 100644 --- a/public/app/features/panel/metrics_panel_ctrl.ts +++ b/public/app/features/panel/metrics_panel_ctrl.ts @@ -31,6 +31,7 @@ class MetricsPanelCtrl extends PanelCtrl { skipDataOnInit: boolean; dataStream: any; dataSubscription: any; + dataList: any; constructor($scope, $injector) { super($scope, $injector); @@ -106,6 +107,16 @@ class MetricsPanelCtrl extends PanelCtrl { this.loading = false; this.error = err.message || "Request Error"; this.inspector = {error: err}; + + if (err.data) { + if (err.data.message) { + this.error = err.data.message; + } + if (err.data.error) { + this.error = err.data.error; + } + } + this.events.emit('data-error', err); console.log('Panel data error:', err); }); @@ -136,7 +147,7 @@ class MetricsPanelCtrl extends PanelCtrl { this.calculateInterval(); return this.datasource; - }; + } calculateInterval() { var intervalOverride = this.panel.interval; @@ -194,7 +205,7 @@ class MetricsPanelCtrl extends PanelCtrl { if (this.panel.hideTimeOverride) { this.timeInfo = ''; } - }; + } issueQueries(datasource) { this.datasource = datasource; diff --git a/public/app/features/panel/partials/query_editor_row.html b/public/app/features/panel/partials/query_editor_row.html index 55933bbbae8..4cdc04da512 100644 --- a/public/app/features/panel/partials/query_editor_row.html +++ b/public/app/features/panel/partials/query_editor_row.html @@ -1,7 +1,7 @@
-
-