From 5b53c0497af412cfb93485255247937c2bafc300 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Tue, 2 Oct 2018 18:29:40 +0900 Subject: [PATCH 01/48] for development --- .../stackdriver/partials/config.html | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/public/app/plugins/datasource/stackdriver/partials/config.html b/public/app/plugins/datasource/stackdriver/partials/config.html index 46b79d8bb0d..71133d13c55 100644 --- a/public/app/plugins/datasource/stackdriver/partials/config.html +++ b/public/app/plugins/datasource/stackdriver/partials/config.html @@ -1,4 +1,4 @@ -
+
GCP Service Account

@@ -24,7 +24,7 @@

-
+

Service Account Authentication

Upload your Service Account key file or paste in the contents of the file. The file @@ -81,6 +81,23 @@
+

Do not forget to save your changes after uploading a file.

+ +
+
+

Local machine

+ For development. +
+ +
+
+ Default Project ID + +
+
+
+ +
Do not forget to save your changes after uploading a file.
From 022a45becd64816b00061cde40901271e90e5f9e Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Tue, 2 Oct 2018 18:30:53 +0900 Subject: [PATCH 02/48] Revert "for development" This reverts commit cf2f110e42bb0ca041be7bb0ebf549bf565c51bc. --- .../stackdriver/partials/config.html | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/public/app/plugins/datasource/stackdriver/partials/config.html b/public/app/plugins/datasource/stackdriver/partials/config.html index 71133d13c55..a236cb6832c 100644 --- a/public/app/plugins/datasource/stackdriver/partials/config.html +++ b/public/app/plugins/datasource/stackdriver/partials/config.html @@ -1,4 +1,4 @@ -
+
GCP Service Account

@@ -24,7 +24,7 @@

-
+

Service Account Authentication

Upload your Service Account key file or paste in the contents of the file. The file @@ -83,20 +83,6 @@

Do not forget to save your changes after uploading a file.

-
-
-

Local machine

- For development. -
- -
-
- Default Project ID - -
-
-
-
Do not forget to save your changes after uploading a file. From 43aa6603a3ef4e7b60192518d32f26dd60a6411d Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Tue, 2 Oct 2018 18:40:39 +0900 Subject: [PATCH 03/48] fetch token from GCE metadata server --- pkg/api/pluginproxy/ds_auth_provider.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pkg/api/pluginproxy/ds_auth_provider.go b/pkg/api/pluginproxy/ds_auth_provider.go index c68da839d13..0cf3489e718 100644 --- a/pkg/api/pluginproxy/ds_auth_provider.go +++ b/pkg/api/pluginproxy/ds_auth_provider.go @@ -12,6 +12,7 @@ import ( m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/util" + "golang.org/x/oauth2/google" ) //ApplyRoute should use the plugin route data to set auth headers and custom headers @@ -61,6 +62,21 @@ func ApplyRoute(ctx context.Context, req *http.Request, proxyPath string, route req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) } } + + if req.Header.Get("Authorization") == "" && ds.Type == "stackdriver" { + tokenSrc, err := google.DefaultTokenSource(ctx, "https://www.googleapis.com/auth/monitoring.read") + if err != nil { + logger.Error("Failed to get access token", "error", err) + } else { + token, err := tokenSrc.Token() + if err != nil { + logger.Error("Failed to get access token", "error", err) + } else { + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token.AccessToken)) + } + } + } + logger.Info("Requesting", "url", req.URL.String()) } From fbe0ffd10b3e861e3023fa17575a4e421167b122 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Wed, 3 Oct 2018 12:29:40 +0900 Subject: [PATCH 04/48] skip jwt token auth if privateKey is empty --- pkg/api/pluginproxy/ds_auth_provider.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/api/pluginproxy/ds_auth_provider.go b/pkg/api/pluginproxy/ds_auth_provider.go index 0cf3489e718..d32d34f6187 100644 --- a/pkg/api/pluginproxy/ds_auth_provider.go +++ b/pkg/api/pluginproxy/ds_auth_provider.go @@ -55,7 +55,7 @@ func ApplyRoute(ctx context.Context, req *http.Request, proxyPath string, route } } - if route.JwtTokenAuth != nil { + if route.JwtTokenAuth != nil && len(ds.SecureJsonData["privateKey"]) != 0 { if token, err := tokenProvider.getJwtAccessToken(ctx, data); err != nil { logger.Error("Failed to get access token", "error", err) } else { From df215d9d09caaab28cc736c5403b31b08f2ef276 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 3 Oct 2018 17:08:13 +0200 Subject: [PATCH 05/48] stackdriver: WIP - test retrieving project id from gce metadata --- pkg/api/pluginproxy/ds_auth_provider.go | 26 +++++++++++++++++++++++-- pkg/tsdb/stackdriver/stackdriver.go | 3 +++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/pkg/api/pluginproxy/ds_auth_provider.go b/pkg/api/pluginproxy/ds_auth_provider.go index d32d34f6187..320c4cc2cd3 100644 --- a/pkg/api/pluginproxy/ds_auth_provider.go +++ b/pkg/api/pluginproxy/ds_auth_provider.go @@ -64,11 +64,33 @@ func ApplyRoute(ctx context.Context, req *http.Request, proxyPath string, route } if req.Header.Get("Authorization") == "" && ds.Type == "stackdriver" { - tokenSrc, err := google.DefaultTokenSource(ctx, "https://www.googleapis.com/auth/monitoring.read") + defaultCredentials, err := google.FindDefaultCredentials(ctx) + projectName := ds.JsonData.Get("defaultProject").MustString() + if projectName == "" { + ds.JsonData.Set("defaultProject", defaultCredentials.ProjectID) + } + if err != nil { + logger.Error("Failed to get default credentials", "error", err) + } else { + token, err := defaultCredentials.TokenSource.Token() + if err != nil { + logger.Error("Failed to get default access token", "error", err) + } else { + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token.AccessToken)) + } + } + } + + if req.Header.Get("Authorization") == "" && ds.Type == "stackdriver" { + credentials, err := google.FindDefaultCredentials(ctx, "https://www.googleapis.com/auth/monitoring.read") + projectName := ds.JsonData.Get("defaultProject").MustString() + if projectName == "" { + ds.JsonData.Set("defaultProject", credentials.ProjectID) + } if err != nil { logger.Error("Failed to get access token", "error", err) } else { - token, err := tokenSrc.Token() + token, err := credentials.TokenSource.Token() if err != nil { logger.Error("Failed to get access token", "error", err) } else { diff --git a/pkg/tsdb/stackdriver/stackdriver.go b/pkg/tsdb/stackdriver/stackdriver.go index 96242dfdec4..f555ebfa3f4 100644 --- a/pkg/tsdb/stackdriver/stackdriver.go +++ b/pkg/tsdb/stackdriver/stackdriver.go @@ -15,6 +15,8 @@ import ( "strings" "time" + "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" + "golang.org/x/net/context/ctxhttp" "github.com/grafana/grafana/pkg/api/pluginproxy" @@ -551,6 +553,7 @@ func (e *StackdriverExecutor) createRequest(ctx context.Context, dsInfo *models. return nil, errors.New("Unable to find datasource plugin Stackdriver") } projectName := dsInfo.JsonData.Get("defaultProject").MustString() + logger.Info("projectName", "projectName", projectName) proxyPass := fmt.Sprintf("stackdriver%s", "v3/projects/"+projectName+"/timeSeries") var stackdriverRoute *plugins.AppPluginRoute From 200216207384f7633ab7ef8b54e2d6caa826f241 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 3 Oct 2018 21:37:00 +0200 Subject: [PATCH 06/48] stackdriver: wip - add scope and remove debug code --- pkg/api/pluginproxy/ds_auth_provider.go | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/pkg/api/pluginproxy/ds_auth_provider.go b/pkg/api/pluginproxy/ds_auth_provider.go index 320c4cc2cd3..85d6beb35e6 100644 --- a/pkg/api/pluginproxy/ds_auth_provider.go +++ b/pkg/api/pluginproxy/ds_auth_provider.go @@ -64,7 +64,7 @@ func ApplyRoute(ctx context.Context, req *http.Request, proxyPath string, route } if req.Header.Get("Authorization") == "" && ds.Type == "stackdriver" { - defaultCredentials, err := google.FindDefaultCredentials(ctx) + defaultCredentials, err := google.FindDefaultCredentials(ctx, "https://www.googleapis.com/auth/monitoring.read") projectName := ds.JsonData.Get("defaultProject").MustString() if projectName == "" { ds.JsonData.Set("defaultProject", defaultCredentials.ProjectID) @@ -81,24 +81,6 @@ func ApplyRoute(ctx context.Context, req *http.Request, proxyPath string, route } } - if req.Header.Get("Authorization") == "" && ds.Type == "stackdriver" { - credentials, err := google.FindDefaultCredentials(ctx, "https://www.googleapis.com/auth/monitoring.read") - projectName := ds.JsonData.Get("defaultProject").MustString() - if projectName == "" { - ds.JsonData.Set("defaultProject", credentials.ProjectID) - } - if err != nil { - logger.Error("Failed to get access token", "error", err) - } else { - token, err := credentials.TokenSource.Token() - if err != nil { - logger.Error("Failed to get access token", "error", err) - } else { - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token.AccessToken)) - } - } - } - logger.Info("Requesting", "url", req.URL.String()) } From b2de0d8c3e0ba10122083091830f2f30aa7090e6 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 3 Oct 2018 21:48:20 +0200 Subject: [PATCH 07/48] stackdriver: wip - always use gce default account for stackdriver --- pkg/api/pluginproxy/ds_auth_provider.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/api/pluginproxy/ds_auth_provider.go b/pkg/api/pluginproxy/ds_auth_provider.go index 85d6beb35e6..b551a9a4d8a 100644 --- a/pkg/api/pluginproxy/ds_auth_provider.go +++ b/pkg/api/pluginproxy/ds_auth_provider.go @@ -55,15 +55,15 @@ func ApplyRoute(ctx context.Context, req *http.Request, proxyPath string, route } } - if route.JwtTokenAuth != nil && len(ds.SecureJsonData["privateKey"]) != 0 { - if token, err := tokenProvider.getJwtAccessToken(ctx, data); err != nil { - logger.Error("Failed to get access token", "error", err) - } else { - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) - } - } + // if route.JwtTokenAuth != nil && len(ds.SecureJsonData["privateKey"]) != 0 { + // if token, err := tokenProvider.getJwtAccessToken(ctx, data); err != nil { + // logger.Error("Failed to get access token", "error", err) + // } else { + // req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) + // } + // } - if req.Header.Get("Authorization") == "" && ds.Type == "stackdriver" { + if ds.Type == "stackdriver" { defaultCredentials, err := google.FindDefaultCredentials(ctx, "https://www.googleapis.com/auth/monitoring.read") projectName := ds.JsonData.Get("defaultProject").MustString() if projectName == "" { From 3f9ed2efaa7ff85cd622efabcb9990f06e961e0c Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Mon, 8 Oct 2018 08:39:58 +0200 Subject: [PATCH 08/48] stackdriver: wip - temp remove jwt token auth --- pkg/api/pluginproxy/ds_auth_provider.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pkg/api/pluginproxy/ds_auth_provider.go b/pkg/api/pluginproxy/ds_auth_provider.go index b551a9a4d8a..5a8bf5605dd 100644 --- a/pkg/api/pluginproxy/ds_auth_provider.go +++ b/pkg/api/pluginproxy/ds_auth_provider.go @@ -64,11 +64,7 @@ func ApplyRoute(ctx context.Context, req *http.Request, proxyPath string, route // } if ds.Type == "stackdriver" { - defaultCredentials, err := google.FindDefaultCredentials(ctx, "https://www.googleapis.com/auth/monitoring.read") - projectName := ds.JsonData.Get("defaultProject").MustString() - if projectName == "" { - ds.JsonData.Set("defaultProject", defaultCredentials.ProjectID) - } + defaultCredentials, err := google.FindDefaultCredentials(ctx, route.JwtTokenAuth.Scopes...) if err != nil { logger.Error("Failed to get default credentials", "error", err) } else { @@ -76,6 +72,7 @@ func ApplyRoute(ctx context.Context, req *http.Request, proxyPath string, route if err != nil { logger.Error("Failed to get default access token", "error", err) } else { + ds.JsonData.Set("defaultProject", defaultCredentials.ProjectID) req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token.AccessToken)) } } From 102f531c1e8e20dc2ef83b988ffe21ee33eee905 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Mon, 8 Oct 2018 13:23:46 +0200 Subject: [PATCH 09/48] stackdriver: break out project name resolving into its own function in the stackdriver.go file --- pkg/tsdb/stackdriver/stackdriver.go | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/pkg/tsdb/stackdriver/stackdriver.go b/pkg/tsdb/stackdriver/stackdriver.go index f555ebfa3f4..87856e3f36c 100644 --- a/pkg/tsdb/stackdriver/stackdriver.go +++ b/pkg/tsdb/stackdriver/stackdriver.go @@ -18,6 +18,7 @@ import ( "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" "golang.org/x/net/context/ctxhttp" + "golang.org/x/oauth2/google" "github.com/grafana/grafana/pkg/api/pluginproxy" "github.com/grafana/grafana/pkg/components/null" @@ -518,6 +519,23 @@ func replaceWithMetricPart(metaPartName string, metricType string) []byte { return nil } +func getProjectName(ctx context.Context, dsInfo *models.DataSource, route *plugins.AppPluginRoute) (string, error) { + var projectName string + gceAutoAuthentication := dsInfo.JsonData.Get("gceAutoAuthentication").MustBool() + if gceAutoAuthentication { + defaultCredentials, err := google.FindDefaultCredentials(ctx, route.JwtTokenAuth.Scopes...) + if err != nil { + return "", err + } else { + projectName = defaultCredentials.ProjectID + } + } else { + projectName = dsInfo.JsonData.Get("defaultProject").MustString() + } + logger.Info("projectName", "projectName", projectName) + return projectName, nil +} + func calcBucketBound(bucketOptions StackdriverBucketOptions, n int) string { bucketBound := "0" if n == 0 { @@ -552,9 +570,6 @@ func (e *StackdriverExecutor) createRequest(ctx context.Context, dsInfo *models. if !ok { return nil, errors.New("Unable to find datasource plugin Stackdriver") } - projectName := dsInfo.JsonData.Get("defaultProject").MustString() - logger.Info("projectName", "projectName", projectName) - proxyPass := fmt.Sprintf("stackdriver%s", "v3/projects/"+projectName+"/timeSeries") var stackdriverRoute *plugins.AppPluginRoute for _, route := range plugin.Routes { @@ -564,6 +579,14 @@ func (e *StackdriverExecutor) createRequest(ctx context.Context, dsInfo *models. } } + // projectName := dsInfo.JsonData.Get("defaultProject").MustString() + // logger.Info("projectName", "projectName", projectName) + projectName, err := getProjectName(ctx, dsInfo, stackdriverRoute) + if err != nil { + return nil, err + } + proxyPass := fmt.Sprintf("stackdriver%s", "v3/projects/"+projectName+"/timeSeries") + pluginproxy.ApplyRoute(ctx, req, proxyPass, stackdriverRoute, dsInfo) return req, nil From 77e4c4249dbe42b2d73e21e49bf17311f353525d Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Mon, 8 Oct 2018 13:48:08 +0200 Subject: [PATCH 10/48] stackdriver: wip - add very basic checkbox for gce auto authentication --- .../plugins/datasource/stackdriver/partials/config.html | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/public/app/plugins/datasource/stackdriver/partials/config.html b/public/app/plugins/datasource/stackdriver/partials/config.html index a236cb6832c..6fd52a898f5 100644 --- a/public/app/plugins/datasource/stackdriver/partials/config.html +++ b/public/app/plugins/datasource/stackdriver/partials/config.html @@ -31,6 +31,9 @@ contents will be encrypted and saved in the Grafana database.
+ +
@@ -60,9 +63,9 @@
- Client Email - -
+ Client Email + +
Token URI From 60617d0bf714aff4ecf75bc0d08581db50c78e4f Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Mon, 8 Oct 2018 13:49:27 +0200 Subject: [PATCH 11/48] stackdriver: wip - add logic for retrieving token from gce metadata server in the auth provider --- pkg/api/pluginproxy/ds_auth_provider.go | 22 +++++++++++----------- pkg/tsdb/stackdriver/stackdriver.go | 5 +++-- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/pkg/api/pluginproxy/ds_auth_provider.go b/pkg/api/pluginproxy/ds_auth_provider.go index 5a8bf5605dd..8c6646f42b5 100644 --- a/pkg/api/pluginproxy/ds_auth_provider.go +++ b/pkg/api/pluginproxy/ds_auth_provider.go @@ -55,24 +55,24 @@ func ApplyRoute(ctx context.Context, req *http.Request, proxyPath string, route } } - // if route.JwtTokenAuth != nil && len(ds.SecureJsonData["privateKey"]) != 0 { - // if token, err := tokenProvider.getJwtAccessToken(ctx, data); err != nil { - // logger.Error("Failed to get access token", "error", err) - // } else { - // req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) - // } - // } + gceAutoAuthentication := ds.JsonData.Get("gceAutomaticAuthentication").MustBool() + if route.JwtTokenAuth != nil && !gceAutoAuthentication { + if token, err := tokenProvider.getJwtAccessToken(ctx, data); err != nil { + logger.Error("Failed to get access token", "error", err) + } else { + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) + } + } - if ds.Type == "stackdriver" { - defaultCredentials, err := google.FindDefaultCredentials(ctx, route.JwtTokenAuth.Scopes...) + if gceAutoAuthentication { + tokenSrc, err := google.DefaultTokenSource(ctx, route.JwtTokenAuth.Scopes...) if err != nil { logger.Error("Failed to get default credentials", "error", err) } else { - token, err := defaultCredentials.TokenSource.Token() + token, err := tokenSrc.Token() if err != nil { logger.Error("Failed to get default access token", "error", err) } else { - ds.JsonData.Set("defaultProject", defaultCredentials.ProjectID) req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token.AccessToken)) } } diff --git a/pkg/tsdb/stackdriver/stackdriver.go b/pkg/tsdb/stackdriver/stackdriver.go index 87856e3f36c..dc7fbdc135e 100644 --- a/pkg/tsdb/stackdriver/stackdriver.go +++ b/pkg/tsdb/stackdriver/stackdriver.go @@ -521,8 +521,9 @@ func replaceWithMetricPart(metaPartName string, metricType string) []byte { func getProjectName(ctx context.Context, dsInfo *models.DataSource, route *plugins.AppPluginRoute) (string, error) { var projectName string - gceAutoAuthentication := dsInfo.JsonData.Get("gceAutoAuthentication").MustBool() - if gceAutoAuthentication { + gceAutomaticAuthentication := dsInfo.JsonData.Get("gceAutomaticAuthentication").MustBool() + logger.Info("gceAutomaticAuthentication", "gceAutomaticAuthentication", gceAutomaticAuthentication) + if gceAutomaticAuthentication { defaultCredentials, err := google.FindDefaultCredentials(ctx, route.JwtTokenAuth.Scopes...) if err != nil { return "", err From ff5f281508979087af273b9bac2edaa891317a7b Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Tue, 9 Oct 2018 09:32:16 +0200 Subject: [PATCH 12/48] stackdriver: wip - get metric descriptors in the backend --- pkg/api/pluginproxy/ds_auth_provider.go | 11 +++-- .../stackdriver/metric_descriptors_query.go | 47 +++++++++++++++++++ pkg/tsdb/stackdriver/stackdriver.go | 11 +++-- 3 files changed, 60 insertions(+), 9 deletions(-) create mode 100644 pkg/tsdb/stackdriver/metric_descriptors_query.go diff --git a/pkg/api/pluginproxy/ds_auth_provider.go b/pkg/api/pluginproxy/ds_auth_provider.go index 8c6646f42b5..d3b628b9de4 100644 --- a/pkg/api/pluginproxy/ds_auth_provider.go +++ b/pkg/api/pluginproxy/ds_auth_provider.go @@ -12,7 +12,6 @@ import ( m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/util" - "golang.org/x/oauth2/google" ) //ApplyRoute should use the plugin route data to set auth headers and custom headers @@ -56,6 +55,7 @@ func ApplyRoute(ctx context.Context, req *http.Request, proxyPath string, route } gceAutoAuthentication := ds.JsonData.Get("gceAutomaticAuthentication").MustBool() + logger.Info("gceAutoAuthentication", "gceAutoAuthentication", gceAutoAuthentication) if route.JwtTokenAuth != nil && !gceAutoAuthentication { if token, err := tokenProvider.getJwtAccessToken(ctx, data); err != nil { logger.Error("Failed to get access token", "error", err) @@ -65,21 +65,22 @@ func ApplyRoute(ctx context.Context, req *http.Request, proxyPath string, route } if gceAutoAuthentication { - tokenSrc, err := google.DefaultTokenSource(ctx, route.JwtTokenAuth.Scopes...) + // tokenSrc, err := google.DefaultTokenSource(ctx, route.JwtTokenAuth.Scopes...) if err != nil { logger.Error("Failed to get default credentials", "error", err) } else { - token, err := tokenSrc.Token() + // token, err := tokenSrc.Token() + token, err := tokenProvider.getJwtAccessToken(ctx, data) if err != nil { logger.Error("Failed to get default access token", "error", err) } else { - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token.AccessToken)) + // req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token.AccessToken)) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) } } } logger.Info("Requesting", "url", req.URL.String()) - } func interpolateString(text string, data templateData) (string, error) { diff --git a/pkg/tsdb/stackdriver/metric_descriptors_query.go b/pkg/tsdb/stackdriver/metric_descriptors_query.go new file mode 100644 index 00000000000..da2e5f5d9e0 --- /dev/null +++ b/pkg/tsdb/stackdriver/metric_descriptors_query.go @@ -0,0 +1,47 @@ +package stackdriver + +import ( + "context" + "fmt" + "io/ioutil" + + "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" + "github.com/grafana/grafana/pkg/components/simplejson" + "golang.org/x/net/context/ctxhttp" + + "github.com/grafana/grafana/pkg/tsdb" +) + +func (e *StackdriverExecutor) executeMetricDescriptors(ctx context.Context, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) { + logger.Info("metricDescriptors", "metricDescriptors", tsdbQuery.Queries[0].RefId) + queryResult := &tsdb.QueryResult{Meta: simplejson.New(), RefId: tsdbQuery.Queries[0].RefId} + result := &tsdb.Response{ + Results: make(map[string]*tsdb.QueryResult), + } + + req, err := e.createRequest(ctx, e.dsInfo, "metricDescriptors") + if err != nil { + slog.Error("Failed to create request", "error", err) + return nil, fmt.Errorf("Failed to create request. error: %v", err) + } + res, err := ctxhttp.Do(ctx, e.httpClient, req) + if err != nil { + logger.Info("error2", err) + return nil, err + } + body, err := ioutil.ReadAll(res.Body) + if err != nil { + logger.Info("error3", err) + return nil, err + } + defer res.Body.Close() + if err != nil { + return nil, err + } + + queryResult.Meta.Set("test", string(body)) + logger.Info("string(body)", "string(body)", string(body)) + result.Results[tsdbQuery.Queries[0].RefId] = queryResult + + return result, nil +} diff --git a/pkg/tsdb/stackdriver/stackdriver.go b/pkg/tsdb/stackdriver/stackdriver.go index dc7fbdc135e..d36487e344a 100644 --- a/pkg/tsdb/stackdriver/stackdriver.go +++ b/pkg/tsdb/stackdriver/stackdriver.go @@ -74,6 +74,8 @@ func (e *StackdriverExecutor) Query(ctx context.Context, dsInfo *models.DataSour switch queryType { case "annotationQuery": result, err = e.executeAnnotationQuery(ctx, tsdbQuery) + case "metricDescriptors": + result, err = e.executeMetricDescriptors(ctx, tsdbQuery) case "timeSeriesQuery": fallthrough default: @@ -268,7 +270,7 @@ func setAggParams(params *url.Values, query *tsdb.Query, durationSeconds int) { func (e *StackdriverExecutor) executeQuery(ctx context.Context, query *StackdriverQuery, tsdbQuery *tsdb.TsdbQuery) (*tsdb.QueryResult, StackdriverResponse, error) { queryResult := &tsdb.QueryResult{Meta: simplejson.New(), RefId: query.RefID} - req, err := e.createRequest(ctx, e.dsInfo) + req, err := e.createRequest(ctx, e.dsInfo, "timeSeries") if err != nil { queryResult.Error = err return queryResult, StackdriverResponse{}, nil @@ -526,7 +528,8 @@ func getProjectName(ctx context.Context, dsInfo *models.DataSource, route *plugi if gceAutomaticAuthentication { defaultCredentials, err := google.FindDefaultCredentials(ctx, route.JwtTokenAuth.Scopes...) if err != nil { - return "", err + // return "", err + projectName = "raintank-dev" } else { projectName = defaultCredentials.ProjectID } @@ -553,7 +556,7 @@ func calcBucketBound(bucketOptions StackdriverBucketOptions, n int) string { return bucketBound } -func (e *StackdriverExecutor) createRequest(ctx context.Context, dsInfo *models.DataSource) (*http.Request, error) { +func (e *StackdriverExecutor) createRequest(ctx context.Context, dsInfo *models.DataSource, endpointName string)) (*http.Request, error) { u, _ := url.Parse(dsInfo.Url) u.Path = path.Join(u.Path, "render") @@ -586,7 +589,7 @@ func (e *StackdriverExecutor) createRequest(ctx context.Context, dsInfo *models. if err != nil { return nil, err } - proxyPass := fmt.Sprintf("stackdriver%s", "v3/projects/"+projectName+"/timeSeries") + proxyPass := fmt.Sprintf("stackdriver%s", "v3/projects/"+projectName+"/"+endpointName) pluginproxy.ApplyRoute(ctx, req, proxyPass, stackdriverRoute, dsInfo) From 412a1f6d998a684b2d106167a8addea69883d210 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Tue, 9 Oct 2018 13:17:29 +0200 Subject: [PATCH 13/48] stackdriver: wip - return metric descriptors in the format of tablew --- .../stackdriver/metric_descriptors_query.go | 56 ++++++++++++-- pkg/tsdb/stackdriver/types.go | 21 ++++++ .../datasource/stackdriver/datasource.ts | 75 +++++++++++-------- 3 files changed, 114 insertions(+), 38 deletions(-) diff --git a/pkg/tsdb/stackdriver/metric_descriptors_query.go b/pkg/tsdb/stackdriver/metric_descriptors_query.go index da2e5f5d9e0..6c005d2966b 100644 --- a/pkg/tsdb/stackdriver/metric_descriptors_query.go +++ b/pkg/tsdb/stackdriver/metric_descriptors_query.go @@ -2,8 +2,11 @@ package stackdriver import ( "context" + "encoding/json" "fmt" "io/ioutil" + "net/http" + "strings" "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" "github.com/grafana/grafana/pkg/components/simplejson" @@ -29,19 +32,56 @@ func (e *StackdriverExecutor) executeMetricDescriptors(ctx context.Context, tsdb logger.Info("error2", err) return nil, err } - body, err := ioutil.ReadAll(res.Body) - if err != nil { - logger.Info("error3", err) - return nil, err - } - defer res.Body.Close() + + data, err := e.unmarshalMetricDescriptors(res) if err != nil { return nil, err } - queryResult.Meta.Set("test", string(body)) - logger.Info("string(body)", "string(body)", string(body)) + parts := strings.Split(req.URL.Path, "/") + defaultProject := parts[3] + + table := transformMetricDescriptorResponseToTable(data) + queryResult.Tables = append(queryResult.Tables, table) result.Results[tsdbQuery.Queries[0].RefId] = queryResult + result.Results[tsdbQuery.Queries[0].RefId].Meta.Set("defaultProject", defaultProject) return result, nil } + +func transformMetricDescriptorResponseToTable(data MetricDescriptorsResponse) *tsdb.Table { + table := &tsdb.Table{ + Columns: make([]tsdb.TableColumn, 1), + Rows: make([]tsdb.RowValues, 0), + } + table.Columns[0].Text = "metricDescriptor" + + for _, r := range data.MetricDescriptors { + values := make([]interface{}, 1) + values[0] = r + table.Rows = append(table.Rows, values) + } + return table +} + +func (e *StackdriverExecutor) unmarshalMetricDescriptors(res *http.Response) (MetricDescriptorsResponse, error) { + body, err := ioutil.ReadAll(res.Body) + defer res.Body.Close() + if err != nil { + return MetricDescriptorsResponse{}, err + } + + if res.StatusCode/100 != 2 { + slog.Error("Request failed", "status", res.Status, "body", string(body)) + return MetricDescriptorsResponse{}, fmt.Errorf(string(body)) + } + + var data MetricDescriptorsResponse + err = json.Unmarshal(body, &data) + if err != nil { + slog.Error("Failed to unmarshal MetricDescriptorResponse", "error", err, "status", res.Status, "body", string(body)) + return MetricDescriptorsResponse{}, err + } + + return data, nil +} diff --git a/pkg/tsdb/stackdriver/types.go b/pkg/tsdb/stackdriver/types.go index 3821ce7ceda..de6daa5dc06 100644 --- a/pkg/tsdb/stackdriver/types.go +++ b/pkg/tsdb/stackdriver/types.go @@ -73,3 +73,24 @@ type StackdriverResponse struct { } `json:"points"` } `json:"timeSeries"` } + +type MetricDescriptorsResponse struct { + MetricDescriptors []struct { + Name string `json:"name"` + Labels []struct { + Key string `json:"key"` + Description string `json:"description"` + } `json:"labels,omitempty"` + MetricKind string `json:"metricKind"` + ValueType string `json:"valueType"` + Unit string `json:"unit,omitempty"` + Description string `json:"description"` + DisplayName string `json:"displayName"` + Type string `json:"type"` + Metadata struct { + LaunchStage string `json:"launchStage"` + SamplePeriod string `json:"samplePeriod"` + IngestDelay string `json:"ingestDelay"` + } `json:"metadata"` + } `json:"metricDescriptors"` +} diff --git a/public/app/plugins/datasource/stackdriver/datasource.ts b/public/app/plugins/datasource/stackdriver/datasource.ts index b77abdbdab3..dae9a55bea1 100644 --- a/public/app/plugins/datasource/stackdriver/datasource.ts +++ b/public/app/plugins/datasource/stackdriver/datasource.ts @@ -173,38 +173,53 @@ export default class StackdriverDatasource { throw new Error('Template variables support is not yet imlemented'); } - testDatasource() { - const path = `v3/projects/${this.projectName}/metricDescriptors`; - return this.doRequest(`${this.baseUrl}${path}`) - .then(response => { - if (response.status === 200) { - return { - status: 'success', - message: 'Successfully queried the Stackdriver API.', - title: 'Success', - }; - } + async testDatasource() { + const { data } = await this.backendSrv.datasourceRequest({ + url: '/api/tsdb/query', + method: 'POST', + data: { + queries: [ + { + refId: 'metricDescriptors', + datasourceId: this.id, + type: 'metricDescriptors', + }, + ], + }, + }); + console.log(data); + return data; + // const path = `v3/projects/${this.projectName}/metricDescriptors`; + // return this.doRequest(`${this.baseUrl}${path}`) + // .then(response => { + // if (response.status === 200) { + // return { + // status: 'success', + // message: 'Successfully queried the Stackdriver API.', + // title: 'Success', + // }; + // } - return { - status: 'error', - message: 'Returned http status code ' + response.status, - }; - }) - .catch(error => { - let message = 'Stackdriver: '; - message += error.statusText ? error.statusText + ': ' : ''; + // return { + // status: 'error', + // message: 'Returned http status code ' + response.status, + // }; + // }) + // .catch(error => { + // let message = 'Stackdriver: '; + // message += error.statusText ? error.statusText + ': ' : ''; - if (error.data && error.data.error && error.data.error.code) { - // 400, 401 - message += error.data.error.code + '. ' + error.data.error.message; - } else { - message += 'Cannot connect to Stackdriver API'; - } - return { - status: 'error', - message: message, - }; - }); + // if (error.data && error.data.error && error.data.error.code) { + // // 400, 401 + // message += error.data.error.code + '. ' + error.data.error.message; + // } else { + // message += 'Cannot connect to Stackdriver API'; + // } + // return { + // status: 'error', + // message: message, + // }; + // }); } async getProjects() { From 9c7022be86197107b4a070f7f7da627e178e0883 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Tue, 9 Oct 2018 13:35:50 +0200 Subject: [PATCH 14/48] stackdriver: add status code --- pkg/tsdb/stackdriver/metric_descriptors_query.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pkg/tsdb/stackdriver/metric_descriptors_query.go b/pkg/tsdb/stackdriver/metric_descriptors_query.go index 6c005d2966b..7cf4f2b97d2 100644 --- a/pkg/tsdb/stackdriver/metric_descriptors_query.go +++ b/pkg/tsdb/stackdriver/metric_descriptors_query.go @@ -22,7 +22,7 @@ func (e *StackdriverExecutor) executeMetricDescriptors(ctx context.Context, tsdb Results: make(map[string]*tsdb.QueryResult), } - req, err := e.createRequest(ctx, e.dsInfo, "metricDescriptors") + req, err := e.createRequest(ctx, e.dsInfo, "metricDescriptorss") if err != nil { slog.Error("Failed to create request", "error", err) return nil, fmt.Errorf("Failed to create request. error: %v", err) @@ -35,7 +35,11 @@ func (e *StackdriverExecutor) executeMetricDescriptors(ctx context.Context, tsdb data, err := e.unmarshalMetricDescriptors(res) if err != nil { - return nil, err + queryResult.ErrorString = fmt.Sprintf(`Status code: %d`, res.StatusCode) + logger.Info("error2", "ErrorString", queryResult.ErrorString) + queryResult.Error = err + result.Results[tsdbQuery.Queries[0].RefId] = queryResult + return result, nil } parts := strings.Split(req.URL.Path, "/") @@ -73,7 +77,7 @@ func (e *StackdriverExecutor) unmarshalMetricDescriptors(res *http.Response) (Me if res.StatusCode/100 != 2 { slog.Error("Request failed", "status", res.Status, "body", string(body)) - return MetricDescriptorsResponse{}, fmt.Errorf(string(body)) + return MetricDescriptorsResponse{}, fmt.Errorf(`Status code: %d - %s`, res.StatusCode, string(body)) } var data MetricDescriptorsResponse From dd9e9ed4191a5695eae4fcf38242eb360612ed83 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Tue, 9 Oct 2018 15:20:05 +0200 Subject: [PATCH 15/48] stackdriver: make backend query a pure test datasource function --- .../stackdriver/metric_descriptors_query.go | 91 ------------------- pkg/tsdb/stackdriver/stackdriver.go | 31 +------ .../datasource/stackdriver/datasource.ts | 83 ++++++++--------- 3 files changed, 40 insertions(+), 165 deletions(-) delete mode 100644 pkg/tsdb/stackdriver/metric_descriptors_query.go diff --git a/pkg/tsdb/stackdriver/metric_descriptors_query.go b/pkg/tsdb/stackdriver/metric_descriptors_query.go deleted file mode 100644 index 7cf4f2b97d2..00000000000 --- a/pkg/tsdb/stackdriver/metric_descriptors_query.go +++ /dev/null @@ -1,91 +0,0 @@ -package stackdriver - -import ( - "context" - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "strings" - - "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" - "github.com/grafana/grafana/pkg/components/simplejson" - "golang.org/x/net/context/ctxhttp" - - "github.com/grafana/grafana/pkg/tsdb" -) - -func (e *StackdriverExecutor) executeMetricDescriptors(ctx context.Context, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) { - logger.Info("metricDescriptors", "metricDescriptors", tsdbQuery.Queries[0].RefId) - queryResult := &tsdb.QueryResult{Meta: simplejson.New(), RefId: tsdbQuery.Queries[0].RefId} - result := &tsdb.Response{ - Results: make(map[string]*tsdb.QueryResult), - } - - req, err := e.createRequest(ctx, e.dsInfo, "metricDescriptorss") - if err != nil { - slog.Error("Failed to create request", "error", err) - return nil, fmt.Errorf("Failed to create request. error: %v", err) - } - res, err := ctxhttp.Do(ctx, e.httpClient, req) - if err != nil { - logger.Info("error2", err) - return nil, err - } - - data, err := e.unmarshalMetricDescriptors(res) - if err != nil { - queryResult.ErrorString = fmt.Sprintf(`Status code: %d`, res.StatusCode) - logger.Info("error2", "ErrorString", queryResult.ErrorString) - queryResult.Error = err - result.Results[tsdbQuery.Queries[0].RefId] = queryResult - return result, nil - } - - parts := strings.Split(req.URL.Path, "/") - defaultProject := parts[3] - - table := transformMetricDescriptorResponseToTable(data) - queryResult.Tables = append(queryResult.Tables, table) - result.Results[tsdbQuery.Queries[0].RefId] = queryResult - result.Results[tsdbQuery.Queries[0].RefId].Meta.Set("defaultProject", defaultProject) - - return result, nil -} - -func transformMetricDescriptorResponseToTable(data MetricDescriptorsResponse) *tsdb.Table { - table := &tsdb.Table{ - Columns: make([]tsdb.TableColumn, 1), - Rows: make([]tsdb.RowValues, 0), - } - table.Columns[0].Text = "metricDescriptor" - - for _, r := range data.MetricDescriptors { - values := make([]interface{}, 1) - values[0] = r - table.Rows = append(table.Rows, values) - } - return table -} - -func (e *StackdriverExecutor) unmarshalMetricDescriptors(res *http.Response) (MetricDescriptorsResponse, error) { - body, err := ioutil.ReadAll(res.Body) - defer res.Body.Close() - if err != nil { - return MetricDescriptorsResponse{}, err - } - - if res.StatusCode/100 != 2 { - slog.Error("Request failed", "status", res.Status, "body", string(body)) - return MetricDescriptorsResponse{}, fmt.Errorf(`Status code: %d - %s`, res.StatusCode, string(body)) - } - - var data MetricDescriptorsResponse - err = json.Unmarshal(body, &data) - if err != nil { - slog.Error("Failed to unmarshal MetricDescriptorResponse", "error", err, "status", res.Status, "body", string(body)) - return MetricDescriptorsResponse{}, err - } - - return data, nil -} diff --git a/pkg/tsdb/stackdriver/stackdriver.go b/pkg/tsdb/stackdriver/stackdriver.go index d36487e344a..2674e3a9902 100644 --- a/pkg/tsdb/stackdriver/stackdriver.go +++ b/pkg/tsdb/stackdriver/stackdriver.go @@ -15,10 +15,7 @@ import ( "strings" "time" - "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" - "golang.org/x/net/context/ctxhttp" - "golang.org/x/oauth2/google" "github.com/grafana/grafana/pkg/api/pluginproxy" "github.com/grafana/grafana/pkg/components/null" @@ -75,7 +72,7 @@ func (e *StackdriverExecutor) Query(ctx context.Context, dsInfo *models.DataSour case "annotationQuery": result, err = e.executeAnnotationQuery(ctx, tsdbQuery) case "metricDescriptors": - result, err = e.executeMetricDescriptors(ctx, tsdbQuery) + result, err = e.executeTestDataSource(ctx, tsdbQuery) case "timeSeriesQuery": fallthrough default: @@ -521,25 +518,6 @@ func replaceWithMetricPart(metaPartName string, metricType string) []byte { return nil } -func getProjectName(ctx context.Context, dsInfo *models.DataSource, route *plugins.AppPluginRoute) (string, error) { - var projectName string - gceAutomaticAuthentication := dsInfo.JsonData.Get("gceAutomaticAuthentication").MustBool() - logger.Info("gceAutomaticAuthentication", "gceAutomaticAuthentication", gceAutomaticAuthentication) - if gceAutomaticAuthentication { - defaultCredentials, err := google.FindDefaultCredentials(ctx, route.JwtTokenAuth.Scopes...) - if err != nil { - // return "", err - projectName = "raintank-dev" - } else { - projectName = defaultCredentials.ProjectID - } - } else { - projectName = dsInfo.JsonData.Get("defaultProject").MustString() - } - logger.Info("projectName", "projectName", projectName) - return projectName, nil -} - func calcBucketBound(bucketOptions StackdriverBucketOptions, n int) string { bucketBound := "0" if n == 0 { @@ -583,12 +561,7 @@ func (e *StackdriverExecutor) createRequest(ctx context.Context, dsInfo *models. } } - // projectName := dsInfo.JsonData.Get("defaultProject").MustString() - // logger.Info("projectName", "projectName", projectName) - projectName, err := getProjectName(ctx, dsInfo, stackdriverRoute) - if err != nil { - return nil, err - } + projectName := dsInfo.JsonData.Get("defaultProject").MustString() proxyPass := fmt.Sprintf("stackdriver%s", "v3/projects/"+projectName+"/"+endpointName) pluginproxy.ApplyRoute(ctx, req, proxyPass, stackdriverRoute, dsInfo) diff --git a/public/app/plugins/datasource/stackdriver/datasource.ts b/public/app/plugins/datasource/stackdriver/datasource.ts index dae9a55bea1..c7ce91a0227 100644 --- a/public/app/plugins/datasource/stackdriver/datasource.ts +++ b/public/app/plugins/datasource/stackdriver/datasource.ts @@ -174,52 +174,45 @@ export default class StackdriverDatasource { } async testDatasource() { - const { data } = await this.backendSrv.datasourceRequest({ - url: '/api/tsdb/query', - method: 'POST', - data: { - queries: [ - { - refId: 'metricDescriptors', - datasourceId: this.id, - type: 'metricDescriptors', - }, - ], - }, - }); - console.log(data); - return data; - // const path = `v3/projects/${this.projectName}/metricDescriptors`; - // return this.doRequest(`${this.baseUrl}${path}`) - // .then(response => { - // if (response.status === 200) { - // return { - // status: 'success', - // message: 'Successfully queried the Stackdriver API.', - // title: 'Success', - // }; - // } + try { + await this.backendSrv.datasourceRequest({ + url: '/api/tsdb/query', + method: 'POST', + data: { + queries: [ + { + refId: 'metricDescriptors', + datasourceId: this.id, + type: 'metricDescriptors', + }, + ], + }, + }); + return { + status: 'success', + message: 'Successfully queried the Stackdriver API.', + title: 'Success', + }; + } catch (error) { + console.log(error.data.error); + let message = 'Stackdriver: '; + message += error.statusText ? error.statusText + ': ' : ''; - // return { - // status: 'error', - // message: 'Returned http status code ' + response.status, - // }; - // }) - // .catch(error => { - // let message = 'Stackdriver: '; - // message += error.statusText ? error.statusText + ': ' : ''; - - // if (error.data && error.data.error && error.data.error.code) { - // // 400, 401 - // message += error.data.error.code + '. ' + error.data.error.message; - // } else { - // message += 'Cannot connect to Stackdriver API'; - // } - // return { - // status: 'error', - // message: message, - // }; - // }); + if (error.data && error.data.error && error.data.error) { + try { + const res = JSON.parse(error.data.error); + message += res.error.code + '. ' + res.error.message; + } catch (err) { + message += error.data.error; + } + } else { + message += 'Cannot connect to Stackdriver API'; + } + return { + status: 'error', + message: message, + }; + } } async getProjects() { From 2b1709b8750dc63bdb90350c81439d9c4c77d2b8 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Tue, 9 Oct 2018 15:23:13 +0200 Subject: [PATCH 16/48] stackdriver: only get default token from metadata server when applying route --- pkg/api/pluginproxy/ds_auth_provider.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/pkg/api/pluginproxy/ds_auth_provider.go b/pkg/api/pluginproxy/ds_auth_provider.go index d3b628b9de4..3b7c6ee5c0f 100644 --- a/pkg/api/pluginproxy/ds_auth_provider.go +++ b/pkg/api/pluginproxy/ds_auth_provider.go @@ -12,6 +12,7 @@ import ( m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/util" + "golang.org/x/oauth2/google" ) //ApplyRoute should use the plugin route data to set auth headers and custom headers @@ -64,18 +65,16 @@ func ApplyRoute(ctx context.Context, req *http.Request, proxyPath string, route } } - if gceAutoAuthentication { - // tokenSrc, err := google.DefaultTokenSource(ctx, route.JwtTokenAuth.Scopes...) + if gceAutoAuthentication && route.JwtTokenAuth == nil { + tokenSrc, err := google.DefaultTokenSource(ctx, route.JwtTokenAuth.Scopes...) if err != nil { - logger.Error("Failed to get default credentials", "error", err) + logger.Error("Failed to get default token from meta data server", "error", err) } else { - // token, err := tokenSrc.Token() - token, err := tokenProvider.getJwtAccessToken(ctx, data) + token, err := tokenSrc.Token() if err != nil { - logger.Error("Failed to get default access token", "error", err) + logger.Error("Failed to get default access token from meta data server", "error", err) } else { - // req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token.AccessToken)) - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token.AccessToken)) } } } From 0b82fda49ce52845a2bd0ee027427d3a15dc2334 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Tue, 9 Oct 2018 16:05:35 +0200 Subject: [PATCH 17/48] stackdriver: only get default token from metadata server when applying route --- pkg/tsdb/stackdriver/test_datasource_query.go | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 pkg/tsdb/stackdriver/test_datasource_query.go diff --git a/pkg/tsdb/stackdriver/test_datasource_query.go b/pkg/tsdb/stackdriver/test_datasource_query.go new file mode 100644 index 00000000000..0bebea47c7a --- /dev/null +++ b/pkg/tsdb/stackdriver/test_datasource_query.go @@ -0,0 +1,105 @@ +package stackdriver + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "strings" + + "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" + "github.com/grafana/grafana/pkg/components/simplejson" + "golang.org/x/net/context/ctxhttp" + "golang.org/x/oauth2/google" + + "github.com/grafana/grafana/pkg/tsdb" +) + +func (e *StackdriverExecutor) executeTestDataSource(ctx context.Context, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) { + gceAutomaticAuthentication := e.dsInfo.JsonData.Get("gceAutomaticAuthentication").MustBool() + if gceAutomaticAuthentication { + defaultProject, err := e.getDefaultProject(ctx) + if err != nil { + return nil, err + } + e.dsInfo.JsonData.Set("defaultProject", defaultProject) + } + + queryResult := &tsdb.QueryResult{Meta: simplejson.New(), RefId: tsdbQuery.Queries[0].RefId} + result := &tsdb.Response{ + Results: make(map[string]*tsdb.QueryResult), + } + + req, err := e.createRequest(ctx, e.dsInfo, "metricDescriptors") + if err != nil { + slog.Error("Failed to create request", "error", err) + return nil, fmt.Errorf("Failed to create request. error: %v", err) + } + res, err := ctxhttp.Do(ctx, e.httpClient, req) + if err != nil { + return nil, err + } + + data, err := e.unmarshalMetricDescriptors(res) + if err != nil { + return nil, err + } + + parts := strings.Split(req.URL.Path, "/") + defaultProject := parts[3] + + table := transformMetricDescriptorResponseToTable(data) + queryResult.Tables = append(queryResult.Tables, table) + result.Results[tsdbQuery.Queries[0].RefId] = queryResult + result.Results[tsdbQuery.Queries[0].RefId].Meta.Set("defaultProject", defaultProject) + + return result, nil +} + +func (e *StackdriverExecutor) getDefaultProject(ctx context.Context) (string, error) { + defaultCredentials, err := google.FindDefaultCredentials(ctx, "https://www.googleapis.com/auth/monitoring.read") + if err != nil { + return "", err + } else { + logger.Info("projectName", "projectName", defaultCredentials.ProjectID) + return defaultCredentials.ProjectID, nil + } +} + +func transformMetricDescriptorResponseToTable(data MetricDescriptorsResponse) *tsdb.Table { + table := &tsdb.Table{ + Columns: make([]tsdb.TableColumn, 1), + Rows: make([]tsdb.RowValues, 0), + } + table.Columns[0].Text = "metricDescriptor" + + for _, r := range data.MetricDescriptors { + values := make([]interface{}, 1) + values[0] = r + table.Rows = append(table.Rows, values) + } + return table +} + +func (e *StackdriverExecutor) unmarshalMetricDescriptors(res *http.Response) (MetricDescriptorsResponse, error) { + body, err := ioutil.ReadAll(res.Body) + defer res.Body.Close() + if err != nil { + return MetricDescriptorsResponse{}, err + } + + if res.StatusCode/100 != 2 { + slog.Error("Request failed", "status", res.Status, "body", string(body)) + return MetricDescriptorsResponse{}, fmt.Errorf(`%s`, string(body)) + } + + var data MetricDescriptorsResponse + err = json.Unmarshal(body, &data) + if err != nil { + slog.Error("Failed to unmarshal MetricDescriptorResponse", "error", err, "status", res.Status, "body", string(body)) + return MetricDescriptorsResponse{}, err + } + + return data, nil +} From aa2e64e1603dfc7487ac48d0eaba2c7f00ce10cd Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Tue, 9 Oct 2018 16:06:59 +0200 Subject: [PATCH 18/48] stackdriver: improve ui for toggling authentication type --- .../datasource/stackdriver/config_ctrl.ts | 12 ++++++- .../stackdriver/partials/config.html | 31 +++++++++++-------- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/public/app/plugins/datasource/stackdriver/config_ctrl.ts b/public/app/plugins/datasource/stackdriver/config_ctrl.ts index d3847562156..13dba26ba86 100644 --- a/public/app/plugins/datasource/stackdriver/config_ctrl.ts +++ b/public/app/plugins/datasource/stackdriver/config_ctrl.ts @@ -5,13 +5,23 @@ export class StackdriverConfigCtrl { jsonText: string; validationErrors: string[] = []; inputDataValid: boolean; + authenticationTypes: any[]; + defaultAuthenticationType: string; /** @ngInject */ constructor(datasourceSrv) { + this.defaultAuthenticationType = 'jwt'; this.datasourceSrv = datasourceSrv; this.current.jsonData = this.current.jsonData || {}; + this.current.jsonData.authenticationType = this.current.jsonData.authenticationType + ? this.current.jsonData.authenticationType + : this.defaultAuthenticationType; this.current.secureJsonData = this.current.secureJsonData || {}; this.current.secureJsonFields = this.current.secureJsonFields || {}; + this.authenticationTypes = [ + { key: this.defaultAuthenticationType, value: 'Google JWT File' }, + { key: 'gce', value: 'Use GCE default Authentication' }, + ]; } save(jwt) { @@ -67,7 +77,7 @@ export class StackdriverConfigCtrl { this.inputDataValid = false; this.jsonText = ''; - this.current.jsonData = {}; + this.current.jsonData = Object.assign({}, { authenticationType: this.current.jsonData.authenticationType }); this.current.secureJsonData = {}; this.current.secureJsonFields = {}; } diff --git a/public/app/plugins/datasource/stackdriver/partials/config.html b/public/app/plugins/datasource/stackdriver/partials/config.html index 6fd52a898f5..0cc08a3e9d8 100644 --- a/public/app/plugins/datasource/stackdriver/partials/config.html +++ b/public/app/plugins/datasource/stackdriver/partials/config.html @@ -31,10 +31,16 @@ contents will be encrypted and saved in the Grafana database.
- +
+
+ Authentication Type +
+ +
+
+
-
+
@@ -55,23 +61,23 @@
-
+
Uploaded Key Details
- Project + Project
- Client Email + Client Email
- Token URI + Token URI
- Private Key + Private Key
@@ -84,9 +90,8 @@
-

Do not forget to save your changes after uploading a file.

+

Do not forget to - -

- Do not forget to save your changes after uploading a file. -
+

+ Verify GCE automatic authentication by clicking Save & Test

From 823838a83918f4e47b9c3a2e09b3364188adccd1 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Tue, 9 Oct 2018 16:28:04 +0200 Subject: [PATCH 19/48] stackdriver: wip - remove debug code --- pkg/api/pluginproxy/ds_auth_provider.go | 9 +++++---- pkg/tsdb/stackdriver/test_datasource_query.go | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pkg/api/pluginproxy/ds_auth_provider.go b/pkg/api/pluginproxy/ds_auth_provider.go index 3b7c6ee5c0f..194e4e8e964 100644 --- a/pkg/api/pluginproxy/ds_auth_provider.go +++ b/pkg/api/pluginproxy/ds_auth_provider.go @@ -55,9 +55,9 @@ func ApplyRoute(ctx context.Context, req *http.Request, proxyPath string, route } } - gceAutoAuthentication := ds.JsonData.Get("gceAutomaticAuthentication").MustBool() - logger.Info("gceAutoAuthentication", "gceAutoAuthentication", gceAutoAuthentication) - if route.JwtTokenAuth != nil && !gceAutoAuthentication { + authenticationType := ds.JsonData.Get("authenticationType").MustString() + if route.JwtTokenAuth != nil && authenticationType == "jwt" { + logger.Info("authenticationType", "authenticationType", "jwt") if token, err := tokenProvider.getJwtAccessToken(ctx, data); err != nil { logger.Error("Failed to get access token", "error", err) } else { @@ -65,7 +65,8 @@ func ApplyRoute(ctx context.Context, req *http.Request, proxyPath string, route } } - if gceAutoAuthentication && route.JwtTokenAuth == nil { + if authenticationType == "gce" { + logger.Info("authenticationType", "authenticationType", "gce") tokenSrc, err := google.DefaultTokenSource(ctx, route.JwtTokenAuth.Scopes...) if err != nil { logger.Error("Failed to get default token from meta data server", "error", err) diff --git a/pkg/tsdb/stackdriver/test_datasource_query.go b/pkg/tsdb/stackdriver/test_datasource_query.go index 0bebea47c7a..822ca725f49 100644 --- a/pkg/tsdb/stackdriver/test_datasource_query.go +++ b/pkg/tsdb/stackdriver/test_datasource_query.go @@ -17,8 +17,8 @@ import ( ) func (e *StackdriverExecutor) executeTestDataSource(ctx context.Context, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) { - gceAutomaticAuthentication := e.dsInfo.JsonData.Get("gceAutomaticAuthentication").MustBool() - if gceAutomaticAuthentication { + authenticationType := e.dsInfo.JsonData.Get("authenticationType").MustString() + if authenticationType == "gce" { defaultProject, err := e.getDefaultProject(ctx) if err != nil { return nil, err From cd7b454e25645e86a44ea26d3cea675707a75f50 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Tue, 9 Oct 2018 16:36:45 +0200 Subject: [PATCH 20/48] stackdriver: add default value for authentication type --- pkg/api/pluginproxy/ds_auth_provider.go | 2 +- pkg/tsdb/stackdriver/test_datasource_query.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/api/pluginproxy/ds_auth_provider.go b/pkg/api/pluginproxy/ds_auth_provider.go index 194e4e8e964..60360733923 100644 --- a/pkg/api/pluginproxy/ds_auth_provider.go +++ b/pkg/api/pluginproxy/ds_auth_provider.go @@ -55,7 +55,7 @@ func ApplyRoute(ctx context.Context, req *http.Request, proxyPath string, route } } - authenticationType := ds.JsonData.Get("authenticationType").MustString() + authenticationType := ds.JsonData.Get("authenticationType").MustString("jwt") if route.JwtTokenAuth != nil && authenticationType == "jwt" { logger.Info("authenticationType", "authenticationType", "jwt") if token, err := tokenProvider.getJwtAccessToken(ctx, data); err != nil { diff --git a/pkg/tsdb/stackdriver/test_datasource_query.go b/pkg/tsdb/stackdriver/test_datasource_query.go index 822ca725f49..4ea79a79e43 100644 --- a/pkg/tsdb/stackdriver/test_datasource_query.go +++ b/pkg/tsdb/stackdriver/test_datasource_query.go @@ -17,7 +17,7 @@ import ( ) func (e *StackdriverExecutor) executeTestDataSource(ctx context.Context, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) { - authenticationType := e.dsInfo.JsonData.Get("authenticationType").MustString() + authenticationType := e.dsInfo.JsonData.Get("authenticationType").MustString("jwt") if authenticationType == "gce" { defaultProject, err := e.getDefaultProject(ctx) if err != nil { From 9fb1d3a9a64077635c81abc2a11545395695a922 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Tue, 9 Oct 2018 17:02:41 +0200 Subject: [PATCH 21/48] stackdriver: only load project name if it's not already stored in the ds info json --- .../datasource/stackdriver/datasource.ts | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/public/app/plugins/datasource/stackdriver/datasource.ts b/public/app/plugins/datasource/stackdriver/datasource.ts index c7ce91a0227..91b2d19ba6d 100644 --- a/public/app/plugins/datasource/stackdriver/datasource.ts +++ b/public/app/plugins/datasource/stackdriver/datasource.ts @@ -222,12 +222,19 @@ export default class StackdriverDatasource { async getDefaultProject() { try { - const projects = await this.getProjects(); - if (projects && projects.length > 0) { - const test = projects.filter(p => p.id === this.projectName)[0]; - return test; + if (this.projectName) { + return { + id: this.projectName, + name: this.projectName, + }; } else { - throw new Error('No projects found'); + const projects = await this.getProjects(); + if (projects && projects.length > 0) { + const test = projects.filter(p => p.id === this.projectName)[0]; + return test; + } else { + throw new Error('No projects found'); + } } } catch (error) { let message = 'Projects cannot be fetched: '; From 835f1d788e1f767bb788a781f7c0bb98c1edcaae Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 10 Oct 2018 11:01:02 +0200 Subject: [PATCH 22/48] stackdriver: display error message if project name is not present in jwt file --- public/app/plugins/datasource/stackdriver/config_ctrl.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/public/app/plugins/datasource/stackdriver/config_ctrl.ts b/public/app/plugins/datasource/stackdriver/config_ctrl.ts index 13dba26ba86..9649a8772e9 100644 --- a/public/app/plugins/datasource/stackdriver/config_ctrl.ts +++ b/public/app/plugins/datasource/stackdriver/config_ctrl.ts @@ -45,6 +45,10 @@ export class StackdriverConfigCtrl { this.validationErrors.push('Client Email field missing in JWT file.'); } + if (!jwt.project_id || jwt.project_id.length === 0) { + this.validationErrors.push('Project Id field missing in JWT file.'); + } + if (this.validationErrors.length === 0) { this.inputDataValid = true; return true; From 09fb1760a1d7bd05c6ad2442701ddd9fc6ee0aed Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 10 Oct 2018 11:04:06 +0200 Subject: [PATCH 23/48] stackdriver: only load default project from backend if it's not available on the target. this might happen when using gce authentication and provisioning --- .../datasource/stackdriver/datasource.ts | 78 ++++++++----------- .../stackdriver/partials/query.editor.html | 7 +- .../datasource/stackdriver/query_ctrl.ts | 10 +-- .../stackdriver/query_filter_ctrl.ts | 11 ++- 4 files changed, 46 insertions(+), 60 deletions(-) diff --git a/public/app/plugins/datasource/stackdriver/datasource.ts b/public/app/plugins/datasource/stackdriver/datasource.ts index 91b2d19ba6d..6e079a59fa3 100644 --- a/public/app/plugins/datasource/stackdriver/datasource.ts +++ b/public/app/plugins/datasource/stackdriver/datasource.ts @@ -6,6 +6,7 @@ export default class StackdriverDatasource { url: string; baseUrl: string; projectName: string; + queryPromise: Promise; /** @ngInject */ constructor(instanceSettings, private backendSrv, private templateSrv, private timeSrv) { @@ -99,31 +100,34 @@ export default class StackdriverDatasource { } async query(options) { - const result = []; - const data = await this.getTimeSeries(options); - if (data.results) { - Object['values'](data.results).forEach(queryRes => { - if (!queryRes.series) { - return; - } - - const unit = this.resolvePanelUnitFromTargets(options.targets); - queryRes.series.forEach(series => { - let timeSerie: any = { - target: series.name, - datapoints: series.points, - refId: queryRes.refId, - meta: queryRes.meta, - }; - if (unit) { - timeSerie = { ...timeSerie, unit }; + this.queryPromise = new Promise(async resolve => { + const result = []; + const data = await this.getTimeSeries(options); + if (data.results) { + Object['values'](data.results).forEach(queryRes => { + if (!queryRes.series) { + return; } - result.push(timeSerie); + this.projectName = queryRes.meta.defaultProject; + const unit = this.resolvePanelUnitFromTargets(options.targets); + queryRes.series.forEach(series => { + let timeSerie: any = { + target: series.name, + datapoints: series.points, + refId: queryRes.refId, + meta: queryRes.meta, + }; + if (unit) { + timeSerie = { ...timeSerie, unit }; + } + result.push(timeSerie); + }); }); - }); - } + } - return { data: result }; + resolve({ data: result }); + }); + return this.queryPromise; } async annotationQuery(options) { @@ -181,9 +185,9 @@ export default class StackdriverDatasource { data: { queries: [ { - refId: 'metricDescriptors', + refId: 'testDatasource', datasourceId: this.id, - type: 'metricDescriptors', + type: 'testDatasource', }, ], }, @@ -215,27 +219,10 @@ export default class StackdriverDatasource { } } - async getProjects() { - const response = await this.doRequest(`/cloudresourcemanager/v1/projects`); - return response.data.projects.map(p => ({ id: p.projectId, name: p.name })); - } - async getDefaultProject() { try { - if (this.projectName) { - return { - id: this.projectName, - name: this.projectName, - }; - } else { - const projects = await this.getProjects(); - if (projects && projects.length > 0) { - const test = projects.filter(p => p.id === this.projectName)[0]; - return test; - } else { - throw new Error('No projects found'); - } - } + await this.queryPromise; + return this.projectName; } catch (error) { let message = 'Projects cannot be fetched: '; message += error.statusText ? error.statusText + ': ' : ''; @@ -252,12 +239,13 @@ export default class StackdriverDatasource { message += 'Cannot connect to Stackdriver API'; } appEvents.emit('ds-request-error', message); + return ''; } } - async getMetricTypes(projectId: string) { + async getMetricTypes(projectName: string) { try { - const metricsApiPath = `v3/projects/${projectId}/metricDescriptors`; + const metricsApiPath = `v3/projects/${projectName}/metricDescriptors`; const { data } = await this.doRequest(`${this.baseUrl}${metricsApiPath}`); const metrics = data.metricDescriptors.map(m => { diff --git a/public/app/plugins/datasource/stackdriver/partials/query.editor.html b/public/app/plugins/datasource/stackdriver/partials/query.editor.html index 7cc93e9d1a8..98c8fcc83e8 100755 --- a/public/app/plugins/datasource/stackdriver/partials/query.editor.html +++ b/public/app/plugins/datasource/stackdriver/partials/query.editor.html @@ -15,8 +15,7 @@
Project - +