CloudMonitoring: Move data manipulation to backend (#41379)

go.mod

@@ -144,6 +144,7 @@ require (
 	github.com/PuerkitoBio/purell v1.1.1 // indirect
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/alecthomas/units v0.0.0-20210912230133-d1bdfacee922 // indirect
+	github.com/andybalholm/brotli v1.0.3
 	github.com/apache/arrow/go/arrow v0.0.0-20210223225224-5bea62493d91 // indirect
 	github.com/armon/go-metrics v0.3.8 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
@@ -189,7 +190,7 @@ require (
 	github.com/googleapis/gax-go/v2 v2.1.0 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/grafana/dskit v0.0.0-20211011144203-3a88ec0b675f // indirect
-	github.com/grafana/grafana-google-sdk-go v0.0.0-20211019132340-3ff525a010d5
+	github.com/grafana/grafana-google-sdk-go v0.0.0-20211104130251-b190293eaf58
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.1-0.20191002090509-6af20e3a5340 // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.0 // indirect

go.sum

@@ -265,6 +265,8 @@ github.com/alicebob/miniredis/v2 v2.14.3/go.mod h1:gquAfGbzn92jvtrSC69+6zZnwSODV
 github.com/aliyun/aliyun-oss-go-sdk v2.0.4+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
 github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9/go.mod h1:eliMa/PW+RDr2QLWRmLH1R1ZA4RInpmvOzDDXtaIZkc=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
+github.com/andybalholm/brotli v1.0.3 h1:fpcw+r1N1h0Poc1F/pHbW40cUm/lMEQslZtCkBQ0UnM=
+github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/antonmedv/expr v1.8.9/go.mod h1:5qsM3oLGDND7sDmQGDXHkYfkjYMUX14qsgqmHhwGEk8=
@@ -1219,8 +1221,8 @@ github.com/grafana/go-mssqldb v0.0.0-20210326084033-d0ce3c521036 h1:GplhUk6Xes5J
 github.com/grafana/go-mssqldb v0.0.0-20210326084033-d0ce3c521036/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
 github.com/grafana/grafana-aws-sdk v0.7.0 h1:D+Lhxi3P/7vpyDHUK/fdX9bL2mRz8hLG04ucNf1E02o=
 github.com/grafana/grafana-aws-sdk v0.7.0/go.mod h1:+pPo5U+pX0zWimR7YBc7ASeSQfbRkcTyQYqMiAj7G5U=
-github.com/grafana/grafana-google-sdk-go v0.0.0-20211019132340-3ff525a010d5 h1:o7w/t0nLNfkERMdj09U0h3Fl63z8ws1CxwiImeUKLIk=
+github.com/grafana/grafana-google-sdk-go v0.0.0-20211104130251-b190293eaf58 h1:2ud7NNM7LrGPO4x0NFR8qLq68CqI4SmB7I2yRN2w9oE=
-github.com/grafana/grafana-google-sdk-go v0.0.0-20211019132340-3ff525a010d5/go.mod h1:Vo2TKWfDVmNTELBUM+3lkrZvFtBws0qSZdXhQxRdJrE=
+github.com/grafana/grafana-google-sdk-go v0.0.0-20211104130251-b190293eaf58/go.mod h1:Vo2TKWfDVmNTELBUM+3lkrZvFtBws0qSZdXhQxRdJrE=
 github.com/grafana/grafana-plugin-sdk-go v0.79.0/go.mod h1:NvxLzGkVhnoBKwzkst6CFfpMFKwAdIUZ1q8ssuLeF60=
 github.com/grafana/grafana-plugin-sdk-go v0.114.0 h1:9I55IXw7mOT71tZ/pdqCaWGz8vxfz31CXjaDtBV9ZBo=
 github.com/grafana/grafana-plugin-sdk-go v0.114.0/go.mod h1:D7x3ah+1d4phNXpbnOaxa/osSaZlwh9/ZUnGGzegRbk=

pkg/tsdb/cloudmonitoring/cloudmonitoring.go

@@ -3,7 +3,6 @@ package cloudmonitoring
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -16,8 +15,7 @@ import (
 	"strings"
 	"time"
 
-	"golang.org/x/oauth2/google"
+	"github.com/grafana/grafana-google-sdk-go/pkg/utils"
-
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	"github.com/grafana/grafana-plugin-sdk-go/backend/datasource"
 	"github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt"
@@ -87,6 +85,7 @@ func ProvideService(cfg *setting.Cfg, httpClientProvider httpclient.Provider, re
 	factory := coreplugin.New(backend.ServeOpts{
 		QueryDataHandler:    s,
 		CallResourceHandler: httpadapter.New(mux),
+		CheckHealthHandler:  s,
 	})
 
 	if err := registrar.LoadAndRegister(pluginID, factory); err != nil {
@@ -95,6 +94,45 @@ func ProvideService(cfg *setting.Cfg, httpClientProvider httpclient.Provider, re
 	return s
 }
 
+func (s *Service) CheckHealth(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) {
+	dsInfo, err := s.getDSInfo(req.PluginContext)
+	if err != nil {
+		return nil, err
+	}
+
+	defaultProject, err := s.getDefaultProject(ctx, *dsInfo)
+	if err != nil {
+		return nil, err
+	}
+
+	url := fmt.Sprintf("%v/v3/projects/%v/metricDescriptors", dsInfo.services[cloudMonitor].url, defaultProject)
+	request, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	res, err := dsInfo.services[cloudMonitor].client.Do(request)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err := res.Body.Close(); err != nil {
+			slog.Warn("Failed to close response body", "err", err)
+		}
+	}()
+
+	status := backend.HealthStatusOk
+	message := "Successfully queried the Google Cloud Monitoring API."
+	if res.StatusCode != 200 {
+		status = backend.HealthStatusError
+		message = res.Status
+	}
+	return &backend.CheckHealthResult{
+		Status:  status,
+		Message: message,
+	}, nil
+}
+
 type Service struct {
 	httpClientProvider httpclient.Provider
 	cfg                *setting.Cfg
@@ -206,8 +244,6 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest)
 	switch model.Type {
 	case "annotationQuery":
 		resp, err = s.executeAnnotationQuery(ctx, req, *dsInfo)
-	case "getGCEDefaultProject":
-		resp, err = s.getGCEDefaultProject(ctx, req, *dsInfo)
 	case "timeSeriesQuery":
 		fallthrough
 	default:
@@ -217,26 +253,6 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest)
 	return resp, err
 }
 
-func (s *Service) getGCEDefaultProject(ctx context.Context, req *backend.QueryDataRequest, dsInfo datasourceInfo) (*backend.QueryDataResponse, error) {
-	gceDefaultProject, err := s.getDefaultProject(ctx, dsInfo)
-	if err != nil {
-		return backend.NewQueryDataResponse(), fmt.Errorf(
-			"failed to retrieve default project from GCE metadata server, error: %w", err)
-	}
-
-	return &backend.QueryDataResponse{
-		Responses: backend.Responses{
-			req.Queries[0].RefID: {
-				Frames: data.Frames{data.NewFrame("").SetMeta(&data.FrameMeta{
-					Custom: map[string]interface{}{
-						"defaultProject": gceDefaultProject,
-					},
-				})},
-			},
-		},
-	}, nil
-}
-
 func (s *Service) executeTimeSeriesQuery(ctx context.Context, req *backend.QueryDataRequest, dsInfo datasourceInfo) (
 	*backend.QueryDataResponse, error) {
 	resp := backend.NewQueryDataResponse()
@@ -606,19 +622,7 @@ func (s *Service) createRequest(ctx context.Context, dsInfo *datasourceInfo, pro
 
 func (s *Service) getDefaultProject(ctx context.Context, dsInfo datasourceInfo) (string, error) {
 	if dsInfo.authenticationType == gceAuthentication {
-		defaultCredentials, err := google.FindDefaultCredentials(ctx, "https://www.googleapis.com/auth/monitoring.read")
+		return utils.GCEDefaultProject(ctx)
-		if err != nil {
-			return "", fmt.Errorf("failed to retrieve default project from GCE metadata server: %w", err)
-		}
-		token, err := defaultCredentials.TokenSource.Token()
-		if err != nil {
-			return "", fmt.Errorf("failed to retrieve GCP credential token: %w", err)
-		}
-		if !token.Valid() {
-			return "", errors.New("failed to validate GCP credentials")
-		}
-
-		return defaultCredentials.ProjectID, nil
 	}
 	return dsInfo.defaultProject, nil
 }

pkg/tsdb/cloudmonitoring/resource_handler.go

@@ -1,31 +1,244 @@
 package cloudmonitoring
 
 import (
+	"bytes"
+	"compress/flate"
+	"compress/gzip"
+	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"net/url"
+	"regexp"
 	"strings"
 
+	"github.com/andybalholm/brotli"
+	"github.com/grafana/grafana-google-sdk-go/pkg/utils"
 	"github.com/grafana/grafana-plugin-sdk-go/backend/resource/httpadapter"
 )
 
+// nameExp matches the part after the last '/' symbol
+var nameExp = regexp.MustCompile(`([^\/]*)\/*$`)
+
+const resourceManagerPath = "/v1/projects"
+
+type processResponse func(body []byte) ([]byte, error)
+
 func (s *Service) registerRoutes(mux *http.ServeMux) {
-	mux.HandleFunc("/cloudmonitoring/", s.resourceHandler(cloudMonitor))
+	mux.HandleFunc("/gceDefaultProject", getGCEDefaultProject)
-	mux.HandleFunc("/cloudresourcemanager/", s.resourceHandler(resourceManager))
+
+	mux.HandleFunc("/metricDescriptors/", s.resourceHandler(cloudMonitor, processMetricDescriptors))
+	mux.HandleFunc("/services/", s.resourceHandler(cloudMonitor, processServices))
+	mux.HandleFunc("/slo-services/", s.resourceHandler(cloudMonitor, processSLOs))
+	mux.HandleFunc("/projects", s.resourceHandler(resourceManager, processProjects))
 }
 
-func (s *Service) resourceHandler(subDataSource string) func(rw http.ResponseWriter, req *http.Request) {
+func getGCEDefaultProject(rw http.ResponseWriter, req *http.Request) {
+	project, err := utils.GCEDefaultProject(req.Context())
+	if err != nil {
+		writeResponse(rw, http.StatusBadRequest, fmt.Sprintf("unexpected error %v", err))
+		return
+	}
+	writeResponse(rw, http.StatusOK, project)
+}
+
+func (s *Service) resourceHandler(subDataSource string, responseFn processResponse) func(rw http.ResponseWriter, req *http.Request) {
 	return func(rw http.ResponseWriter, req *http.Request) {
 		client, code, err := s.setRequestVariables(req, subDataSource)
 		if err != nil {
 			writeResponse(rw, code, fmt.Sprintf("unexpected error %v", err))
 			return
 		}
-		doRequest(rw, req, client)
+		s.doRequest(rw, req, client, responseFn)
 	}
 }
 
+func (s *Service) doRequest(rw http.ResponseWriter, req *http.Request, cli *http.Client, responseFn processResponse) http.ResponseWriter {
+	res, err := cli.Do(req)
+	if err != nil {
+		writeResponse(rw, http.StatusBadRequest, fmt.Sprintf("unexpected error %v", err))
+		return rw
+	}
+	defer func() {
+		if err := res.Body.Close(); err != nil {
+			slog.Warn("Failed to close response body", "err", err)
+		}
+	}()
+
+	if responseFn == nil {
+		writeResponse(rw, http.StatusInternalServerError, "responseFn should not be nil")
+		return rw
+	}
+
+	body, code, err := processData(res, responseFn)
+	if err != nil {
+		writeResponse(rw, code, fmt.Sprintf("unexpected error %v", err))
+		return rw
+	}
+	writeResponseBytes(rw, res.StatusCode, body)
+
+	for k, v := range res.Header {
+		rw.Header().Set(k, v[0])
+		for _, v := range v[1:] {
+			rw.Header().Add(k, v)
+		}
+	}
+	return rw
+}
+
+func processMetricDescriptors(body []byte) ([]byte, error) {
+	resp := metricDescriptorResponse{}
+	err := json.Unmarshal(body, &resp)
+	if err != nil {
+		return nil, err
+	}
+
+	for i := range resp.Descriptors {
+		resp.Descriptors[i].Service = strings.SplitN(resp.Descriptors[i].Type, "/", 2)[0]
+		resp.Descriptors[i].ServiceShortName = strings.SplitN(resp.Descriptors[i].Service, ".", 2)[0]
+		if resp.Descriptors[i].DisplayName == "" {
+			resp.Descriptors[i].DisplayName = resp.Descriptors[i].Type
+		}
+	}
+	return json.Marshal(resp.Descriptors)
+}
+
+func processServices(body []byte) ([]byte, error) {
+	resp := serviceResponse{}
+	err := json.Unmarshal(body, &resp)
+	if err != nil {
+		return nil, err
+	}
+
+	values := []selectableValue{}
+	for _, service := range resp.Services {
+		name := nameExp.FindString(service.Name)
+		if name == "" {
+			return nil, fmt.Errorf("unexpected service name: %v", service.Name)
+		}
+		label := service.DisplayName
+		if label == "" {
+			label = name
+		}
+		values = append(values, selectableValue{
+			Value: name,
+			Label: label,
+		})
+	}
+	return json.Marshal(values)
+}
+
+func processSLOs(body []byte) ([]byte, error) {
+	resp := sloResponse{}
+	err := json.Unmarshal(body, &resp)
+	if err != nil {
+		return nil, err
+	}
+
+	values := []selectableValue{}
+	for _, slo := range resp.SLOs {
+		name := nameExp.FindString(slo.Name)
+		if name == "" {
+			return nil, fmt.Errorf("unexpected service name: %v", slo.Name)
+		}
+		values = append(values, selectableValue{
+			Value: name,
+			Label: slo.DisplayName,
+			Goal:  slo.Goal,
+		})
+	}
+	return json.Marshal(values)
+}
+
+func processProjects(body []byte) ([]byte, error) {
+	resp := projectResponse{}
+	err := json.Unmarshal(body, &resp)
+	if err != nil {
+		return nil, err
+	}
+
+	values := []selectableValue{}
+	for _, project := range resp.Projects {
+		values = append(values, selectableValue{
+			Value: project.ProjectID,
+			Label: project.Name,
+		})
+	}
+	return json.Marshal(values)
+}
+
+func processData(res *http.Response, responseFn processResponse) ([]byte, int, error) {
+	encoding := res.Header.Get("Content-Encoding")
+
+	var reader io.Reader
+	var err error
+	switch encoding {
+	case "gzip":
+		reader, err = gzip.NewReader(res.Body)
+		if err != nil {
+			return nil, http.StatusBadRequest, fmt.Errorf("unexpected error %v", err)
+		}
+		defer func() {
+			if err := reader.(io.ReadCloser).Close(); err != nil {
+				slog.Warn("Failed to close reader body", "err", err)
+			}
+		}()
+	case "deflate":
+		reader = flate.NewReader(res.Body)
+		defer func() {
+			if err := reader.(io.ReadCloser).Close(); err != nil {
+				slog.Warn("Failed to close reader body", "err", err)
+			}
+		}()
+	case "br":
+		reader = brotli.NewReader(res.Body)
+	case "":
+		reader = res.Body
+	default:
+		return nil, http.StatusInternalServerError, fmt.Errorf("unexpected encoding type %v", err)
+	}
+
+	body, err := ioutil.ReadAll(reader)
+	if err != nil {
+		return nil, http.StatusBadRequest, fmt.Errorf("unexpected error %v", err)
+	}
+
+	body, err = responseFn(body)
+	if err != nil {
+		return nil, http.StatusInternalServerError, fmt.Errorf("data processing error %v", err)
+	}
+
+	buf := new(bytes.Buffer)
+	var writer io.Writer = buf
+	switch encoding {
+	case "gzip":
+		writer = gzip.NewWriter(writer)
+	case "deflate":
+		writer, err = flate.NewWriter(writer, -1)
+		if err != nil {
+			return nil, http.StatusInternalServerError, fmt.Errorf("unexpected error %v", err)
+		}
+	case "br":
+		writer = brotli.NewWriter(writer)
+	case "":
+	default:
+		return nil, http.StatusInternalServerError, fmt.Errorf("unexpected encoding type %v", encoding)
+	}
+
+	_, err = writer.Write(body)
+	if writeCloser, ok := writer.(io.WriteCloser); ok {
+		if err := writeCloser.Close(); err != nil {
+			slog.Warn("Failed to close writer body", "err", err)
+		}
+	}
+	if err != nil {
+		return nil, http.StatusInternalServerError, fmt.Errorf("unable to encode response %v", err)
+	}
+
+	return buf.Bytes(), 0, nil
+}
+
 func (s *Service) setRequestVariables(req *http.Request, subDataSource string) (*http.Client, int, error) {
 	slog.Debug("Received resource call", "url", req.URL.String(), "method", req.Method)
 
@@ -50,48 +263,10 @@ func (s *Service) setRequestVariables(req *http.Request, subDataSource string) (
 	return dsInfo.services[subDataSource].client, 0, nil
 }
 
-func doRequest(rw http.ResponseWriter, req *http.Request, cli *http.Client) http.ResponseWriter {
-	res, err := cli.Do(req)
-	if err != nil {
-		rw.WriteHeader(http.StatusBadRequest)
-		_, err = rw.Write([]byte(fmt.Sprintf("unexpected error %v", err)))
-		if err != nil {
-			slog.Error("Unable to write HTTP response", "error", err)
-		}
-		return nil
-	}
-	defer func() {
-		if err := res.Body.Close(); err != nil {
-			slog.Warn("Failed to close response body", "err", err)
-		}
-	}()
-
-	body, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		rw.WriteHeader(http.StatusInternalServerError)
-		_, err = rw.Write([]byte(fmt.Sprintf("unexpected error %v", err)))
-		if err != nil {
-			slog.Error("Unable to write HTTP response", "error", err)
-		}
-		return nil
-	}
-	rw.WriteHeader(res.StatusCode)
-	_, err = rw.Write(body)
-	if err != nil {
-		slog.Error("Unable to write HTTP response", "error", err)
-	}
-
-	for k, v := range res.Header {
-		rw.Header().Set(k, v[0])
-		for _, v := range v[1:] {
-			rw.Header().Add(k, v)
-		}
-	}
-	// Returning the response write for testing purposes
-	return rw
-}
-
 func getTarget(original string) (target string, err error) {
+	if original == "/projects" {
+		return resourceManagerPath, nil
+	}
 	splittedPath := strings.SplitN(original, "/", 3)
 	if len(splittedPath) < 3 {
 		err = fmt.Errorf("the request should contain the service on its path")
@@ -101,14 +276,18 @@ func getTarget(original string) (target string, err error) {
 	return
 }
 
-func writeResponse(rw http.ResponseWriter, code int, msg string) {
+func writeResponseBytes(rw http.ResponseWriter, code int, msg []byte) {
 	rw.WriteHeader(code)
-	_, err := rw.Write([]byte(msg))
+	_, err := rw.Write(msg)
 	if err != nil {
 		slog.Error("Unable to write HTTP response", "error", err)
 	}
 }
 
+func writeResponse(rw http.ResponseWriter, code int, msg string) {
+	writeResponseBytes(rw, code, []byte(msg))
+}
+
 func (s *Service) getDataSourceFromHTTPReq(req *http.Request) (*datasourceInfo, error) {
 	ctx := req.Context()
 	pluginContext := httpadapter.PluginConfigFromContext(ctx)

pkg/tsdb/cloudmonitoring/resource_handler_test.go

@@ -1,6 +1,7 @@
 package cloudmonitoring
 
 import (
+	"encoding/json"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
@@ -42,6 +43,10 @@ func Test_parseResourcePath(t *testing.T) {
 	}
 }
 
+func fakeResponseFn(input []byte) ([]byte, error) {
+	return input, nil
+}
+
 func Test_doRequest(t *testing.T) {
 	// test that it forwards the header and body
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -55,8 +60,10 @@ func Test_doRequest(t *testing.T) {
 	if err != nil {
 		t.Error(err)
 	}
+	s := Service{}
+
 	rw := httptest.NewRecorder()
-	res := doRequest(rw, req, srv.Client())
+	res := s.doRequest(rw, req, srv.Client(), fakeResponseFn)
 	if res.Header().Get("foo") != "bar" {
 		t.Errorf("Unexpected headers: %v", res.Header())
 	}
@@ -112,3 +119,149 @@ func Test_setRequestVariables(t *testing.T) {
 		t.Errorf("Unexpected result URL. Got %s, expecting %s", req.URL.String(), expectedURL)
 	}
 }
+
+func Test_processData_functions(t *testing.T) {
+	// metricDescriptors
+	metricDescriptorResp := metricDescriptorResponse{
+		Descriptors: []metricDescriptor{
+			{
+				ValueType:        "INT64",
+				MetricKind:       "DELTA",
+				Type:             "actions.googleapis.com/smarthome_action/local_event_count",
+				Unit:             "1",
+				Service:          "foo",
+				ServiceShortName: "bar",
+				DisplayName:      "Local event count",
+				Description:      "baz",
+			},
+		},
+	}
+	marshaledMDResponse, _ := json.Marshal(metricDescriptorResp)
+	metricDescriptorResult := []metricDescriptor{
+		{
+			ValueType:        "INT64",
+			MetricKind:       "DELTA",
+			Type:             "actions.googleapis.com/smarthome_action/local_event_count",
+			Unit:             "1",
+			Service:          "actions.googleapis.com",
+			ServiceShortName: "actions",
+			DisplayName:      "Local event count",
+			Description:      "baz",
+		},
+	}
+	marshaledMDResult, _ := json.Marshal(metricDescriptorResult)
+
+	// services
+	serviceResp := serviceResponse{
+		Services: []serviceDescription{
+			{
+				Name:        "blah/foo",
+				DisplayName: "bar",
+			},
+			{
+				Name:        "abc",
+				DisplayName: "",
+			},
+		},
+	}
+	marshaledServiceResponse, _ := json.Marshal(serviceResp)
+	serviceResult := []selectableValue{
+		{
+			Value: "foo",
+			Label: "bar",
+		},
+		{
+			Value: "abc",
+			Label: "abc",
+		},
+	}
+	marshaledServiceResult, _ := json.Marshal(serviceResult)
+
+	// slos
+	sloResp := sloResponse{
+		SLOs: []sloDescription{
+			{
+				Name:        "blah/foo",
+				DisplayName: "bar",
+				Goal:        0.1,
+			},
+			{
+				Name:        "abc",
+				DisplayName: "xyz",
+				Goal:        0.2,
+			},
+		},
+	}
+	marshaledSLOResponse, _ := json.Marshal(sloResp)
+	sloResult := []selectableValue{
+		{
+			Value: "foo",
+			Label: "bar",
+			Goal:  0.1,
+		},
+		{
+			Value: "abc",
+			Label: "xyz",
+			Goal:  0.2,
+		},
+	}
+	marshaledSLOResult, _ := json.Marshal(sloResult)
+
+	// cloudresourcemanager
+	cloudResourceResp := projectResponse{
+		Projects: []projectDescription{
+			{
+				ProjectID: "foo",
+				Name:      "bar",
+			},
+			{
+				ProjectID: "abc",
+				Name:      "abc",
+			},
+		},
+	}
+	marshaledCRResponse, _ := json.Marshal(cloudResourceResp)
+
+	tests := []struct {
+		name       string
+		responseFn processResponse
+		input      []byte
+		result     []byte
+	}{
+		{
+			"metricDescriptor",
+			processMetricDescriptors,
+			marshaledMDResponse,
+			marshaledMDResult,
+		},
+		{
+			"services",
+			processServices,
+			marshaledServiceResponse,
+			marshaledServiceResult,
+		},
+		{
+			"slos",
+			processSLOs,
+			marshaledSLOResponse,
+			marshaledSLOResult,
+		},
+		{
+			"cloudresourcemanager",
+			processProjects,
+			marshaledCRResponse,
+			marshaledServiceResult,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			res, err := test.responseFn(test.input)
+			if err != nil {
+				t.Errorf("Unexpected error %v", err)
+			}
+			if string(test.result) != string(res) {
+				t.Errorf("Unexpected result. Got %s, expecting %s", res, test.result)
+			}
+		})
+	}
+}

pkg/tsdb/cloudmonitoring/types.go

@@ -189,3 +189,50 @@ type timeSeries struct {
 		} `json:"value"`
 	} `json:"points"`
 }
+
+type metricDescriptorResponse struct {
+	Descriptors []metricDescriptor `json:"metricDescriptors"`
+}
+type metricDescriptor struct {
+	ValueType        string `json:"valueType"`
+	MetricKind       string `json:"metricKind"`
+	Type             string `json:"type"`
+	Unit             string `json:"unit"`
+	Service          string `json:"service"`
+	ServiceShortName string `json:"serviceShortName"`
+	DisplayName      string `json:"displayName"`
+	Description      string `json:"description"`
+}
+
+type projectResponse struct {
+	Projects []projectDescription `json:"projects"`
+}
+
+type projectDescription struct {
+	ProjectID string `json:"projectId"`
+	Name      string `json:"name"`
+}
+
+type serviceResponse struct {
+	Services []serviceDescription `json:"services"`
+}
+type serviceDescription struct {
+	Name        string `json:"name"`
+	DisplayName string `json:"displayName"`
+}
+
+type sloResponse struct {
+	SLOs []sloDescription `json:"serviceLevelObjectives"`
+}
+
+type sloDescription struct {
+	Name        string  `json:"name"`
+	DisplayName string  `json:"displayName"`
+	Goal        float64 `json:"goal"`
+}
+
+type selectableValue struct {
+	Value string  `json:"value"`
+	Label string  `json:"label"`
+	Goal  float64 `json:"goal,omitempty"`
+}

public/app/plugins/datasource/cloud-monitoring/api.test.ts

@@ -1,79 +0,0 @@
-import { of } from 'rxjs';
-
-import Api from './api';
-import { backendSrv } from 'app/core/services/backend_srv'; // will use the version in __mocks__
-import { createFetchResponse } from 'test/helpers/createFetchResponse';
-
-jest.mock('@grafana/runtime', () => ({
-  ...((jest.requireActual('@grafana/runtime') as unknown) as object),
-  getBackendSrv: () => backendSrv,
-}));
-
-const response = [
-  { label: 'test1', value: 'test1' },
-  { label: 'test2', value: 'test2' },
-];
-
-type Args = { path?: string; options?: any; response?: any; cache?: any };
-
-async function getTestContext({ path = 'some-resource', options = {}, response = {}, cache }: Args = {}) {
-  jest.clearAllMocks();
-
-  const fetchMock = jest.spyOn(backendSrv, 'fetch');
-
-  fetchMock.mockImplementation((options: any) => {
-    const data = { [options.url.match(/([^\/]*)\/*$/)![1].split('?')[0]]: response };
-    return of(createFetchResponse(data));
-  });
-
-  const api = new Api('/cloudmonitoring/');
-
-  if (cache) {
-    api.cache[path] = cache;
-  }
-
-  const res = await api.get(path, options);
-
-  return { res, api, fetchMock };
-}
-
-describe('api', () => {
-  describe('when resource was cached', () => {
-    test.each(['some-resource', 'some-resource?some=param', 'test/some-resource?param'])(
-      'should return cached value and not load from source',
-      async (path) => {
-        const { res, api, fetchMock } = await getTestContext({ path, cache: response });
-
-        expect(res).toEqual(response);
-        expect(api.cache[path]).toEqual(response);
-        expect(fetchMock).not.toHaveBeenCalled();
-      }
-    );
-  });
-
-  describe('when resource was not cached', () => {
-    test.each(['some-resource', 'some-resource?some=param', 'test/some-resource?param'])(
-      'should return from source and not from cache',
-      async (path) => {
-        const { res, api, fetchMock } = await getTestContext({ path, response });
-
-        expect(res).toEqual(response);
-        expect(api.cache[path]).toEqual(response);
-        expect(fetchMock).toHaveBeenCalled();
-      }
-    );
-  });
-
-  describe('when cache should be bypassed', () => {
-    test.each(['some-resource', 'some-resource?some=param', 'test/some-resource?param'])(
-      'should return from source and not from cache',
-      async (path) => {
-        const options = { useCache: false };
-        const { res, fetchMock } = await getTestContext({ path, response, cache: response, options });
-
-        expect(res).toEqual(response);
-        expect(fetchMock).toHaveBeenCalled();
-      }
-    );
-  });
-});

public/app/plugins/datasource/cloud-monitoring/api.ts

@@ -1,87 +0,0 @@
-import { lastValueFrom, Observable, of } from 'rxjs';
-import { catchError, map } from 'rxjs/operators';
-import { SelectableValue } from '@grafana/data';
-import { FetchResponse, getBackendSrv } from '@grafana/runtime';
-
-import appEvents from 'app/core/app_events';
-import { CoreEvents } from 'app/types';
-import { formatCloudMonitoringError } from './functions';
-import { MetricDescriptor } from './types';
-
-export interface PostResponse {
-  results: Record<string, any>;
-}
-
-interface Options {
-  responseMap?: (res: any) => SelectableValue<string> | MetricDescriptor;
-  baseUrl?: string;
-  useCache?: boolean;
-}
-
-export default class Api {
-  cache: { [key: string]: Array<SelectableValue<string>> };
-  defaultOptions: Options;
-
-  constructor(private baseUrl: string) {
-    this.cache = {};
-    this.defaultOptions = {
-      useCache: true,
-      responseMap: (res: any) => res,
-      baseUrl: this.baseUrl,
-    };
-  }
-
-  get(path: string, options?: Options): Promise<Array<SelectableValue<string>> | MetricDescriptor[]> {
-    const { useCache, responseMap, baseUrl } = { ...this.defaultOptions, ...options };
-
-    if (useCache && this.cache[path]) {
-      return Promise.resolve(this.cache[path]);
-    }
-
-    return lastValueFrom(
-      getBackendSrv()
-        .fetch<Record<string, any>>({
-          url: baseUrl + path,
-          method: 'GET',
-        })
-        .pipe(
-          map((response) => {
-            const responsePropName = path.match(/([^\/]*)\/*$/)![1].split('?')[0];
-            let res = [];
-            if (response && response.data && response.data[responsePropName]) {
-              res = response.data[responsePropName].map(responseMap);
-            }
-
-            if (useCache) {
-              this.cache[path] = res;
-            }
-
-            return res;
-          }),
-          catchError((error) => {
-            appEvents.emit(CoreEvents.dsRequestError, {
-              error: { data: { error: formatCloudMonitoringError(error) } },
-            });
-            return of([]);
-          })
-        )
-    );
-  }
-
-  post(data: Record<string, any>): Observable<FetchResponse<PostResponse>> {
-    return getBackendSrv().fetch<PostResponse>({
-      url: '/api/ds/query',
-      method: 'POST',
-      data,
-    });
-  }
-
-  test(projectName: string) {
-    return lastValueFrom(
-      getBackendSrv().fetch<any>({
-        url: `${this.baseUrl}${projectName}/metricDescriptors`,
-        method: 'GET',
-      })
-    );
-  }
-}

public/app/plugins/datasource/cloud-monitoring/components/Project.tsx

@@ -1,4 +1,4 @@
-import React, { useEffect, useState } from 'react';
+import React, { useEffect, useMemo, useState } from 'react';
 import { SelectableValue } from '@grafana/data';
 import { Select } from '@grafana/ui';
 import CloudMonitoringDatasource from '../datasource';
@@ -15,16 +15,20 @@ export interface Props {
 export function Project({ projectName, datasource, onChange, templateVariableOptions }: Props) {
   const [projects, setProjects] = useState<Array<SelectableValue<string>>>([]);
   useEffect(() => {
-    datasource.getProjects().then((projects) =>
+    datasource.getProjects().then((projects) => setProjects(projects));
-      setProjects([
+  }, [datasource]);
-        {
+
-          label: 'Template Variables',
+  const projectsWithTemplateVariables = useMemo(
-          options: templateVariableOptions,
+    () => [
-        },
+      projects,
-        ...projects,
+      {
-      ])
+        label: 'Template Variables',
-    );
+        options: templateVariableOptions,
-  }, [datasource, templateVariableOptions]);
+      },
+      ...projects,
+    ],
+    [projects, templateVariableOptions]
+  );
 
   return (
     <QueryEditorRow label="Project">
@@ -34,7 +38,7 @@ export function Project({ projectName, datasource, onChange, templateVariableOpt
         allowCustomValue
         formatCreateLabel={(v) => `Use project: ${v}`}
         onChange={({ value }) => onChange(value!)}
-        options={projects}
+        options={projectsWithTemplateVariables}
         value={{ value: projectName, label: projectName }}
         placeholder="Select Project"
       />

public/app/plugins/datasource/cloud-monitoring/datasource.ts

@@ -1,6 +1,6 @@
 import { chunk, flatten, isString } from 'lodash';
-import { from, lastValueFrom, Observable, of, throwError } from 'rxjs';
+import { from, lastValueFrom, Observable, of } from 'rxjs';
-import { catchError, map, mergeMap } from 'rxjs/operators';
+import { map, mergeMap } from 'rxjs/operators';
 import {
   DataQueryRequest,
   DataQueryResponse,
@@ -8,19 +8,25 @@ import {
   ScopedVars,
   SelectableValue,
 } from '@grafana/data';
-import { DataSourceWithBackend, toDataQueryResponse } from '@grafana/runtime';
+import { DataSourceWithBackend, getBackendSrv, toDataQueryResponse } from '@grafana/runtime';
 
 import { getTemplateSrv, TemplateSrv } from 'app/features/templating/template_srv';
 import { getTimeSrv, TimeSrv } from 'app/features/dashboard/services/TimeSrv';
-import { CloudMonitoringOptions, CloudMonitoringQuery, EditorMode, Filter, MetricDescriptor, QueryType } from './types';
+import {
-import API from './api';
+  CloudMonitoringOptions,
+  CloudMonitoringQuery,
+  EditorMode,
+  Filter,
+  MetricDescriptor,
+  QueryType,
+  PostResponse,
+} from './types';
 import { CloudMonitoringVariableSupport } from './variables';
 
 export default class CloudMonitoringDatasource extends DataSourceWithBackend<
   CloudMonitoringQuery,
   CloudMonitoringOptions
 > {
-  api: API;
   authenticationType: string;
   intervalMs: number;
 
@@ -31,7 +37,6 @@ export default class CloudMonitoringDatasource extends DataSourceWithBackend<
   ) {
     super(instanceSettings);
     this.authenticationType = instanceSettings.jsonData.authenticationType || 'jwt';
-    this.api = new API(`/api/datasources/${this.id}/resources/cloudmonitoring/v3/projects/`);
     this.variables = new CloudMonitoringVariableSupport(this);
     this.intervalMs = 0;
   }
@@ -72,11 +77,15 @@ export default class CloudMonitoringDatasource extends DataSourceWithBackend<
     ];
 
     return lastValueFrom(
-      this.api
+      getBackendSrv()
-        .post({
+        .fetch<PostResponse>({
-          from: options.range.from.valueOf().toString(),
+          url: '/api/ds/query',
-          to: options.range.to.valueOf().toString(),
+          method: 'POST',
-          queries,
+          data: {
+            from: options.range.from.valueOf().toString(),
+            to: options.range.to.valueOf().toString(),
+            queries,
+          },
         })
         .pipe(
           map(({ data }) => {
@@ -156,10 +165,14 @@ export default class CloudMonitoringDatasource extends DataSourceWithBackend<
     return lastValueFrom(
       from(this.ensureGCEDefaultProject()).pipe(
         mergeMap(() => {
-          return this.api.post({
+          return getBackendSrv().fetch<PostResponse>({
-            from: options.range.from.valueOf().toString(),
+            url: '/api/ds/query',
-            to: options.range.to.valueOf().toString(),
+            method: 'POST',
-            queries,
+            data: {
+              from: options.range.from.valueOf().toString(),
+              to: options.range.to.valueOf().toString(),
+              queries,
+            },
           });
         }),
         map(({ data }) => {
@@ -172,62 +185,8 @@ export default class CloudMonitoringDatasource extends DataSourceWithBackend<
     );
   }
 
-  async testDatasource() {
-    let status, message;
-    const defaultErrorMessage = 'Cannot connect to Google Cloud Monitoring API';
-    try {
-      await this.ensureGCEDefaultProject();
-      const response = await this.api.test(this.getDefaultProject());
-      if (response.status === 200) {
-        status = 'success';
-        message = 'Successfully queried the Google Cloud Monitoring API.';
-      } else {
-        status = 'error';
-        message = response.statusText ? response.statusText : defaultErrorMessage;
-      }
-    } catch (error) {
-      status = 'error';
-      if (isString(error)) {
-        message = error;
-      } else {
-        message = 'Google Cloud Monitoring: ';
-        message += error.statusText ? error.statusText : defaultErrorMessage;
-        if (error.data && error.data.error && error.data.error.code) {
-          message += ': ' + error.data.error.code + '. ' + error.data.error.message;
-        }
-      }
-    } finally {
-      return {
-        status,
-        message,
-      };
-    }
-  }
-
   async getGCEDefaultProject() {
-    return lastValueFrom(
+    return this.getResource(`gceDefaultProject`);
-      this.api
-        .post({
-          queries: [
-            {
-              refId: 'getGCEDefaultProject',
-              type: 'getGCEDefaultProject',
-              datasource: this.getRef(),
-            },
-          ],
-        })
-        .pipe(
-          map(({ data }) => {
-            const dataQueryResponse = toDataQueryResponse({
-              data: data,
-            });
-            return dataQueryResponse?.data[0]?.meta?.custom?.defaultProject ?? '';
-          }),
-          catchError((err) => {
-            return throwError(err.data.error);
-          })
-        )
-    );
   }
 
   getDefaultProject(): string {
@@ -251,26 +210,13 @@ export default class CloudMonitoringDatasource extends DataSourceWithBackend<
       return [];
     }
 
-    return this.api.get(`${this.templateSrv.replace(projectName)}/metricDescriptors`, {
+    return this.getResource(
-      responseMap: (m: MetricDescriptor) => {
+      `metricDescriptors/v3/projects/${this.templateSrv.replace(projectName)}/metricDescriptors`
-        const [service] = m.type.split('/');
+    ) as Promise<MetricDescriptor[]>;
-        const [serviceShortName] = service.split('.');
-        m.service = service;
-        m.serviceShortName = serviceShortName;
-        m.displayName = m.displayName || m.type;
-
-        return m;
-      },
-    }) as Promise<MetricDescriptor[]>;
   }
 
   async getSLOServices(projectName: string): Promise<Array<SelectableValue<string>>> {
-    return this.api.get(`${this.templateSrv.replace(projectName)}/services?pageSize=1000`, {
+    return this.getResource(`services/v3/projects/${this.templateSrv.replace(projectName)}/services?pageSize=1000`);
-      responseMap: ({ name, displayName }: { name: string; displayName: string }) => ({
-        value: name.match(/([^\/]*)\/*$/)![1],
-        label: displayName || name.match(/([^\/]*)\/*$/)![1],
-      }),
-    });
   }
 
   async getServiceLevelObjectives(projectName: string, serviceId: string): Promise<Array<SelectableValue<string>>> {
@@ -278,23 +224,11 @@ export default class CloudMonitoringDatasource extends DataSourceWithBackend<
       return Promise.resolve([]);
     }
     let { projectName: p, serviceId: s } = this.interpolateProps({ projectName, serviceId });
-    return this.api.get(`${p}/services/${s}/serviceLevelObjectives`, {
+    return this.getResource(`slo-services/v3/projects/${p}/services/${s}/serviceLevelObjectives`);
-      responseMap: ({ name, displayName, goal }: { name: string; displayName: string; goal: number }) => ({
-        value: name.match(/([^\/]*)\/*$/)![1],
-        label: displayName,
-        goal,
-      }),
-    });
   }
 
   getProjects(): Promise<Array<SelectableValue<string>>> {
-    return this.api.get(`projects`, {
+    return this.getResource(`projects`);
-      responseMap: ({ projectId, name }: { projectId: string; name: string }) => ({
-        value: projectId,
-        label: name,
-      }),
-      baseUrl: `/api/datasources/${this.id}/resources/cloudresourcemanager/v1/`,
-    });
   }
 
   migrateQuery(query: CloudMonitoringQuery): CloudMonitoringQuery {

public/app/plugins/datasource/cloud-monitoring/specs/datasource.test.ts

@@ -2,7 +2,6 @@ import { of, throwError } from 'rxjs';
 import { DataSourceInstanceSettings, toUtc } from '@grafana/data';
 
 import CloudMonitoringDataSource from '../datasource';
-import { metricDescriptors } from './testData';
 import { TemplateSrv } from 'app/features/templating/template_srv';
 import { CloudMonitoringOptions } from '../types';
 import { backendSrv } from 'app/core/services/backend_srv'; // will use the version in __mocks__
@@ -41,47 +40,6 @@ function getTestcontext({ response = {}, throws = false, templateSrv = new Templ
 }
 
 describe('CloudMonitoringDataSource', () => {
-  describe('when performing testDataSource', () => {
-    describe('and call to cloud monitoring api succeeds', () => {
-      it('should return successfully', async () => {
-        const { ds } = getTestcontext();
-
-        const result = await ds.testDatasource();
-
-        expect(result.status).toBe('success');
-      });
-    });
-
-    describe('and a list of metricDescriptors are returned', () => {
-      it('should return status success', async () => {
-        const { ds } = getTestcontext({ response: metricDescriptors });
-
-        const result = await ds.testDatasource();
-
-        expect(result.status).toBe('success');
-      });
-    });
-
-    describe('and call to cloud monitoring api fails with 400 error', () => {
-      it('should return error status and a detailed error message', async () => {
-        const response = {
-          statusText: 'Bad Request',
-          data: {
-            error: { code: 400, message: 'Field interval.endTime had an invalid value' },
-          },
-        };
-        const { ds } = getTestcontext({ response, throws: true });
-
-        const result = await ds.testDatasource();
-
-        expect(result.status).toEqual('error');
-        expect(result.message).toBe(
-          'Google Cloud Monitoring: Bad Request: 400. Field interval.endTime had an invalid value'
-        );
-      });
-    });
-  });
-
   describe('When performing query', () => {
     describe('and no time series data is returned', () => {
       it('should return a list of datapoints', async () => {
@@ -124,37 +82,6 @@ describe('CloudMonitoringDataSource', () => {
     });
   });
 
-  describe('when performing getMetricTypes', () => {
-    describe('and call to cloud monitoring api succeeds', () => {
-      it('should return successfully', async () => {
-        const response = {
-          metricDescriptors: [
-            {
-              displayName: 'test metric name 1',
-              type: 'compute.googleapis.com/instance/cpu/test-metric-type-1',
-              description: 'A description',
-            },
-            {
-              type: 'logging.googleapis.com/user/logbased-metric-with-no-display-name',
-            },
-          ],
-        };
-        const { ds } = getTestcontext({ response });
-
-        const result = await ds.getMetricTypes('proj');
-
-        expect(result.length).toBe(2);
-        expect(result[0].service).toBe('compute.googleapis.com');
-        expect(result[0].serviceShortName).toBe('compute');
-        expect(result[0].type).toBe('compute.googleapis.com/instance/cpu/test-metric-type-1');
-        expect(result[0].displayName).toBe('test metric name 1');
-        expect(result[0].description).toBe('A description');
-        expect(result[1].type).toBe('logging.googleapis.com/user/logbased-metric-with-no-display-name');
-        expect(result[1].displayName).toBe('logging.googleapis.com/user/logbased-metric-with-no-display-name');
-      });
-    });
-  });
-
   describe('when interpolating a template variable for the filter', () => {
     describe('and is single value variable', () => {
       it('should replace the variable with the value', () => {

public/app/plugins/datasource/cloud-monitoring/types.ts

@@ -204,3 +204,7 @@ export interface CustomMetaData {
   perSeriesAligner?: string;
   alignmentPeriod?: string;
 }
+
+export interface PostResponse {
+  results: Record<string, any>;
+}