From 7de093738dfad25964a801a9141a2e7a1dd65ab0 Mon Sep 17 00:00:00 2001 From: owensmallwood Date: Wed, 2 Nov 2022 13:35:57 -0600 Subject: [PATCH] PublicDashboards: Adds command to generate map of supported datasources (#57841) generates map of supported datasources for pubdash --- .gitignore | 1 + .../commands/generate_datasources/generate.go | 106 ++++++++++++++++++ .../commands/generate_datasources/main.go | 40 +++++++ .../generate_datasources/main_test.go | 36 ++++++ .../publicdashboards/publicdashboard.go | 1 + .../supported_datasources_gen.go | 65 +++++++++++ 6 files changed, 249 insertions(+) create mode 100644 pkg/services/publicdashboards/commands/generate_datasources/generate.go create mode 100644 pkg/services/publicdashboards/commands/generate_datasources/main.go create mode 100644 pkg/services/publicdashboards/commands/generate_datasources/main_test.go create mode 100644 pkg/services/publicdashboards/supported_datasources_gen.go diff --git a/.gitignore b/.gitignore index 1d598a8d73f..a77f35f8747 100644 --- a/.gitignore +++ b/.gitignore @@ -177,6 +177,7 @@ compilation-stats.json !pkg/framework/**/*_gen.go !pkg/plugins/pfs/**/*_gen.go !public/app/plugins/**/*_gen.go +!pkg/services/publicdashboards/*_gen.go # Auto-generated internationalization files public/locales/_build/ diff --git a/pkg/services/publicdashboards/commands/generate_datasources/generate.go b/pkg/services/publicdashboards/commands/generate_datasources/generate.go new file mode 100644 index 00000000000..d36852860e2 --- /dev/null +++ b/pkg/services/publicdashboards/commands/generate_datasources/generate.go @@ -0,0 +1,106 @@ +package generate_datasources + +import ( + "encoding/json" + "fmt" + "net/http" + "sort" + "sync" +) + +type listPluginResponse struct { + Items []struct { + Slug string `json:"slug"` + } `json:"items"` +} + +type datasourceDetails struct { + Slug string `json:"slug"` + Backend bool `json:"backend"` + Alerting bool `json:"alerting"` +} + +type datasource struct { + Json datasourceDetails `json:"json"` +} + +// Get the slugs of all datasource plugins that are compatible with public dashboards +func GetCompatibleDatasources(baseUrl string) ([]string, error) { + slugs, err := getDatasourcePluginSlugs(baseUrl) + if err != nil { + return nil, err + } + + datasources := getDatasourceDetails(slugs, baseUrl) + + // we only consider a datasource to be supported when alerting and backend are both true + var supported []string + for _, datasource := range datasources { + if datasource.Alerting && datasource.Backend { + supported = append(supported, datasource.Slug) + } + } + + sort.Strings(supported) + return supported, nil +} + +// Get a list of all the datasource plugin slugs +func getDatasourcePluginSlugs(baseUrl string) ([]string, error) { + resp, err := http.Get(baseUrl + allPluginsEndpoint()) + if err != nil { + return nil, err + } + res := &listPluginResponse{} + err = json.NewDecoder(resp.Body).Decode(res) + if err != nil { + return nil, err + } + var slugs []string + for _, meta := range res.Items { + slugs = append(slugs, meta.Slug) + } + _ = resp.Body.Close() + return slugs, nil +} + +// Get the details for each datasource plugin by its slug +func getDatasourceDetails(slugs []string, baseUrl string) []datasourceDetails { + datasources := make([]datasourceDetails, len(slugs)) + var wg sync.WaitGroup + for i, slug := range slugs { + wg.Add(1) + // create new goroutine for each outgoing request + go func(i int, slug string) { + defer wg.Done() + url := baseUrl + pluginEndpoint(slug) + // url can not be constant since it gets generated based on slug + //nolint:gosec + r, err := http.Get(url) + if err != nil { + panic(err) + } + + datasource := &datasource{} + err = json.NewDecoder(r.Body).Decode(datasource) + if err != nil { + panic(err) + } + datasource.Json.Slug = slug + datasources[i] = datasource.Json + _ = r.Body.Close() + }(i, slug) + } + + wg.Wait() + + return datasources +} + +func pluginEndpoint(slug string) string { + return fmt.Sprintf("/api/plugins/%s?version=latest", slug) +} + +func allPluginsEndpoint() string { + return "/api/plugins?orderBy=weight&direction=asc&typeCodeIn[]=datasource" +} diff --git a/pkg/services/publicdashboards/commands/generate_datasources/main.go b/pkg/services/publicdashboards/commands/generate_datasources/main.go new file mode 100644 index 00000000000..f18fa4685ab --- /dev/null +++ b/pkg/services/publicdashboards/commands/generate_datasources/main.go @@ -0,0 +1,40 @@ +//go:build ignore + +package main + +import ( + "fmt" + "os" + "text/template" + + "github.com/grafana/grafana/pkg/services/publicdashboards/commands/generate_datasources" +) + +var goTemplate = ` +// Code generated by go generate; DO NOT EDIT. + +package publicdashboards + +var SupportedDatasources = map[string]bool{ +{{- range . }} + "{{ printf "%s" . }}": true, +{{- end }} +} +` + +func main() { + baseUrl := "https://grafana.com" + slugs, err := generate_datasources.GetCompatibleDatasources(baseUrl) + + myTemplate := template.Must(template.New("").Parse(goTemplate)) + + file, err := os.Create("supported_datasources_gen.go") + if err != nil { + fmt.Println(err) + } + + err = myTemplate.Execute(file, slugs) + if err != nil { + fmt.Println(err) + } +} diff --git a/pkg/services/publicdashboards/commands/generate_datasources/main_test.go b/pkg/services/publicdashboards/commands/generate_datasources/main_test.go new file mode 100644 index 00000000000..f535446fac8 --- /dev/null +++ b/pkg/services/publicdashboards/commands/generate_datasources/main_test.go @@ -0,0 +1,36 @@ +package generate_datasources + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCanGetCompatibleDatasources(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var err error + w.WriteHeader(http.StatusOK) + // response for getting metadata for all datasources + if r.URL.String() == allPluginsEndpoint() { + _, err = w.Write([]byte(`{"items":[{"slug":"postgres"},{"slug":"frontendDatasource"}]}`)) + } + // responses for specific datasource plugins + if r.URL.String() == pluginEndpoint("postgres") { + _, err = w.Write([]byte(`{"json":{"alerting":true,"backend":true}}`)) + } + if r.URL.String() == pluginEndpoint("frontendDatasource") { + _, err = w.Write([]byte(`{"json":{}}`)) + } + require.NoError(t, err) + })) + defer server.Close() + + datasources, err := GetCompatibleDatasources(server.URL) + + require.NoError(t, err) + assert.Len(t, datasources, 1) + assert.Equal(t, "postgres", datasources[0]) +} diff --git a/pkg/services/publicdashboards/publicdashboard.go b/pkg/services/publicdashboards/publicdashboard.go index 889b58188d7..6c1163e0fc5 100644 --- a/pkg/services/publicdashboards/publicdashboard.go +++ b/pkg/services/publicdashboards/publicdashboard.go @@ -12,6 +12,7 @@ import ( // These are the api contracts. The API should match the underlying service and store +//go:generate go run ./commands/generate_datasources/main.go //go:generate mockery --name Service --structname FakePublicDashboardService --inpackage --filename public_dashboard_service_mock.go type Service interface { FindPublicDashboardAndDashboardByAccessToken(ctx context.Context, accessToken string) (*PublicDashboard, *models.Dashboard, error) diff --git a/pkg/services/publicdashboards/supported_datasources_gen.go b/pkg/services/publicdashboards/supported_datasources_gen.go new file mode 100644 index 00000000000..5540138c0e1 --- /dev/null +++ b/pkg/services/publicdashboards/supported_datasources_gen.go @@ -0,0 +1,65 @@ +// Code generated by go generate; DO NOT EDIT. + +package publicdashboards + +var SupportedDatasources = map[string]bool{ + "aquaqanalytics-kdbbackend-datasource": true, + "cloudwatch": true, + "dlopes7-appdynamics-datasource": true, + "doitintl-bigquery-datasource": true, + "elasticsearch": true, + "frser-sqlite-datasource": true, + "grafadruid-druid-datasource": true, + "grafana-athena-datasource": true, + "grafana-azure-data-explorer-datasource": true, + "grafana-azure-monitor-datasource": true, + "grafana-bigquery-datasource": true, + "grafana-clickhouse-datasource": true, + "grafana-databricks-datasource": true, + "grafana-datadog-datasource": true, + "grafana-db2-datasource": true, + "grafana-dynatrace-datasource": true, + "grafana-es-open-distro-datasource": true, + "grafana-github-datasource": true, + "grafana-honeycomb-datasource": true, + "grafana-iot-sitewise-datasource": true, + "grafana-jira-datasource": true, + "grafana-mock-datasource": true, + "grafana-mongodb-datasource": true, + "grafana-newrelic-datasource": true, + "grafana-odbc-datasource": true, + "grafana-opcua-datasource": true, + "grafana-opensearch-datasource": true, + "grafana-oracle-datasource": true, + "grafana-orbit-datasource": true, + "grafana-redshift-datasource": true, + "grafana-salesforce-datasource": true, + "grafana-saphana-datasource": true, + "grafana-sentry-datasource": true, + "grafana-servicenow-datasource": true, + "grafana-snowflake-datasource": true, + "grafana-splunk-datasource": true, + "grafana-splunk-monitoring-datasource": true, + "grafana-timestream-datasource": true, + "grafana-wavefront-datasource": true, + "grafana-x-ray-datasource": true, + "graphite": true, + "hadesarchitect-cassandra-datasource": true, + "influxdb": true, + "innius-grpc-datasource": true, + "kniepdennis-neo4j-datasource": true, + "loki": true, + "marcusolsson-csv-datasource": true, + "marcusolsson-ynab-datasource": true, + "mssql": true, + "mysql": true, + "opentsdb": true, + "postgres": true, + "prometheus": true, + "redis-datasource": true, + "sentinelone-dataset-datasource": true, + "tdengine-datasource": true, + "vertamedia-clickhouse-datasource": true, + "vertica-grafana-datasource": true, + "yesoreyeram-infinity-datasource": true, +}