public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

K8s: Add basic query service (#80325)

Browse Source
This commit is contained in:
Ryan McKinley
2024-01-31 20:36:51 +02:00
committed by GitHub
parent d1b938ba15
commit e013cd427c
38 changed files with 2143 additions and 203 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/registry/apis/datasource/query.go
-104
View File
@@ -1,104 +0,0 @@
package datasource
import (
"encoding/json"
"fmt"
"time"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/tsdb/legacydata"
)
// Copied from: https://github.com/grafana/grafana/blob/main/pkg/api/dtos/models.go#L62
type rawMetricRequest struct {
// From Start time in epoch timestamps in milliseconds or relative using Grafana time units.
// required: true
// example: now-1h
From string `json:"from"`
// To End time in epoch timestamps in milliseconds or relative using Grafana time units.
// required: true
// example: now
To string `json:"to"`
// queries.refId – Specifies an identifier of the query. Is optional and default to “A”.
// queries.datasourceId – Specifies the data source to be queried. Each query in the request must have an unique datasourceId.
// queries.maxDataPoints - Species maximum amount of data points that dashboard panel can render. Is optional and default to 100.
// queries.intervalMs - Specifies the time interval in milliseconds of time series. Is optional and defaults to 1000.
// required: true
// example: [ { "refId": "A", "intervalMs": 86400000, "maxDataPoints": 1092, "datasource":{ "uid":"PD8C576611E62080A" }, "rawSql": "SELECT 1 as valueOne, 2 as valueTwo", "format": "table" } ]
Queries []rawDataQuery `json:"queries"`
// required: false
Debug bool `json:"debug"`
}
type rawDataQuery = map[string]interface{}
func readQueries(in []byte) ([]backend.DataQuery, error) {
reqDTO := &rawMetricRequest{}
err := json.Unmarshal(in, &reqDTO)
if err != nil {
return nil, err
}
if len(reqDTO.Queries) == 0 {
return nil, fmt.Errorf("expected queries")
}
tr := legacydata.NewDataTimeRange(reqDTO.From, reqDTO.To)
backendTr := backend.TimeRange{
From: tr.MustGetFrom(),
To: tr.MustGetTo(),
}
queries := make([]backend.DataQuery, 0)
for _, query := range reqDTO.Queries {
dataQuery := backend.DataQuery{
TimeRange: backendTr,
}
v, ok := query["refId"]
if ok {
dataQuery.RefID, ok = v.(string)
if !ok {
return nil, fmt.Errorf("expeted string refId")
}
}
v, ok = query["queryType"]
if ok {
dataQuery.QueryType, ok = v.(string)
if !ok {
return nil, fmt.Errorf("expeted string queryType")
}
}
v, ok = query["maxDataPoints"]
if ok {
vInt, ok := v.(float64)
if !ok {
return nil, fmt.Errorf("expected float64 maxDataPoints")
}
dataQuery.MaxDataPoints = int64(vInt)
}
v, ok = query["intervalMs"]
if ok {
vInt, ok := v.(float64)
if !ok {
return nil, fmt.Errorf("expected float64 intervalMs")
}
dataQuery.Interval = time.Duration(vInt)
}
dataQuery.JSON, err = json.Marshal(query)
if err != nil {
return nil, err
}
queries = append(queries, dataQuery)
}
return queries, nil
}
pkg/registry/apis/datasource/query_test.go
-33
View File
@@ -1,33 +0,0 @@
package datasource
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestParseQueriesIntoQueryDataRequest(t *testing.T) {
request := []byte(`{
"queries": [
{
"refId": "A",
"datasource": {
"type": "grafana-googlesheets-datasource",
"uid": "b1808c48-9fc9-4045-82d7-081781f8a553"
},
"cacheDurationSeconds": 300,
"spreadsheet": "spreadsheetID",
"range": "",
"datasourceId": 4,
"intervalMs": 30000,
"maxDataPoints": 794
}
],
"from": "1692624667389",
"to": "1692646267389"
}`)
parsedDataQuery, err := readQueries(request)
require.NoError(t, err)
require.Equal(t, len(parsedDataQuery), 1)
}
pkg/registry/apis/datasource/register.go
+4 -2
View File
@@ -20,6 +20,7 @@ import (
common "github.com/grafana/grafana/pkg/apis/common/v0alpha1"
"github.com/grafana/grafana/pkg/apis/datasource/v0alpha1"
query "github.com/grafana/grafana/pkg/apis/query/v0alpha1"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/services/accesscontrol"
"github.com/grafana/grafana/pkg/services/featuremgmt"
@@ -112,7 +113,8 @@ func addKnownTypes(scheme *runtime.Scheme, gv schema.GroupVersion) {
&v0alpha1.DataSourceConnectionList{},
&v0alpha1.HealthCheckResult{},
&unstructured.Unstructured{},
// Added for subresource stubs
// Query handler
&query.QueryDataResponse{},
&metav1.Status{},
)
}
@@ -138,7 +140,7 @@ func (b *DataSourceAPIBuilder) InstallSchema(scheme *runtime.Scheme) error {
}
func resourceFromPluginID(pluginID string) (common.ResourceInfo, error) {
group, err := getDatasourceGroupNameFromPluginID(pluginID)
group, err := plugins.GetDatasourceGroupNameFromPluginID(pluginID)
if err != nil {
return common.ResourceInfo{}, err
}
pkg/registry/apis/datasource/sub_query.go
+74 -43
View File
@@ -3,14 +3,18 @@ package datasource
import (
"context"
"encoding/json"
"io"
"fmt"
"net/http"
"time"
"strconv"
"github.com/grafana/grafana-plugin-sdk-go/backend"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apiserver/pkg/registry/rest"
"github.com/grafana/grafana/pkg/apis/query/v0alpha1"
"github.com/grafana/grafana/pkg/middleware/requestmeta"
"github.com/grafana/grafana/pkg/tsdb/legacydata"
"github.com/grafana/grafana/pkg/web"
)
type subQueryREST struct {
@@ -20,11 +24,10 @@ type subQueryREST struct {
var _ = rest.Connecter(&subQueryREST{})
func (r *subQueryREST) New() runtime.Object {
return &metav1.Status{}
return &v0alpha1.QueryDataResponse{}
}
func (r *subQueryREST) Destroy() {
}
func (r *subQueryREST) Destroy() {}
func (r *subQueryREST) ConnectMethods() []string {
return []string{"POST", "GET"}
@@ -34,40 +37,39 @@ func (r *subQueryREST) NewConnectOptions() (runtime.Object, bool, string) {
return nil, false, ""
}
func (r *subQueryREST) readQueries(req *http.Request) ([]backend.DataQuery, error) {
func (r *subQueryREST) readQueries(req *http.Request) ([]backend.DataQuery, *v0alpha1.DataSourceRef, error) {
reqDTO := v0alpha1.GenericQueryRequest{}
// Simple URL to JSON mapping
if req.Method == http.MethodGet {
body := make(map[string]any, 0)
for k, v := range req.URL.Query() {
switch len(v) {
case 0:
body[k] = true
case 1:
body[k] = v[0] // TODO, convert numbers
query := v0alpha1.GenericDataQuery{
RefID: "A",
MaxDataPoints: 1000,
IntervalMS: 10,
}
params := req.URL.Query()
for k := range params {
v := params.Get(k) // the singular value
switch k {
case "to":
reqDTO.To = v
case "from":
reqDTO.From = v
case "maxDataPoints":
query.MaxDataPoints, _ = strconv.ParseInt(v, 10, 64)
case "intervalMs":
query.IntervalMS, _ = strconv.ParseFloat(v, 64)
case "queryType":
query.QueryType = v
default:
body[k] = v // TODO, convert numbers
query.AdditionalProperties()[k] = v
}
}
var err error
dq := backend.DataQuery{
RefID: "A",
TimeRange: backend.TimeRange{
From: time.Now().Add(-1 * time.Hour), // last hour
To: time.Now(),
},
MaxDataPoints: 1000,
Interval: time.Second * 10,
}
dq.JSON, err = json.Marshal(body)
return []backend.DataQuery{dq}, err
reqDTO.Queries = []v0alpha1.GenericDataQuery{query}
} else if err := web.Bind(req, &reqDTO); err != nil {
return nil, nil, err
}
body, err := io.ReadAll(req.Body)
if err != nil {
return nil, err
}
return readQueries(body)
return legacydata.ToDataSourceQueries(reqDTO)
}
func (r *subQueryREST) Connect(ctx context.Context, name string, opts runtime.Object, responder rest.Responder) (http.Handler, error) {
@@ -77,27 +79,56 @@ func (r *subQueryREST) Connect(ctx context.Context, name string, opts runtime.Ob
}
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
queries, err := r.readQueries(req)
queries, dsRef, err := r.readQueries(req)
if err != nil {
responder.Error(err)
return
}
if dsRef != nil && dsRef.UID != name {
responder.Error(fmt.Errorf("expected the datasource in the request url and body to match"))
return
}
queryResponse, err := r.builder.client.QueryData(ctx, &backend.QueryDataRequest{
qdr, err := r.builder.client.QueryData(ctx, &backend.QueryDataRequest{
PluginContext: pluginCtx,
Queries: queries,
// Headers: // from context
})
if err != nil {
return
}
jsonRsp, err := json.Marshal(queryResponse)
if err != nil {
responder.Error(err)
return
}
w.WriteHeader(200)
_, _ = w.Write(jsonRsp)
statusCode := http.StatusOK
for _, res := range qdr.Responses {
if res.Error != nil {
statusCode = http.StatusMultiStatus
}
}
if statusCode != http.StatusOK {
requestmeta.WithDownstreamStatusSource(ctx)
}
// TODO... someday :) can return protobuf for machine-machine communication
// will avoid some hops the current response workflow (for external plugins)
// 1. Plugin:
// creates: golang structs
// returns: arrow + protobuf |
// 2. Client: | direct when local/non grpc
// reads: protobuf+arrow V
// returns: golang structs
// 3. Datasource Server (eg right here):
// reads: golang structs
// returns: JSON
// 4. Query service (alerting etc):
// reads: JSON? (TODO! raw output from 1???)
// returns: JSON (after more operations)
// 5. Browser
// reads: JSON
w.WriteHeader(statusCode)
w.Header().Set("Content-Type", "application/json")
err = json.NewEncoder(w).Encode(qdr)
if err != nil {
responder.Error(err)
}
}), nil
}
pkg/registry/apis/datasource/utils.go
-25
View File
@@ -1,25 +0,0 @@
package datasource
import (
"fmt"
"strings"
)
func getDatasourceGroupNameFromPluginID(pluginId string) (string, error) {
if pluginId == "" {
return "", fmt.Errorf("bad pluginID (empty)")
}
parts := strings.Split(pluginId, "-")
if len(parts) == 1 {
return fmt.Sprintf("%s.datasource.grafana.app", parts[0]), nil
}
last := parts[len(parts)-1]
if last != "datasource" {
return "", fmt.Errorf("bad pluginID (%s)", pluginId)
}
if parts[0] == "grafana" {
parts = parts[1:] // strip the first value
}
return fmt.Sprintf("%s.datasource.grafana.app", strings.Join(parts[:len(parts)-1], "-")), nil
}
pkg/registry/apis/datasource/utils_test.go
-32
View File
@@ -1,32 +0,0 @@
package datasource
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestUtils(t *testing.T) {
// multiple flavors of the same idea
require.Equal(t, "tempo.datasource.grafana.app", getIDIgnoreError("tempo"))
require.Equal(t, "tempo.datasource.grafana.app", getIDIgnoreError("grafana-tempo-datasource"))
require.Equal(t, "tempo.datasource.grafana.app", getIDIgnoreError("tempo-datasource"))
// Multiple dashes in the name
require.Equal(t, "org-name.datasource.grafana.app", getIDIgnoreError("org-name-datasource"))
require.Equal(t, "org-name-more.datasource.grafana.app", getIDIgnoreError("org-name-more-datasource"))
require.Equal(t, "org-name-more-more.datasource.grafana.app", getIDIgnoreError("org-name-more-more-datasource"))
require.Error(t, getErrorIgnoreValue("graph-panel"))
require.Error(t, getErrorIgnoreValue("anything-notdatasource"))
}
func getIDIgnoreError(id string) string {
v, _ := getDatasourceGroupNameFromPluginID(id)
return v
}
func getErrorIgnoreValue(id string) error {
_, err := getDatasourceGroupNameFromPluginID(id)
return err
}