K8S/Shared Queries: Add Experimental PeakQ API (#80839)

First pass at a backend api built on top off app platform for shared-queries in explore and a query library with templating. Highly Experimental.

Under grafanaAPIServerWithExperimentalAPIs = true

Co-authored-by: Ryan McKinley <ryantxu@gmail.com>
Co-authored-by: Kyle Brandt <kyle@grafana.com>
This commit is contained in:
Todd Treece
2024-02-06 11:22:41 -05:00
committed by GitHub
co-authored by Ryan McKinley Kyle Brandt
parent ef9eca3a75
commit abaed01d7e
17 changed files with 1648 additions and 0 deletions
+2
View File
@@ -10,6 +10,7 @@ import (
"github.com/grafana/grafana/pkg/registry/apis/example"
"github.com/grafana/grafana/pkg/registry/apis/featuretoggle"
"github.com/grafana/grafana/pkg/registry/apis/folders"
"github.com/grafana/grafana/pkg/registry/apis/peakq"
"github.com/grafana/grafana/pkg/registry/apis/playlist"
"github.com/grafana/grafana/pkg/registry/apis/query"
"github.com/grafana/grafana/pkg/registry/apis/service"
@@ -31,6 +32,7 @@ func ProvideRegistryServiceSink(
_ *featuretoggle.FeatureFlagAPIBuilder,
_ *datasource.DataSourceAPIBuilder,
_ *folders.FolderAPIBuilder,
_ *peakq.PeakQAPIBuilder,
_ *service.ServiceAPIBuilder,
_ *query.QueryAPIBuilder,
) *Service {
+177
View File
@@ -0,0 +1,177 @@
package peakq
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/registry/generic"
"k8s.io/apiserver/pkg/registry/rest"
genericapiserver "k8s.io/apiserver/pkg/server"
"k8s.io/kube-openapi/pkg/common"
"k8s.io/kube-openapi/pkg/spec3"
"k8s.io/kube-openapi/pkg/validation/spec"
peakq "github.com/grafana/grafana/pkg/apis/peakq/v0alpha1"
"github.com/grafana/grafana/pkg/services/apiserver/builder"
"github.com/grafana/grafana/pkg/services/featuremgmt"
)
var _ builder.APIGroupBuilder = (*PeakQAPIBuilder)(nil)
// This is used just so wire has something unique to return
type PeakQAPIBuilder struct{}
func NewPeakQAPIBuilder() *PeakQAPIBuilder {
return &PeakQAPIBuilder{}
}
func RegisterAPIService(features featuremgmt.FeatureToggles, apiregistration builder.APIRegistrar) *PeakQAPIBuilder {
if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) {
return nil // skip registration unless opting into experimental apis
}
builder := NewPeakQAPIBuilder()
apiregistration.RegisterAPI(NewPeakQAPIBuilder())
return builder
}
func (b *PeakQAPIBuilder) GetAuthorizer() authorizer.Authorizer {
return nil // default authorizer is fine
}
func (b *PeakQAPIBuilder) GetGroupVersion() schema.GroupVersion {
return peakq.SchemeGroupVersion
}
func (b *PeakQAPIBuilder) InstallSchema(scheme *runtime.Scheme) error {
gv := peakq.SchemeGroupVersion
err := peakq.AddToScheme(scheme)
if err != nil {
return err
}
// Link this version to the internal representation.
// This is used for server-side-apply (PATCH), and avoids the error:
// "no kind is registered for the type"
// addKnownTypes(scheme, schema.GroupVersion{
// Group: peakq.GROUP,
// Version: runtime.APIVersionInternal,
// })
metav1.AddToGroupVersion(scheme, gv)
return scheme.SetVersionPriority(gv)
}
func (b *PeakQAPIBuilder) GetAPIGroupInfo(
scheme *runtime.Scheme,
codecs serializer.CodecFactory,
optsGetter generic.RESTOptionsGetter,
_ bool, // dual write (not relevant)
) (*genericapiserver.APIGroupInfo, error) {
apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(peakq.GROUP, scheme, metav1.ParameterCodec, codecs)
resourceInfo := peakq.QueryTemplateResourceInfo
storage := map[string]rest.Storage{}
peakqStorage, err := newStorage(scheme, optsGetter)
if err != nil {
return nil, err
}
storage[resourceInfo.StoragePath()] = peakqStorage
storage[resourceInfo.StoragePath("render")] = &renderREST{
getter: peakqStorage,
}
apiGroupInfo.VersionedResourcesStorageMap[peakq.VERSION] = storage
return &apiGroupInfo, nil
}
func (b *PeakQAPIBuilder) GetOpenAPIDefinitions() common.GetOpenAPIDefinitions {
return peakq.GetOpenAPIDefinitions
}
// NOT A GREAT APPROACH... BUT will make a UI for statically defined
func (b *PeakQAPIBuilder) GetAPIRoutes() *builder.APIRoutes {
defs := peakq.GetOpenAPIDefinitions(func(path string) spec.Ref { return spec.Ref{} })
renderedQuerySchema := defs["github.com/grafana/grafana/pkg/apis/peakq/v0alpha1.RenderedQuery"].Schema
queryTemplateSpecSchema := defs["github.com/grafana/grafana/pkg/apis/peakq/v0alpha1.QueryTemplateSpec"].Schema
params := []*spec3.Parameter{
{
ParameterProps: spec3.ParameterProps{
// Arbitrary name. It won't appear in the request URL,
// but will be used in code generated from this OAS spec
Name: "variables",
In: "query",
Schema: spec.MapProperty(spec.ArrayProperty(spec.StringProperty())),
Style: "form",
Explode: true,
Description: "Each variable is prefixed with var-{variable}={value}",
Example: map[string][]string{
"var-metricName": {"up"},
"var-another": {"first", "second"},
},
},
},
}
return &builder.APIRoutes{
Root: []builder.APIRouteHandler{
{
Path: "render",
Spec: &spec3.PathProps{
Summary: "an example at the root level",
Description: "longer description here?",
Post: &spec3.Operation{
OperationProps: spec3.OperationProps{
Parameters: params,
RequestBody: &spec3.RequestBody{
RequestBodyProps: spec3.RequestBodyProps{
Content: map[string]*spec3.MediaType{
"application/json": {
MediaTypeProps: spec3.MediaTypeProps{
Schema: &queryTemplateSpecSchema,
// Example: basicTemplateSpec,
Examples: map[string]*spec3.Example{
"test": {
ExampleProps: spec3.ExampleProps{
Summary: "hello",
Value: basicTemplateSpec,
},
},
"test2": {
ExampleProps: spec3.ExampleProps{
Summary: "hello2",
Value: basicTemplateSpec,
},
},
},
},
},
},
},
},
Responses: &spec3.Responses{
ResponsesProps: spec3.ResponsesProps{
StatusCodeResponses: map[int]*spec3.Response{
200: {
ResponseProps: spec3.ResponseProps{
Description: "OK",
Content: map[string]*spec3.MediaType{
"application/json": {
MediaTypeProps: spec3.MediaTypeProps{
Schema: &renderedQuerySchema,
},
},
},
},
},
},
},
},
},
},
},
Handler: renderPOSTHandler,
},
},
}
}
+216
View File
@@ -0,0 +1,216 @@
package peakq
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/url"
"sort"
"strings"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apiserver/pkg/registry/rest"
"github.com/grafana/grafana/pkg/apis/common/v0alpha1"
peakq "github.com/grafana/grafana/pkg/apis/peakq/v0alpha1"
"github.com/spyzhov/ajson"
)
type renderREST struct {
getter rest.Getter
}
var _ = rest.Connecter(&renderREST{})
func (r *renderREST) New() runtime.Object {
return &peakq.RenderedQuery{}
}
func (r *renderREST) Destroy() {
}
func (r *renderREST) ConnectMethods() []string {
return []string{"GET"}
}
func (r *renderREST) NewConnectOptions() (runtime.Object, bool, string) {
return nil, false, "" // true means you can use the trailing path as a variable
}
func (r *renderREST) Connect(ctx context.Context, name string, opts runtime.Object, responder rest.Responder) (http.Handler, error) {
obj, err := r.getter.Get(ctx, name, &v1.GetOptions{})
if err != nil {
return nil, err
}
template, ok := obj.(*peakq.QueryTemplate)
if !ok {
return nil, fmt.Errorf("expected template")
}
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
input, err := makeVarMapFromParams(req.URL.Query())
if err != nil {
responder.Error(err)
return
}
rq, err := Render(template.Spec, input)
if err != nil {
responder.Error(fmt.Errorf("failed to render: %w", err))
return
}
responder.Object(http.StatusOK, rq)
}), nil
}
func renderPOSTHandler(w http.ResponseWriter, req *http.Request) {
input, err := makeVarMapFromParams(req.URL.Query())
if err != nil {
_, _ = w.Write([]byte("ERROR: " + err.Error()))
w.WriteHeader(500)
return
}
var qT peakq.QueryTemplate
err = json.NewDecoder(req.Body).Decode(&qT.Spec)
if err != nil {
_, _ = w.Write([]byte("ERROR: " + err.Error()))
w.WriteHeader(500)
return
}
results, err := Render(qT.Spec, input)
if err != nil {
_, _ = w.Write([]byte("ERROR: " + err.Error()))
w.WriteHeader(500)
return
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
_ = json.NewEncoder(w).Encode(results)
}
// Replicate the grafana dashboard URL syntax
// &var-abc=1&var=abc=2&var-xyz=3...
func makeVarMapFromParams(v url.Values) (map[string][]string, error) {
input := make(map[string][]string, len(v))
for key, vals := range v {
if !strings.HasPrefix(key, "var-") {
continue
}
input[key[4:]] = vals
}
return input, nil
}
type replacement struct {
*peakq.Position
*peakq.TemplateVariable
}
func getReplacementMap(qt peakq.QueryTemplateSpec) map[int]map[string][]replacement {
byTargetPath := make(map[int]map[string][]replacement)
varMap := make(map[string]*peakq.TemplateVariable, len(qt.Variables))
for i, v := range qt.Variables {
varMap[v.Key] = &qt.Variables[i]
}
for i, target := range qt.Targets {
if byTargetPath[i] == nil {
byTargetPath[i] = make(map[string][]replacement)
}
for k, vReps := range target.Variables {
for rI, rep := range vReps {
byTargetPath[i][rep.Path] = append(byTargetPath[i][rep.Path],
replacement{
Position: vReps[rI].Position,
TemplateVariable: varMap[k],
},
)
}
}
}
for idx, byTargetIdx := range byTargetPath {
for path := range byTargetIdx {
sort.Slice(byTargetPath[idx][path], func(i, j int) bool {
return byTargetPath[idx][path][i].Start < byTargetPath[idx][path][j].Start
})
}
}
return byTargetPath
}
func Render(qt peakq.QueryTemplateSpec, selectedValues map[string][]string) (*peakq.RenderedQuery, error) {
targets := qt.DeepCopy().Targets
rawTargetObjects := make([]*ajson.Node, len(qt.Targets))
for i, t := range qt.Targets {
b, err := t.Properties.MarshalJSON()
if err != nil {
return nil, err
}
rawTargetObjects[i], err = ajson.Unmarshal(b)
if err != nil {
return nil, err
}
}
rm := getReplacementMap(qt)
for targetIdx, byTargetIdx := range rm {
for path, reps := range byTargetIdx {
o := rawTargetObjects[targetIdx]
nodes, err := o.JSONPath(path)
if err != nil {
return nil, err
}
if len(nodes) != 1 {
return nil, fmt.Errorf("expected one lead node at path %v but got %v", path, len(nodes))
}
n := nodes[0]
if !n.IsString() {
return nil, fmt.Errorf("only string type leaf notes supported currently, %v is not a string", path)
}
s := n.String()
s = s[1 : len(s)-1]
var offSet int64
for _, r := range reps {
// I think breaks with utf...something...?
// TODO: Probably simpler to store the non-template parts and insert the values into that, then don't have to track
// offsets
if r.Position == nil {
return nil, fmt.Errorf("nil position not support yet, will be full replacement")
}
if len(selectedValues[r.Key]) != 1 {
return nil, fmt.Errorf("selected value missing, or more then one provided")
}
value := selectedValues[r.Key][0]
s = s[:r.Start+offSet] + value + s[r.End+offSet:]
offSet = int64(len(value)+int(offSet)) - (r.End - r.Start)
}
if err = n.SetString(s); err != nil {
return nil, err
}
}
}
for i, aT := range rawTargetObjects {
raw, err := ajson.Marshal(aT)
if err != nil {
return nil, err
}
u := v0alpha1.Unstructured{}
err = u.UnmarshalJSON(raw)
if err != nil {
return nil, err
}
targets[i].Properties = u
}
return &peakq.RenderedQuery{
Targets: targets,
}, nil
}
@@ -0,0 +1,78 @@
package peakq
import (
"github.com/grafana/grafana-plugin-sdk-go/data"
common "github.com/grafana/grafana/pkg/apis/common/v0alpha1"
peakq "github.com/grafana/grafana/pkg/apis/peakq/v0alpha1"
)
var basicTemplateSpec = peakq.QueryTemplateSpec{
Title: "Test",
Variables: []peakq.TemplateVariable{
{
Key: "metricName",
DefaultValues: []string{`down`},
},
},
Targets: []peakq.Target{
{
DataType: data.FrameTypeUnknown,
//DataTypeVersion: data.FrameTypeVersion{0, 0},
Variables: map[string][]peakq.VariableReplacement{
"metricName": {
{
Path: "$.expr",
Position: &peakq.Position{
Start: 0,
End: 10,
},
},
{
Path: "$.expr",
Position: &peakq.Position{
Start: 13,
End: 23,
},
},
},
},
Properties: common.Unstructured{
Object: map[string]any{
"refId": "A", // TODO: Set when Where?
"datasource": map[string]any{
"type": "prometheus",
"uid": "foo", // TODO: Probably a default templating thing to set this.
},
"editorMode": "builder",
"expr": "metricName + metricName + 42",
"instant": true,
"range": false,
"exemplar": false,
},
},
},
},
}
var basicTemplateRenderedTargets = []peakq.Target{
{
DataType: data.FrameTypeUnknown,
//DataTypeVersion: data.FrameTypeVersion{0, 0},
Properties: common.Unstructured{
Object: map[string]any{
"refId": "A", // TODO: Set when Where?
"datasource": map[string]any{
"type": "prometheus",
"uid": "foo", // TODO: Probably a default templating thing to set this.
},
"editorMode": "builder",
"expr": "up + up + 42",
"instant": true,
"range": false,
"exemplar": false,
},
},
},
}
@@ -0,0 +1,17 @@
package peakq
import (
"encoding/json"
"fmt"
"testing"
"github.com/stretchr/testify/require"
)
func TestRender(t *testing.T) {
rT, err := Render(basicTemplateSpec, map[string][]string{"metricName": {"up"}})
require.NoError(t, err)
require.Equal(t, basicTemplateRenderedTargets[0].Properties.Object["expr"], rT.Targets[0].Properties.Object["expr"])
b, _ := json.MarshalIndent(basicTemplateSpec, "", " ")
fmt.Println(string(b))
}
+182
View File
@@ -0,0 +1,182 @@
package peakq
import (
"testing"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/stretchr/testify/require"
common "github.com/grafana/grafana/pkg/apis/common/v0alpha1"
peakq "github.com/grafana/grafana/pkg/apis/peakq/v0alpha1"
)
var nestedFieldRender = peakq.QueryTemplateSpec{
Title: "Test",
Variables: []peakq.TemplateVariable{
{
Key: "metricName",
},
},
Targets: []peakq.Target{
{
DataType: data.FrameTypeUnknown,
//DataTypeVersion: data.FrameTypeVersion{0, 0},
Variables: map[string][]peakq.VariableReplacement{
"metricName": {
{
Path: "$.nestedObject.anArray[0]",
Position: &peakq.Position{
Start: 0,
End: 3,
},
},
},
},
Properties: common.Unstructured{
Object: map[string]any{
"nestedObject": map[string]any{
"anArray": []any{"foo", .2},
},
},
},
},
},
}
var nestedFieldRenderedTargets = []peakq.Target{
{
DataType: data.FrameTypeUnknown,
Variables: map[string][]peakq.VariableReplacement{
"metricName": {
{
Path: "$.nestedObject.anArray[0]",
Position: &peakq.Position{
Start: 0,
End: 3,
},
},
},
},
//DataTypeVersion: data.FrameTypeVersion{0, 0},
Properties: common.Unstructured{
Object: map[string]any{
"nestedObject": map[string]any{
"anArray": []any{"up", .2},
},
},
},
},
}
func TestNestedFieldRender(t *testing.T) {
rT, err := Render(nestedFieldRender, map[string][]string{"metricName": {"up"}})
require.NoError(t, err)
require.Equal(t,
nestedFieldRenderedTargets,
rT.Targets,
)
}
var multiVarTemplate = peakq.QueryTemplateSpec{
Title: "Test",
Variables: []peakq.TemplateVariable{
{
Key: "metricName",
},
{
Key: "anotherMetric",
},
},
Targets: []peakq.Target{
{
DataType: data.FrameTypeUnknown,
//DataTypeVersion: data.FrameTypeVersion{0, 0},
Variables: map[string][]peakq.VariableReplacement{
"metricName": {
{
Path: "$.expr",
Position: &peakq.Position{
Start: 4,
End: 14,
},
},
{
Path: "$.expr",
Position: &peakq.Position{
Start: 37,
End: 47,
},
},
},
"anotherMetric": {
{
Path: "$.expr",
Position: &peakq.Position{
Start: 21,
End: 34,
},
},
},
},
Properties: common.Unstructured{
Object: map[string]any{
"expr": "1 + metricName + 1 + anotherMetric + metricName",
},
},
},
},
}
var multiVarRenderedTargets = []peakq.Target{
{
DataType: data.FrameTypeUnknown,
Variables: map[string][]peakq.VariableReplacement{
"metricName": {
{
Path: "$.expr",
Position: &peakq.Position{
Start: 4,
End: 14,
},
},
{
Path: "$.expr",
Position: &peakq.Position{
Start: 37,
End: 47,
},
},
},
"anotherMetric": {
{
Path: "$.expr",
Position: &peakq.Position{
Start: 21,
End: 34,
},
},
},
},
//DataTypeVersion: data.FrameTypeVersion{0, 0},
Properties: common.Unstructured{
Object: map[string]any{
"expr": "1 + up + 1 + sloths_do_like_a_good_nap + up",
},
},
},
}
func TestMultiVarTemplate(t *testing.T) {
rT, err := Render(multiVarTemplate, map[string][]string{
"metricName": {"up"},
"anotherMetric": {"sloths_do_like_a_good_nap"},
})
require.NoError(t, err)
require.Equal(t,
multiVarRenderedTargets,
rT.Targets,
)
}
+62
View File
@@ -0,0 +1,62 @@
package peakq
import (
"fmt"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apiserver/pkg/registry/generic"
genericregistry "k8s.io/apiserver/pkg/registry/generic/registry"
peakq "github.com/grafana/grafana/pkg/apis/peakq/v0alpha1"
grafanaregistry "github.com/grafana/grafana/pkg/services/apiserver/registry/generic"
grafanarest "github.com/grafana/grafana/pkg/services/apiserver/rest"
"github.com/grafana/grafana/pkg/services/apiserver/utils"
)
var _ grafanarest.Storage = (*storage)(nil)
type storage struct {
*genericregistry.Store
}
func newStorage(scheme *runtime.Scheme, optsGetter generic.RESTOptionsGetter) (*storage, error) {
strategy := grafanaregistry.NewStrategy(scheme)
resourceInfo := peakq.QueryTemplateResourceInfo
store := &genericregistry.Store{
NewFunc: resourceInfo.NewFunc,
NewListFunc: resourceInfo.NewListFunc,
PredicateFunc: grafanaregistry.Matcher,
DefaultQualifiedResource: resourceInfo.GroupResource(),
SingularQualifiedResource: resourceInfo.SingularGroupResource(),
TableConvertor: utils.NewTableConverter(
resourceInfo.GroupResource(),
[]metav1.TableColumnDefinition{
{Name: "Name", Type: "string", Format: "name"},
{Name: "Title", Type: "string"},
{Name: "Created At", Type: "date"},
},
func(obj any) ([]interface{}, error) {
m, ok := obj.(*peakq.QueryTemplate)
if !ok {
return nil, fmt.Errorf("expected query template")
}
return []interface{}{
m.Name,
m.Spec.Title,
m.CreationTimestamp.UTC().Format(time.RFC3339),
}, nil
},
),
CreateStrategy: strategy,
UpdateStrategy: strategy,
DeleteStrategy: strategy,
}
options := &generic.StoreOptions{RESTOptions: optsGetter, AttrFunc: grafanaregistry.GetAttrs}
if err := store.CompleteWithOptions(options); err != nil {
return nil, err
}
return &storage{Store: store}, nil
}
+2
View File
@@ -9,6 +9,7 @@ import (
"github.com/grafana/grafana/pkg/registry/apis/example"
"github.com/grafana/grafana/pkg/registry/apis/featuretoggle"
"github.com/grafana/grafana/pkg/registry/apis/folders"
"github.com/grafana/grafana/pkg/registry/apis/peakq"
"github.com/grafana/grafana/pkg/registry/apis/playlist"
"github.com/grafana/grafana/pkg/registry/apis/query"
"github.com/grafana/grafana/pkg/registry/apis/service"
@@ -31,6 +32,7 @@ var WireSet = wire.NewSet(
featuretoggle.RegisterAPIService,
datasource.RegisterAPIService,
folders.RegisterAPIService,
peakq.RegisterAPIService,
service.RegisterAPIService,
query.RegisterAPIService,
)