public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Expressions: Move GEL into core as expressions (#29072)

Browse Source 
* comes from grafana/gel-app
* remove transform plugin code
* move __expr__ and -100 constants to expr pkg
* set OrgID on request plugin context
* use gtime for resample duration
* in resample, rename "rule" to "window", use gtime for duration, parse duration before exec
* remove gel entry from plugins-bundled/external.json
which creates an empty array for plugins
This commit is contained in:
Kyle Brandt
2020-11-19 07:17:00 -05:00
committed by GitHub
parent f01c3f35e1
commit 0cb29d337a
44 changed files with 5770 additions and 401 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/plugins/backendplugin/grpcplugin/client.go
+2 -4
View File
@@ -71,7 +71,6 @@ func getV2PluginSet() goplugin.PluginSet {
"diagnostics": &grpcplugin.DiagnosticsGRPCPlugin{},
"resource": &grpcplugin.ResourceGRPCPlugin{},
"data": &grpcplugin.DataGRPCPlugin{},
"transform": &grpcplugin.TransformGRPCPlugin{},
"renderer": &pluginextensionv2.RendererGRPCPlugin{},
}
}
@@ -116,7 +115,6 @@ type LegacyClient struct {
// Client client for communicating with a plugin using the current (v2) plugin protocol.
type Client struct {
DataPlugin grpcplugin.DataClient
TransformPlugin grpcplugin.TransformClient
RendererPlugin pluginextensionv2.RendererPlugin
DataPlugin grpcplugin.DataClient
RendererPlugin pluginextensionv2.RendererPlugin
}
pkg/plugins/backendplugin/grpcplugin/client_v2.go
+2 -36
View File
@@ -21,7 +21,6 @@ type clientV2 struct {
grpcplugin.DiagnosticsClient
grpcplugin.ResourceClient
grpcplugin.DataClient
grpcplugin.TransformClient
pluginextensionv2.RendererPlugin
}
@@ -41,11 +40,6 @@ func newClientV2(descriptor PluginDescriptor, logger log.Logger, rpcClient plugi
return nil, err
}
rawTransform, err := rpcClient.Dispense("transform")
if err != nil {
return nil, err
}
rawRenderer, err := rpcClient.Dispense("renderer")
if err != nil {
return nil, err
@@ -70,12 +64,6 @@ func newClientV2(descriptor PluginDescriptor, logger log.Logger, rpcClient plugi
}
}
if rawTransform != nil {
if plugin, ok := rawTransform.(grpcplugin.TransformClient); ok {
c.TransformClient = instrumentTransformPlugin(plugin)
}
}
if rawRenderer != nil {
if plugin, ok := rawRenderer.(pluginextensionv2.RendererPlugin); ok {
c.RendererPlugin = plugin
@@ -84,9 +72,8 @@ func newClientV2(descriptor PluginDescriptor, logger log.Logger, rpcClient plugi
if descriptor.startFns.OnStart != nil {
client := &Client{
DataPlugin: c.DataClient,
TransformPlugin: c.TransformClient,
RendererPlugin: c.RendererPlugin,
DataPlugin: c.DataClient,
RendererPlugin: c.RendererPlugin,
}
if err := descriptor.startFns.OnStart(descriptor.pluginID, client, logger); err != nil {
return nil, err
@@ -189,24 +176,3 @@ func instrumentDataClient(plugin grpcplugin.DataClient) grpcplugin.DataClient {
return resp, err
})
}
type transformPluginTransformDataFunc func(ctx context.Context, req *pluginv2.QueryDataRequest, callback grpcplugin.TransformDataCallBack) (*pluginv2.QueryDataResponse, error)
func (fn transformPluginTransformDataFunc) TransformData(ctx context.Context, req *pluginv2.QueryDataRequest, callback grpcplugin.TransformDataCallBack) (*pluginv2.QueryDataResponse, error) {
return fn(ctx, req, callback)
}
func instrumentTransformPlugin(plugin grpcplugin.TransformClient) grpcplugin.TransformClient {
if plugin == nil {
return nil
}
return transformPluginTransformDataFunc(func(ctx context.Context, req *pluginv2.QueryDataRequest, callback grpcplugin.TransformDataCallBack) (*pluginv2.QueryDataResponse, error) {
var resp *pluginv2.QueryDataResponse
err := backendplugin.InstrumentTransformDataRequest(req.PluginContext.PluginId, func() (innerErr error) {
resp, innerErr = plugin.TransformData(ctx, req, callback)
return
})
return resp, err
})
}
pkg/plugins/backendplugin/instrumentation.go
-5
View File
@@ -65,11 +65,6 @@ func InstrumentQueryDataRequest(pluginID string, fn func() error) error {
return instrumentPluginRequest(pluginID, "queryData", fn)
}
// InstrumentTransformDataRequest instruments success rate and latency of transform data request.
func InstrumentTransformDataRequest(pluginID string, fn func() error) error {
return instrumentPluginRequest(pluginID, "transformData", fn)
}
// InstrumentQueryDataHandler wraps a backend.QueryDataHandler with instrumentation of success rate and latency.
func InstrumentQueryDataHandler(handler backend.QueryDataHandler) backend.QueryDataHandler {
if handler == nil {
pkg/plugins/plugins.go
+1 -13
View File
@@ -31,7 +31,6 @@ var (
Plugins map[string]*PluginBase
PluginTypes map[string]interface{}
Renderer *RendererPlugin
Transform *TransformPlugin
GrafanaLatestVersion string
GrafanaHasUpdate bool
@@ -82,7 +81,6 @@ func (pm *PluginManager) Init() error {
"datasource": DataSourcePlugin{},
"app": AppPlugin{},
"renderer": RendererPlugin{},
"transform": TransformPlugin{},
}
pluginScanningErrors = map[string]*PluginError{}
@@ -357,16 +355,6 @@ func (s *PluginScanner) loadPlugin(pluginJSONFilePath string) error {
return errors.New("did not find type or id properties in plugin.json")
}
// The expressions feature toggle corresponds to transform plug-ins.
if pluginCommon.Type == "transform" {
isEnabled := s.cfg.IsExpressionsEnabled()
if !isEnabled {
s.log.Debug("Transform plugin is disabled since the expressions feature toggle is not enabled",
"pluginID", pluginCommon.Id)
return nil
}
}
pluginCommon.PluginDir = filepath.Dir(pluginJSONFilePath)
pluginCommon.Signature = getPluginSignatureState(s.log, &pluginCommon)
@@ -376,7 +364,7 @@ func (s *PluginScanner) loadPlugin(pluginJSONFilePath string) error {
}
func (*PluginScanner) IsBackendOnlyPlugin(pluginType string) bool {
return pluginType == "renderer" || pluginType == "transform"
return pluginType == "renderer"
}
// validateSignature validates a plugin's signature.
pkg/plugins/plugins_test.go
-23
View File
@@ -142,29 +142,6 @@ func TestPluginManager_Init(t *testing.T) {
assert.Empty(t, fm.registeredPlugins)
})
t.Run("Transform plugins should be loaded when expressions feature is on", func(t *testing.T) {
origPluginsPath := setting.PluginsPath
t.Cleanup(func() {
setting.PluginsPath = origPluginsPath
})
setting.PluginsPath = "testdata/behind-feature-flag"
fm := &fakeBackendPluginManager{}
pm := &PluginManager{
Cfg: &setting.Cfg{
FeatureToggles: map[string]bool{
"expressions": true,
},
},
BackendPluginManager: fm,
}
err := pm.Init()
require.NoError(t, err)
require.Empty(t, pm.scanningErrors)
assert.Equal(t, []string{"gel"}, fm.registeredPlugins)
})
t.Run("With nested plugin duplicating parent", func(t *testing.T) {
origPluginsPath := setting.PluginsPath
t.Cleanup(func() {
pkg/plugins/testdata/behind-feature-flag/gel/MANIFEST.txt
Vendored
-23
View File
@@ -1,23 +0,0 @@
-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA512
{
"plugin": "gel",
"version": "1.0.0",
"files": {
"plugin.json": "b9b3bb0dab3c4655a929a1e48a957466e3e2717992bdd29da27e5eed2fae090c"
},
"time": 1589274667427,
"keyId": "7e4d0c6a708866e7"
}
-----BEGIN PGP SIGNATURE-----
Version: OpenPGP.js v4.10.1
Comment: https://openpgpjs.org
wqIEARMKAAYFAl66aCsACgkQfk0ManCIZufDMAIJAWoNVihI9ZSBpUpgXrzY
XXsI3OmHuVpzrv6M6bk5jYdzY4SyzZmdw4CB51TIDJW9SnUajlXxWLXGYY+w
B2rSYvuhAgkBlG9w5OV3jcyg/wfUrIcCO5XRHMydCg0hIOznClzuG0uWn3wm
d4RT/ap1ezislQ/91zvhsLgAIztZlm3EsNBv7sI=
=WPLw
-----END PGP SIGNATURE-----
pkg/plugins/testdata/behind-feature-flag/gel/plugin.json
Vendored
-14
View File
@@ -1,14 +0,0 @@
{
"type": "transform",
"name": "GEL",
"id": "gel",
"backend": true,
"info": {
"description": "Test",
"version": "1.0.0",
"author": {
"name": "Grafana Labs",
"url": "https://grafana.com"
}
}
}
pkg/plugins/transform_plugin.go
-228
View File
@@ -1,228 +0,0 @@
package plugins
import (
"context"
"encoding/json"
"fmt"
"path/filepath"
"strconv"
sdkgrpcplugin "github.com/grafana/grafana-plugin-sdk-go/backend/grpcplugin"
"github.com/grafana/grafana-plugin-sdk-go/genproto/pluginv2"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins/backendplugin"
"github.com/grafana/grafana/pkg/plugins/backendplugin/grpcplugin"
"github.com/grafana/grafana/pkg/plugins/datasource/wrapper"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/grafana/grafana/pkg/util/errutil"
)
type TransformPlugin struct {
PluginBase
Executable string `json:"executable,omitempty"`
*TransformWrapper
}
func (p *TransformPlugin) Load(decoder *json.Decoder, base *PluginBase, backendPluginManager backendplugin.Manager) error {
if err := decoder.Decode(p); err != nil {
return err
}
if err := p.registerPlugin(base); err != nil {
return err
}
cmd := ComposePluginStartCommand(p.Executable)
fullpath := filepath.Join(p.PluginDir, cmd)
factory := grpcplugin.NewBackendPlugin(p.Id, fullpath, grpcplugin.PluginStartFuncs{
OnStart: p.onPluginStart,
})
if err := backendPluginManager.Register(p.Id, factory); err != nil {
return errutil.Wrapf(err, "Failed to register backend plugin")
}
Transform = p
return nil
}
func (p *TransformPlugin) onPluginStart(pluginID string, client *grpcplugin.Client, logger log.Logger) error {
p.TransformWrapper = NewTransformWrapper(logger, client.TransformPlugin)
if client.DataPlugin != nil {
tsdb.RegisterTsdbQueryEndpoint(pluginID, func(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
return wrapper.NewDatasourcePluginWrapperV2(logger, p.Id, p.Type, client.DataPlugin), nil
})
}
return nil
}
// ...
// Wrapper Code
// ...
func NewTransformWrapper(log log.Logger, plugin sdkgrpcplugin.TransformClient) *TransformWrapper {
return &TransformWrapper{plugin, log, &transformCallback{log}}
}
type TransformWrapper struct {
sdkgrpcplugin.TransformClient
logger log.Logger
Callback *transformCallback
}
func (tw *TransformWrapper) Transform(ctx context.Context, query *tsdb.TsdbQuery) (*tsdb.Response, error) {
pbQuery := &pluginv2.QueryDataRequest{
PluginContext: &pluginv2.PluginContext{
// TODO: Things probably
},
Queries: []*pluginv2.DataQuery{},
}
for _, q := range query.Queries {
modelJSON, err := q.Model.MarshalJSON()
if err != nil {
return nil, err
}
pbQuery.Queries = append(pbQuery.Queries, &pluginv2.DataQuery{
Json: modelJSON,
IntervalMS: q.IntervalMs,
RefId: q.RefId,
MaxDataPoints: q.MaxDataPoints,
QueryType: q.QueryType,
TimeRange: &pluginv2.TimeRange{
ToEpochMS: query.TimeRange.GetToAsMsEpoch(),
FromEpochMS: query.TimeRange.GetFromAsMsEpoch(),
},
})
}
pbRes, err := tw.TransformClient.TransformData(ctx, pbQuery, tw.Callback)
if err != nil {
return nil, err
}
tR := &tsdb.Response{
Results: make(map[string]*tsdb.QueryResult, len(pbRes.Responses)),
}
for refID, res := range pbRes.Responses {
tRes := &tsdb.QueryResult{
RefId: refID,
Dataframes: tsdb.NewEncodedDataFrames(res.Frames),
}
if len(res.JsonMeta) != 0 {
tRes.Meta = simplejson.NewFromAny(res.JsonMeta)
}
if res.Error != "" {
tRes.Error = fmt.Errorf(res.Error)
tRes.ErrorString = res.Error
}
tR.Results[refID] = tRes
}
return tR, nil
}
type transformCallback struct {
logger log.Logger
}
func (s *transformCallback) QueryData(ctx context.Context, req *pluginv2.QueryDataRequest) (*pluginv2.QueryDataResponse, error) {
if len(req.Queries) == 0 {
return nil, fmt.Errorf("zero queries found in datasource request")
}
datasourceID := int64(0)
if req.PluginContext.DataSourceInstanceSettings != nil {
datasourceID = req.PluginContext.DataSourceInstanceSettings.Id
}
getDsInfo := &models.GetDataSourceByIdQuery{
OrgId: req.PluginContext.OrgId,
Id: datasourceID,
}
if err := bus.Dispatch(getDsInfo); err != nil {
return nil, fmt.Errorf("could not find datasource: %w", err)
}
// Convert plugin-model (datasource) queries to tsdb queries
queries := make([]*tsdb.Query, len(req.Queries))
for i, query := range req.Queries {
sj, err := simplejson.NewJson(query.Json)
if err != nil {
return nil, err
}
queries[i] = &tsdb.Query{
RefId: query.RefId,
IntervalMs: query.IntervalMS,
MaxDataPoints: query.MaxDataPoints,
QueryType: query.QueryType,
DataSource: getDsInfo.Result,
Model: sj,
}
}
// For now take Time Range from first query.
timeRange := tsdb.NewTimeRange(strconv.FormatInt(req.Queries[0].TimeRange.FromEpochMS, 10), strconv.FormatInt(req.Queries[0].TimeRange.ToEpochMS, 10))
tQ := &tsdb.TsdbQuery{
TimeRange: timeRange,
Queries: queries,
}
// Execute the converted queries
tsdbRes, err := tsdb.HandleRequest(ctx, getDsInfo.Result, tQ)
if err != nil {
return nil, err
}
// Convert tsdb results (map) to plugin-model/datasource (slice) results.
// Only error, tsdb.Series, and encoded Dataframes responses are mapped.
responses := make(map[string]*pluginv2.DataResponse, len(tsdbRes.Results))
for refID, res := range tsdbRes.Results {
pRes := &pluginv2.DataResponse{}
if res.Error != nil {
pRes.Error = res.Error.Error()
}
if res.Dataframes != nil {
encoded, err := res.Dataframes.Encoded()
if err != nil {
return nil, err
}
pRes.Frames = encoded
responses[refID] = pRes
continue
}
for _, series := range res.Series {
frame, err := tsdb.SeriesToFrame(series)
frame.RefID = refID
if err != nil {
return nil, err
}
encFrame, err := frame.MarshalArrow()
if err != nil {
return nil, err
}
pRes.Frames = append(pRes.Frames, encFrame)
}
if res.Meta != nil {
b, err := res.Meta.MarshalJSON()
if err != nil {
s.logger.Error("failed to marshal json metadata", err)
}
pRes.JsonMeta = b
}
responses[refID] = pRes
}
return &pluginv2.QueryDataResponse{
Responses: responses,
}, nil
}