Backend plugins: Refactor to allow shared contract between core and external backend plugins (#25472)

Refactor to allow shared contract between core and external backend plugins 
allowing core backend data sources in Grafana to be implemented in same 
way as an external backend plugin.
Use v0.67.0 of sdk.
Add tests for verifying plugin is restarted when process is killed.
Enable strict linting for backendplugin packages
This commit is contained in:
Marcus Efraimsson
2020-06-11 16:14:05 +02:00
committed by GitHub
parent 40b3473a10
commit c0f3b2929c
29 changed files with 1495 additions and 612 deletions
@@ -0,0 +1,128 @@
package grpcplugin
import (
"os/exec"
datasourceV1 "github.com/grafana/grafana-plugin-model/go/datasource"
rendererV1 "github.com/grafana/grafana-plugin-model/go/renderer"
"github.com/grafana/grafana-plugin-sdk-go/backend/grpcplugin"
sdkgrpcplugin "github.com/grafana/grafana-plugin-sdk-go/backend/grpcplugin"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/plugins/backendplugin"
"github.com/grafana/grafana/pkg/plugins/backendplugin/pluginextensionv2"
goplugin "github.com/hashicorp/go-plugin"
)
const (
// DefaultProtocolVersion is the protocol version assumed for legacy clients that don't specify
// a particular version or version 1 during their handshake. This is currently the version used
// since Grafana launched support for backend plugins.
DefaultProtocolVersion = 1
)
// Handshake is the HandshakeConfig used to configure clients and servers.
var handshake = goplugin.HandshakeConfig{
// The ProtocolVersion is the version that must match between Grafana core
// and Grafana plugins. This should be bumped whenever a (breaking) change
// happens in one or the other that makes it so that they can't safely communicate.
ProtocolVersion: DefaultProtocolVersion,
// The magic cookie values should NEVER be changed.
MagicCookieKey: sdkgrpcplugin.MagicCookieKey,
MagicCookieValue: sdkgrpcplugin.MagicCookieValue,
}
func newClientConfig(executablePath string, env []string, logger log.Logger, versionedPlugins map[int]goplugin.PluginSet) *goplugin.ClientConfig {
cmd := exec.Command(executablePath)
cmd.Env = env
return &goplugin.ClientConfig{
Cmd: cmd,
HandshakeConfig: handshake,
VersionedPlugins: versionedPlugins,
Logger: logWrapper{Logger: logger},
AllowedProtocols: []goplugin.Protocol{goplugin.ProtocolGRPC},
}
}
// LegacyStartFunc callback function called when a plugin with old plugin protocol is started.
type LegacyStartFunc func(pluginID string, client *LegacyClient, logger log.Logger) error
// StartFunc callback function called when a plugin with current plugin protocol version is started.
type StartFunc func(pluginID string, client *Client, logger log.Logger) error
// PluginStartFuncs functions called for plugin when started.
type PluginStartFuncs struct {
OnLegacyStart LegacyStartFunc
OnStart StartFunc
}
// PluginDescriptor is a descriptor used for registering backend plugins.
type PluginDescriptor struct {
pluginID string
executablePath string
managed bool
versionedPlugins map[int]goplugin.PluginSet
startFns PluginStartFuncs
}
// PluginID returns the plugin ID.
func (pd PluginDescriptor) PluginID() string {
return pd.pluginID
}
// getV2PluginSet returns list of plugins supported on v2.
func getV2PluginSet() goplugin.PluginSet {
return goplugin.PluginSet{
"diagnostics": &sdkgrpcplugin.DiagnosticsGRPCPlugin{},
"resource": &sdkgrpcplugin.ResourceGRPCPlugin{},
"data": &sdkgrpcplugin.DataGRPCPlugin{},
"transform": &sdkgrpcplugin.TransformGRPCPlugin{},
"renderer": &pluginextensionv2.RendererGRPCPlugin{},
}
}
// NewBackendPlugin creates a new backend plugin factory used for registering a backend plugin.
func NewBackendPlugin(pluginID, executablePath string, startFns PluginStartFuncs) backendplugin.PluginFactoryFunc {
return New(PluginDescriptor{
pluginID: pluginID,
executablePath: executablePath,
managed: true,
versionedPlugins: map[int]goplugin.PluginSet{
DefaultProtocolVersion: {
pluginID: &datasourceV1.DatasourcePluginImpl{},
},
sdkgrpcplugin.ProtocolVersion: getV2PluginSet(),
},
startFns: startFns,
})
}
// NewRendererPlugin creates a new renderer plugin factory used for registering a backend renderer plugin.
func NewRendererPlugin(pluginID, executablePath string, startFns PluginStartFuncs) backendplugin.PluginFactoryFunc {
return New(PluginDescriptor{
pluginID: pluginID,
executablePath: executablePath,
managed: false,
versionedPlugins: map[int]goplugin.PluginSet{
DefaultProtocolVersion: {
pluginID: &rendererV1.RendererPluginImpl{},
},
sdkgrpcplugin.ProtocolVersion: getV2PluginSet(),
},
startFns: startFns,
})
}
// LegacyClient client for communicating with a plugin using the v1 plugin protocol.
type LegacyClient struct {
DatasourcePlugin datasourceV1.DatasourcePlugin
RendererPlugin rendererV1.RendererPlugin
}
// Client client for communicating with a plugin using the current (v2) plugin protocol.
type Client struct {
DataPlugin grpcplugin.DataClient
TransformPlugin grpcplugin.TransformClient
RendererPlugin pluginextensionv2.RendererPlugin
}
@@ -0,0 +1,84 @@
package grpcplugin
import (
"context"
datasourceV1 "github.com/grafana/grafana-plugin-model/go/datasource"
rendererV1 "github.com/grafana/grafana-plugin-model/go/renderer"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/plugins/backendplugin"
"github.com/hashicorp/go-plugin"
)
type clientV1 struct {
logger log.Logger
datasourceV1.DatasourcePlugin
rendererV1.RendererPlugin
}
func newClientV1(descriptor PluginDescriptor, logger log.Logger, rpcClient plugin.ClientProtocol) (pluginClient, error) {
logger.Warn("Plugin uses a deprecated version of Grafana's backend plugin system which will be removed in a future release. " +
"Consider upgrading to a newer plugin version or reach out to the plugin repository/developer and request an upgrade.")
raw, err := rpcClient.Dispense(descriptor.pluginID)
if err != nil {
return nil, err
}
c := clientV1{
logger: logger,
}
if plugin, ok := raw.(datasourceV1.DatasourcePlugin); ok {
c.DatasourcePlugin = instrumentDatasourcePluginV1(plugin)
}
if plugin, ok := raw.(rendererV1.RendererPlugin); ok {
c.RendererPlugin = plugin
}
if descriptor.startFns.OnLegacyStart != nil {
legacyClient := &LegacyClient{
DatasourcePlugin: c.DatasourcePlugin,
RendererPlugin: c.RendererPlugin,
}
if err := descriptor.startFns.OnLegacyStart(descriptor.pluginID, legacyClient, logger); err != nil {
return nil, err
}
}
return &c, nil
}
func (c *clientV1) CollectMetrics(ctx context.Context) (*backend.CollectMetricsResult, error) {
return nil, backendplugin.ErrMethodNotImplemented
}
func (c *clientV1) CheckHealth(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) {
return nil, backendplugin.ErrMethodNotImplemented
}
func (c *clientV1) CallResource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error {
return backendplugin.ErrMethodNotImplemented
}
type datasourceV1QueryFunc func(ctx context.Context, req *datasourceV1.DatasourceRequest) (*datasourceV1.DatasourceResponse, error)
func (fn datasourceV1QueryFunc) Query(ctx context.Context, req *datasourceV1.DatasourceRequest) (*datasourceV1.DatasourceResponse, error) {
return fn(ctx, req)
}
func instrumentDatasourcePluginV1(plugin datasourceV1.DatasourcePlugin) datasourceV1.DatasourcePlugin {
if plugin == nil {
return nil
}
return datasourceV1QueryFunc(func(ctx context.Context, req *datasourceV1.DatasourceRequest) (*datasourceV1.DatasourceResponse, error) {
var resp *datasourceV1.DatasourceResponse
err := backendplugin.InstrumentQueryDataRequest(req.Datasource.Type, func() (innerErr error) {
resp, innerErr = plugin.Query(ctx, req)
return
})
return resp, err
})
}
@@ -0,0 +1,212 @@
package grpcplugin
import (
"context"
"io"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/backend/grpcplugin"
"github.com/grafana/grafana-plugin-sdk-go/genproto/pluginv2"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/plugins/backendplugin"
"github.com/grafana/grafana/pkg/plugins/backendplugin/pluginextensionv2"
"github.com/grafana/grafana/pkg/util/errutil"
"github.com/hashicorp/go-plugin"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
type clientV2 struct {
grpcplugin.DiagnosticsClient
grpcplugin.ResourceClient
grpcplugin.DataClient
grpcplugin.TransformClient
pluginextensionv2.RendererPlugin
}
func newClientV2(descriptor PluginDescriptor, logger log.Logger, rpcClient plugin.ClientProtocol) (pluginClient, error) {
rawDiagnostics, err := rpcClient.Dispense("diagnostics")
if err != nil {
return nil, err
}
rawResource, err := rpcClient.Dispense("resource")
if err != nil {
return nil, err
}
rawData, err := rpcClient.Dispense("data")
if err != nil {
return nil, err
}
rawTransform, err := rpcClient.Dispense("transform")
if err != nil {
return nil, err
}
rawRenderer, err := rpcClient.Dispense("renderer")
if err != nil {
return nil, err
}
c := clientV2{}
if rawDiagnostics != nil {
if plugin, ok := rawDiagnostics.(grpcplugin.DiagnosticsClient); ok {
c.DiagnosticsClient = plugin
}
}
if rawResource != nil {
if plugin, ok := rawResource.(grpcplugin.ResourceClient); ok {
c.ResourceClient = plugin
}
}
if rawData != nil {
if plugin, ok := rawData.(grpcplugin.DataClient); ok {
c.DataClient = instrumentDataClient(plugin)
}
}
if rawTransform != nil {
if plugin, ok := rawTransform.(grpcplugin.TransformClient); ok {
c.TransformClient = instrumentTransformPlugin(plugin)
}
}
if rawRenderer != nil {
if plugin, ok := rawRenderer.(pluginextensionv2.RendererPlugin); ok {
c.RendererPlugin = plugin
}
}
if descriptor.startFns.OnStart != nil {
client := &Client{
DataPlugin: c.DataClient,
TransformPlugin: c.TransformClient,
RendererPlugin: c.RendererPlugin,
}
if err := descriptor.startFns.OnStart(descriptor.pluginID, client, logger); err != nil {
return nil, err
}
}
return &c, nil
}
func (c *clientV2) CollectMetrics(ctx context.Context) (*backend.CollectMetricsResult, error) {
if c.DiagnosticsClient == nil {
return &backend.CollectMetricsResult{}, nil
}
protoResp, err := c.DiagnosticsClient.CollectMetrics(ctx, &pluginv2.CollectMetricsRequest{})
if err != nil {
if status.Code(err) == codes.Unimplemented {
return &backend.CollectMetricsResult{}, nil
}
return nil, err
}
return backend.FromProto().CollectMetricsResponse(protoResp), nil
}
func (c *clientV2) CheckHealth(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) {
if c.DiagnosticsClient == nil {
return nil, backendplugin.ErrMethodNotImplemented
}
protoContext := backend.ToProto().PluginContext(req.PluginContext)
protoResp, err := c.DiagnosticsClient.CheckHealth(ctx, &pluginv2.CheckHealthRequest{PluginContext: protoContext})
if err != nil {
if status.Code(err) == codes.Unimplemented {
return &backend.CheckHealthResult{
Status: backend.HealthStatusUnknown,
Message: "Health check not implemented",
}, nil
}
return nil, err
}
return backend.FromProto().CheckHealthResponse(protoResp), nil
}
func (c *clientV2) CallResource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error {
if c.ResourceClient == nil {
return backendplugin.ErrMethodNotImplemented
}
protoReq := backend.ToProto().CallResourceRequest(req)
protoStream, err := c.ResourceClient.CallResource(ctx, protoReq)
if err != nil {
if status.Code(err) == codes.Unimplemented {
return backendplugin.ErrMethodNotImplemented
}
return errutil.Wrap("Failed to call resource", err)
}
for {
protoResp, err := protoStream.Recv()
if err != nil {
if status.Code(err) == codes.Unimplemented {
return backendplugin.ErrMethodNotImplemented
}
if err == io.EOF {
return nil
}
return errutil.Wrap("Failed to receive call resource response", err)
}
if err := sender.Send(backend.FromProto().CallResourceResponse(protoResp)); err != nil {
return err
}
}
}
type dataClientQueryDataFunc func(ctx context.Context, req *pluginv2.QueryDataRequest, opts ...grpc.CallOption) (*pluginv2.QueryDataResponse, error)
func (fn dataClientQueryDataFunc) QueryData(ctx context.Context, req *pluginv2.QueryDataRequest, opts ...grpc.CallOption) (*pluginv2.QueryDataResponse, error) {
return fn(ctx, req, opts...)
}
func instrumentDataClient(plugin grpcplugin.DataClient) grpcplugin.DataClient {
if plugin == nil {
return nil
}
return dataClientQueryDataFunc(func(ctx context.Context, req *pluginv2.QueryDataRequest, opts ...grpc.CallOption) (*pluginv2.QueryDataResponse, error) {
var resp *pluginv2.QueryDataResponse
err := backendplugin.InstrumentQueryDataRequest(req.PluginContext.PluginId, func() (innerErr error) {
resp, innerErr = plugin.QueryData(ctx, req)
return
})
return resp, err
})
}
type transformPluginTransformDataFunc func(ctx context.Context, req *pluginv2.QueryDataRequest, callback grpcplugin.TransformDataCallBack) (*pluginv2.QueryDataResponse, error)
func (fn transformPluginTransformDataFunc) TransformData(ctx context.Context, req *pluginv2.QueryDataRequest, callback grpcplugin.TransformDataCallBack) (*pluginv2.QueryDataResponse, error) {
return fn(ctx, req, callback)
}
func instrumentTransformPlugin(plugin grpcplugin.TransformClient) grpcplugin.TransformClient {
if plugin == nil {
return nil
}
return transformPluginTransformDataFunc(func(ctx context.Context, req *pluginv2.QueryDataRequest, callback grpcplugin.TransformDataCallBack) (*pluginv2.QueryDataResponse, error) {
var resp *pluginv2.QueryDataResponse
err := backendplugin.InstrumentTransformDataRequest(req.PluginContext.PluginId, func() (innerErr error) {
resp, innerErr = plugin.TransformData(ctx, req, callback)
return
})
return resp, err
})
}
@@ -0,0 +1,136 @@
package grpcplugin
import (
"context"
"errors"
"sync"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/plugins/backendplugin"
"github.com/hashicorp/go-plugin"
)
type pluginClient interface {
backend.CollectMetricsHandler
backend.CheckHealthHandler
backend.CallResourceHandler
}
type grpcPlugin struct {
descriptor PluginDescriptor
clientFactory func() *plugin.Client
client *plugin.Client
pluginClient pluginClient
logger log.Logger
mutex sync.RWMutex
}
// New allocates and returns a new gRPC (external) backendplugin.Plugin.
func New(descriptor PluginDescriptor) backendplugin.PluginFactoryFunc {
return backendplugin.PluginFactoryFunc(func(pluginID string, logger log.Logger, env []string) (backendplugin.Plugin, error) {
return &grpcPlugin{
descriptor: descriptor,
logger: logger,
clientFactory: func() *plugin.Client {
return plugin.NewClient(newClientConfig(descriptor.executablePath, env, logger, descriptor.versionedPlugins))
},
}, nil
})
}
func (p *grpcPlugin) PluginID() string {
return p.descriptor.pluginID
}
func (p *grpcPlugin) Logger() log.Logger {
return p.logger
}
func (p *grpcPlugin) Start(ctx context.Context) error {
p.mutex.Lock()
defer p.mutex.Unlock()
p.client = p.clientFactory()
rpcClient, err := p.client.Client()
if err != nil {
return err
}
if p.client.NegotiatedVersion() > 1 {
p.pluginClient, err = newClientV2(p.descriptor, p.logger, rpcClient)
if err != nil {
return err
}
} else {
p.pluginClient, err = newClientV1(p.descriptor, p.logger, rpcClient)
if err != nil {
return err
}
}
if p.pluginClient == nil {
return errors.New("no compatible plugin implementation found")
}
return nil
}
func (p *grpcPlugin) Stop(ctx context.Context) error {
p.mutex.Lock()
defer p.mutex.Unlock()
if p.client != nil {
p.client.Kill()
}
return nil
}
func (p *grpcPlugin) IsManaged() bool {
return p.descriptor.managed
}
func (p *grpcPlugin) Exited() bool {
p.mutex.RLock()
defer p.mutex.RUnlock()
if p.client != nil {
return p.client.Exited()
}
return true
}
func (p *grpcPlugin) CollectMetrics(ctx context.Context) (*backend.CollectMetricsResult, error) {
p.mutex.RLock()
if p.client == nil || p.client.Exited() || p.pluginClient == nil {
p.mutex.RUnlock()
return nil, backendplugin.ErrPluginUnavailable
}
pluginClient := p.pluginClient
p.mutex.RUnlock()
return pluginClient.CollectMetrics(ctx)
}
func (p *grpcPlugin) CheckHealth(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) {
p.mutex.RLock()
if p.client == nil || p.client.Exited() || p.pluginClient == nil {
p.mutex.RUnlock()
return nil, backendplugin.ErrPluginUnavailable
}
pluginClient := p.pluginClient
p.mutex.RUnlock()
return pluginClient.CheckHealth(ctx, req)
}
func (p *grpcPlugin) CallResource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error {
p.mutex.RLock()
if p.client == nil || p.client.Exited() || p.pluginClient == nil {
p.mutex.RUnlock()
return backendplugin.ErrPluginUnavailable
}
pluginClient := p.pluginClient
p.mutex.RUnlock()
return pluginClient.CallResource(ctx, req, sender)
}
@@ -0,0 +1,88 @@
package grpcplugin
import (
"io"
"io/ioutil"
"log"
glog "github.com/grafana/grafana/pkg/infra/log"
hclog "github.com/hashicorp/go-hclog"
)
type logWrapper struct {
Logger glog.Logger
}
func formatArgs(args ...interface{}) []interface{} {
if len(args) == 0 || len(args)%2 != 0 {
return args
}
res := []interface{}{}
for n := 0; n < len(args); n += 2 {
key := args[n]
if stringKey, ok := key.(string); ok && stringKey == "timestamp" {
continue
}
res = append(res, key)
res = append(res, args[n+1])
}
return res
}
func (lw logWrapper) Trace(msg string, args ...interface{}) {
lw.Logger.Debug(msg, formatArgs(args...)...)
}
func (lw logWrapper) Debug(msg string, args ...interface{}) {
lw.Logger.Debug(msg, formatArgs(args...)...)
}
func (lw logWrapper) Info(msg string, args ...interface{}) {
lw.Logger.Info(msg, formatArgs(args...)...)
}
func (lw logWrapper) Warn(msg string, args ...interface{}) {
lw.Logger.Warn(msg, formatArgs(args...)...)
}
func (lw logWrapper) Error(msg string, args ...interface{}) {
lw.Logger.Error(msg, formatArgs(args...)...)
}
func (lw logWrapper) IsTrace() bool { return true }
func (lw logWrapper) IsDebug() bool { return true }
func (lw logWrapper) IsInfo() bool { return true }
func (lw logWrapper) IsWarn() bool { return true }
func (lw logWrapper) IsError() bool { return true }
func (lw logWrapper) With(args ...interface{}) hclog.Logger {
return logWrapper{Logger: lw.Logger.New(args...)}
}
func (lw logWrapper) Named(name string) hclog.Logger {
if name == "stdio" {
// discard logs from stdio hashicorp/go-plugin gRPC service since
// it's not enabled/in use per default.
// discard debug log of "waiting for stdio data".
// discard warn log of "received EOF, stopping recv loop".
return hclog.NewNullLogger()
}
return logWrapper{
Logger: lw.Logger.New(),
}
}
func (lw logWrapper) ResetNamed(name string) hclog.Logger {
return logWrapper{Logger: lw.Logger.New()}
}
func (lw logWrapper) StandardLogger(ops *hclog.StandardLoggerOptions) *log.Logger {
return nil
}
func (lw logWrapper) SetLevel(level hclog.Level) {}
// Return a value that conforms to io.Writer, which can be passed into log.SetOutput()
func (lw logWrapper) StandardWriter(opts *hclog.StandardLoggerOptions) io.Writer {
return ioutil.Discard
}
@@ -0,0 +1,29 @@
package grpcplugin
import (
"fmt"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestLogWrapper(t *testing.T) {
tcs := []struct {
args []interface{}
expectedResult []interface{}
}{
{args: []interface{}{}, expectedResult: []interface{}{}},
{args: []interface{}{"1", "2", "3"}, expectedResult: []interface{}{"1", "2", "3"}},
{args: []interface{}{"1", "2"}, expectedResult: []interface{}{"1", "2"}},
{args: []interface{}{"1", "2", "timestamp", time.Now()}, expectedResult: []interface{}{"1", "2"}},
{args: []interface{}{"1", "2", "timestamp", time.Now(), "3", "4"}, expectedResult: []interface{}{"1", "2", "3", "4"}},
}
for i, tc := range tcs {
t.Run(fmt.Sprintf("formatArgs testcase %d", i), func(t *testing.T) {
res := formatArgs(tc.args...)
assert.Exactly(t, tc.expectedResult, res)
})
}
}