public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Plugins: Transform plugin support (#20036)

Browse Source 
currently temporary separate http api
This commit is contained in:
Kyle Brandt
2019-10-29 12:22:31 -04:00
committed by GitHub
parent 69691fbd6e
commit 009d58c4a2
32 changed files with 2022 additions and 1187 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/plugins/backendplugin/client.go
+81
View File
@@ -0,0 +1,81 @@
package backendplugin
import (
"os/exec"
"github.com/grafana/grafana/pkg/infra/log"
datasourceV1 "github.com/grafana/grafana-plugin-model/go/datasource"
rendererV1 "github.com/grafana/grafana-plugin-model/go/renderer"
sdk "github.com/grafana/grafana-plugin-sdk-go/common"
datasourceV2 "github.com/grafana/grafana-plugin-sdk-go/datasource"
transformV2 "github.com/grafana/grafana-plugin-sdk-go/transform"
"github.com/hashicorp/go-plugin"
)
const (
// DefaultProtocolVersion is the protocol version assumed for legacy clients that don't specify
// a particular version or version 1 during their handshake. This is currently the version used
// since Grafana launched support for backend plugins.
DefaultProtocolVersion = 1
)
// Handshake is the HandshakeConfig used to configure clients and servers.
var handshake = plugin.HandshakeConfig{
// The ProtocolVersion is the version that must match between Grafana core
// and Grafana plugins. This should be bumped whenever a (breaking) change
// happens in one or the other that makes it so that they can't safely communicate.
ProtocolVersion: DefaultProtocolVersion,
// The magic cookie values should NEVER be changed.
MagicCookieKey: sdk.MagicCookieKey,
MagicCookieValue: sdk.MagicCookieValue,
}
// NewClientConfig returns a configuration object that can be used to instantiate
// a client for the plugin described by the given metadata.
func NewClientConfig(executablePath string, logger log.Logger, versionedPlugins map[int]plugin.PluginSet) *plugin.ClientConfig {
return &plugin.ClientConfig{
Cmd: exec.Command(executablePath),
HandshakeConfig: handshake,
VersionedPlugins: versionedPlugins,
Logger: logWrapper{Logger: logger},
AllowedProtocols: []plugin.Protocol{plugin.ProtocolGRPC},
}
}
// NewDatasourceClient returns a datasource plugin client.
func NewDatasourceClient(pluginID, executablePath string, logger log.Logger) *plugin.Client {
versionedPlugins := map[int]plugin.PluginSet{
1: {
pluginID: &datasourceV1.DatasourcePluginImpl{},
},
2: {
pluginID: &datasourceV2.DatasourcePluginImpl{},
},
}
return plugin.NewClient(NewClientConfig(executablePath, logger, versionedPlugins))
}
// NewRendererClient returns a renderer plugin client.
func NewRendererClient(pluginID, executablePath string, logger log.Logger) *plugin.Client {
versionedPlugins := map[int]plugin.PluginSet{
1: {
pluginID: &rendererV1.RendererPluginImpl{},
},
}
return plugin.NewClient(NewClientConfig(executablePath, logger, versionedPlugins))
}
// NewTransformClient returns a transform plugin client.
func NewTransformClient(pluginID, executablePath string, logger log.Logger) *plugin.Client {
versionedPlugins := map[int]plugin.PluginSet{
2: {
pluginID: &transformV2.TransformPluginImpl{},
},
}
return plugin.NewClient(NewClientConfig(executablePath, logger, versionedPlugins))
}
pkg/plugins/backendplugin/log_wrapper.go
+58
View File
@@ -0,0 +1,58 @@
package backendplugin
import (
"io"
"io/ioutil"
"log"
glog "github.com/grafana/grafana/pkg/infra/log"
hclog "github.com/hashicorp/go-hclog"
)
type logWrapper struct {
Logger glog.Logger
}
func (lw logWrapper) Trace(msg string, args ...interface{}) {
lw.Logger.Debug(msg, args...)
}
func (lw logWrapper) Debug(msg string, args ...interface{}) {
lw.Logger.Debug(msg, args...)
}
func (lw logWrapper) Info(msg string, args ...interface{}) {
lw.Logger.Info(msg, args...)
}
func (lw logWrapper) Warn(msg string, args ...interface{}) {
lw.Logger.Warn(msg, args...)
}
func (lw logWrapper) Error(msg string, args ...interface{}) {
lw.Logger.Error(msg, args...)
}
func (lw logWrapper) IsTrace() bool { return true }
func (lw logWrapper) IsDebug() bool { return true }
func (lw logWrapper) IsInfo() bool { return true }
func (lw logWrapper) IsWarn() bool { return true }
func (lw logWrapper) IsError() bool { return true }
func (lw logWrapper) With(args ...interface{}) hclog.Logger {
return logWrapper{Logger: lw.Logger.New(args...)}
}
func (lw logWrapper) Named(name string) hclog.Logger {
return logWrapper{Logger: lw.Logger.New()}
}
func (lw logWrapper) ResetNamed(name string) hclog.Logger {
return logWrapper{Logger: lw.Logger.New()}
}
func (lw logWrapper) StandardLogger(ops *hclog.StandardLoggerOptions) *log.Logger {
return nil
}
func (lw logWrapper) SetLevel(level hclog.Level) {}
// Return a value that conforms to io.Writer, which can be passed into log.SetOutput()
func (lw logWrapper) StandardWriter(opts *hclog.StandardLoggerOptions) io.Writer {
return ioutil.Discard
}
pkg/plugins/datasource/wrapper/datasource_plugin_wrapper_v2.go
+8 -89
View File
@@ -3,96 +3,15 @@ package wrapper
import (
"context"
"errors"
"fmt"
sdk "github.com/grafana/grafana-plugin-sdk-go"
"github.com/grafana/grafana-plugin-sdk-go/dataframe"
"github.com/grafana/grafana-plugin-sdk-go/genproto/datasource"
"github.com/grafana/grafana/pkg/bus"
sdk "github.com/grafana/grafana-plugin-sdk-go/datasource"
"github.com/grafana/grafana-plugin-sdk-go/genproto/pluginv2"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/tsdb"
)
type grafanaAPI struct {
logger log.Logger
}
func (s *grafanaAPI) QueryDatasource(ctx context.Context, req *datasource.QueryDatasourceRequest) (*datasource.QueryDatasourceResponse, error) {
if len(req.Queries) == 0 {
return nil, fmt.Errorf("zero queries found in datasource request")
}
getDsInfo := &models.GetDataSourceByIdQuery{
Id: req.DatasourceId,
OrgId: req.OrgId,
}
if err := bus.Dispatch(getDsInfo); err != nil {
return nil, fmt.Errorf("Could not find datasource %v", err)
}
// Convert plugin-model (datasource) queries to tsdb queries
queries := make([]*tsdb.Query, len(req.Queries))
for i, query := range req.Queries {
sj, err := simplejson.NewJson([]byte(query.ModelJson))
if err != nil {
return nil, err
}
queries[i] = &tsdb.Query{
RefId: query.RefId,
IntervalMs: query.IntervalMs,
MaxDataPoints: query.MaxDataPoints,
DataSource: getDsInfo.Result,
Model: sj,
}
}
timeRange := tsdb.NewTimeRange(req.TimeRange.FromRaw, req.TimeRange.ToRaw)
tQ := &tsdb.TsdbQuery{
TimeRange: timeRange,
Queries: queries,
}
// Execute the converted queries
tsdbRes, err := tsdb.HandleRequest(ctx, getDsInfo.Result, tQ)
if err != nil {
return nil, err
}
// Convert tsdb results (map) to plugin-model/datasource (slice) results
// Only error and Series responses mapped.
results := make([]*datasource.QueryResult, len(tsdbRes.Results))
resIdx := 0
for refID, res := range tsdbRes.Results {
qr := &datasource.QueryResult{
RefId: refID,
}
if res.Error != nil {
qr.Error = res.ErrorString
results[resIdx] = qr
resIdx++
continue
}
encodedFrames := make([][]byte, len(res.Series))
for sIdx, series := range res.Series {
frame, err := tsdb.SeriesToFrame(series)
if err != nil {
return nil, err
}
encodedFrames[sIdx], err = dataframe.MarshalArrow(frame)
if err != nil {
return nil, err
}
}
qr.Dataframes = encodedFrames
results[resIdx] = qr
resIdx++
}
return &datasource.QueryDatasourceResponse{Results: results}, nil
}
func NewDatasourcePluginWrapperV2(log log.Logger, plugin sdk.DatasourcePlugin) *DatasourcePluginWrapperV2 {
return &DatasourcePluginWrapperV2{DatasourcePlugin: plugin, logger: log}
}
@@ -108,8 +27,8 @@ func (tw *DatasourcePluginWrapperV2) Query(ctx context.Context, ds *models.DataS
return nil, err
}
pbQuery := &datasource.DatasourceRequest{
Datasource: &datasource.DatasourceInfo{
pbQuery := &pluginv2.DatasourceRequest{
Datasource: &pluginv2.DatasourceInfo{
Name: ds.Name,
Type: ds.Type,
Url: ds.Url,
@@ -118,13 +37,13 @@ func (tw *DatasourcePluginWrapperV2) Query(ctx context.Context, ds *models.DataS
JsonData: string(jsonData),
DecryptedSecureJsonData: ds.SecureJsonData.Decrypt(),
},
TimeRange: &datasource.TimeRange{
TimeRange: &pluginv2.TimeRange{
FromRaw: query.TimeRange.From,
ToRaw: query.TimeRange.To,
ToEpochMs: query.TimeRange.GetToAsMsEpoch(),
FromEpochMs: query.TimeRange.GetFromAsMsEpoch(),
},
Queries: []*datasource.Query{},
Queries: []*pluginv2.DatasourceQuery{},
}
for _, q := range query.Queries {
@@ -132,7 +51,7 @@ func (tw *DatasourcePluginWrapperV2) Query(ctx context.Context, ds *models.DataS
if err != nil {
return nil, err
}
pbQuery.Queries = append(pbQuery.Queries, &datasource.Query{
pbQuery.Queries = append(pbQuery.Queries, &pluginv2.DatasourceQuery{
ModelJson: string(modelJSON),
IntervalMs: q.IntervalMs,
RefId: q.RefId,
@@ -140,7 +59,7 @@ func (tw *DatasourcePluginWrapperV2) Query(ctx context.Context, ds *models.DataS
})
}
pbres, err := tw.DatasourcePlugin.Query(ctx, pbQuery, &grafanaAPI{logger: tw.logger})
pbres, err := tw.DatasourcePlugin.Query(ctx, pbQuery)
if err != nil {
return nil, err
pkg/plugins/datasource_plugin.go
+6 -30
View File
@@ -5,15 +5,16 @@ import (
"encoding/json"
"errors"
"fmt"
"os/exec"
"path"
"time"
"github.com/grafana/grafana/pkg/plugins/backendplugin"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util/errutil"
datasourceV1 "github.com/grafana/grafana-plugin-model/go/datasource"
sdk "github.com/grafana/grafana-plugin-sdk-go"
sdk "github.com/grafana/grafana-plugin-sdk-go/datasource"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins/datasource/wrapper"
@@ -63,12 +64,6 @@ func (p *DataSourcePlugin) Load(decoder *json.Decoder, pluginDir string) error {
return nil
}
var handshakeConfig = plugin.HandshakeConfig{
ProtocolVersion: 1,
MagicCookieKey: "grafana_plugin_type",
MagicCookieValue: "datasource",
}
func (p *DataSourcePlugin) startBackendPlugin(ctx context.Context, log log.Logger) error {
p.log = log.New("plugin-id", p.Id)
@@ -84,6 +79,7 @@ func (p *DataSourcePlugin) startBackendPlugin(ctx context.Context, log log.Logge
return nil
}
func (p *DataSourcePlugin) isVersionOne() bool {
return !p.SDK
}
@@ -92,27 +88,7 @@ func (p *DataSourcePlugin) spawnSubProcess() error {
cmd := ComposePluginStartCommmand(p.Executable)
fullpath := path.Join(p.PluginDir, cmd)
var newClient *plugin.Client
if p.isVersionOne() {
newClient = plugin.NewClient(&plugin.ClientConfig{
HandshakeConfig: handshakeConfig,
Plugins: map[string]plugin.Plugin{p.Id: &datasourceV1.DatasourcePluginImpl{}},
Cmd: exec.Command(fullpath),
AllowedProtocols: []plugin.Protocol{plugin.ProtocolGRPC},
Logger: LogWrapper{Logger: p.log},
})
} else {
newClient = plugin.NewClient(&plugin.ClientConfig{
HandshakeConfig: handshakeConfig,
Plugins: map[string]plugin.Plugin{p.Id: &sdk.DatasourcePluginImpl{}},
Cmd: exec.Command(fullpath),
AllowedProtocols: []plugin.Protocol{plugin.ProtocolGRPC},
Logger: LogWrapper{Logger: p.log},
})
}
p.client = newClient
p.client = backendplugin.NewDatasourceClient(p.Id, fullpath, p.log)
rpcClient, err := p.client.Client()
if err != nil {
@@ -124,7 +100,7 @@ func (p *DataSourcePlugin) spawnSubProcess() error {
return err
}
if p.isVersionOne() {
if p.client.NegotiatedVersion() == 1 {
plugin := raw.(datasourceV1.DatasourcePlugin)
tsdb.RegisterTsdbQueryEndpoint(p.Id, func(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
pkg/plugins/plugins.go
+8 -1
View File
@@ -29,6 +29,7 @@ var (
Plugins map[string]*PluginBase
PluginTypes map[string]interface{}
Renderer *RendererPlugin
Transform *TransformPlugin
GrafanaLatestVersion string
GrafanaHasUpdate bool
@@ -62,6 +63,7 @@ func (pm *PluginManager) Init() error {
"datasource": DataSourcePlugin{},
"app": AppPlugin{},
"renderer": RendererPlugin{},
"transform": TransformPlugin{},
}
pm.log.Info("Starting plugin search")
@@ -118,6 +120,11 @@ func (pm *PluginManager) startBackendPlugins(ctx context.Context) {
pm.log.Error("Failed to init plugin.", "error", err, "plugin", ds.Id)
}
}
if Transform != nil {
if err := Transform.startBackendPlugin(ctx, plog); err != nil {
pm.log.Error("Failed to init plugin.", "error", err, "plugin", Transform.Id)
}
}
}
func (pm *PluginManager) Run(ctx context.Context) error {
@@ -263,7 +270,7 @@ func (scanner *PluginScanner) loadPluginJson(pluginJsonFilePath string) error {
}
func (scanner *PluginScanner) IsBackendOnlyPlugin(pluginType string) bool {
return pluginType == "renderer"
return pluginType == "renderer" || pluginType == "transform"
}
func GetPluginMarkdown(pluginId string, name string) ([]byte, error) {
pkg/plugins/transform_plugin.go
+288
View File
@@ -0,0 +1,288 @@
package plugins
import (
"context"
"encoding/json"
"errors"
"fmt"
"path"
"time"
"github.com/grafana/grafana-plugin-sdk-go/dataframe"
"github.com/grafana/grafana-plugin-sdk-go/genproto/pluginv2"
"github.com/grafana/grafana-plugin-sdk-go/transform"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins/backendplugin"
"github.com/grafana/grafana/pkg/tsdb"
plugin "github.com/hashicorp/go-plugin"
"golang.org/x/xerrors"
)
type TransformPlugin struct {
PluginBase
// TODO we probably want a Backend Plugin Base? Or some way to dedup proc management code
Executable string `json:"executable,omitempty"`
//transform.TransformPlugin
*TransformWrapper
client *plugin.Client
log log.Logger
}
func (tp *TransformPlugin) Load(decoder *json.Decoder, pluginDir string) error {
if err := decoder.Decode(&tp); err != nil {
return err
}
if err := tp.registerPlugin(pluginDir); err != nil {
return err
}
Transform = tp
return nil
}
func (p *TransformPlugin) startBackendPlugin(ctx context.Context, log log.Logger) error {
p.log = log.New("plugin-id", p.Id)
if err := p.spawnSubProcess(); err != nil {
return err
}
go func() {
if err := p.restartKilledProcess(ctx); err != nil {
p.log.Error("Attempting to restart killed process failed", "err", err)
}
}()
return nil
}
func (p *TransformPlugin) spawnSubProcess() error {
cmd := ComposePluginStartCommmand(p.Executable)
fullpath := path.Join(p.PluginDir, cmd)
p.client = backendplugin.NewTransformClient(p.Id, fullpath, p.log)
rpcClient, err := p.client.Client()
if err != nil {
return err
}
raw, err := rpcClient.Dispense(p.Id)
if err != nil {
return err
}
plugin, ok := raw.(transform.TransformPlugin)
if !ok {
return fmt.Errorf("unexpected type %T, expected *transform.GRPCClient", raw)
}
p.TransformWrapper = NewTransformWrapper(p.log, plugin)
return nil
}
func (p *TransformPlugin) restartKilledProcess(ctx context.Context) error {
ticker := time.NewTicker(time.Second * 1)
for {
select {
case <-ctx.Done():
if err := ctx.Err(); err != nil && !xerrors.Is(err, context.Canceled) {
return err
}
return nil
case <-ticker.C:
if !p.client.Exited() {
continue
}
if err := p.spawnSubProcess(); err != nil {
p.log.Error("Failed to restart plugin", "err", err)
continue
}
p.log.Debug("Plugin process restarted")
}
}
}
func (p *TransformPlugin) Kill() {
if p.client != nil {
p.log.Debug("Killing subprocess ", "name", p.Name)
p.client.Kill()
}
}
// ...
// Wrapper Code
// ...
func NewTransformWrapper(log log.Logger, plugin transform.TransformPlugin) *TransformWrapper {
return &TransformWrapper{plugin, log, &grafanaAPI{log}}
}
type TransformWrapper struct {
transform.TransformPlugin
logger log.Logger
api *grafanaAPI
}
func (tw *TransformWrapper) Transform(ctx context.Context, ds *models.DataSource, query *tsdb.TsdbQuery) (*tsdb.Response, error) {
jsonData, err := ds.JsonData.MarshalJSON()
if err != nil {
return nil, err
}
pbQuery := &pluginv2.TransformRequest{
Datasource: &pluginv2.DatasourceInfo{
Name: ds.Name,
Type: ds.Type,
Url: ds.Url,
Id: ds.Id,
OrgId: ds.OrgId,
JsonData: string(jsonData),
DecryptedSecureJsonData: ds.SecureJsonData.Decrypt(),
},
TimeRange: &pluginv2.TimeRange{
FromRaw: query.TimeRange.From,
ToRaw: query.TimeRange.To,
ToEpochMs: query.TimeRange.GetToAsMsEpoch(),
FromEpochMs: query.TimeRange.GetFromAsMsEpoch(),
},
Queries: []*pluginv2.TransformQuery{},
}
for _, q := range query.Queries {
modelJSON, err := q.Model.MarshalJSON()
if err != nil {
return nil, err
}
pbQuery.Queries = append(pbQuery.Queries, &pluginv2.TransformQuery{
ModelJson: string(modelJSON),
IntervalMs: q.IntervalMs,
RefId: q.RefId,
MaxDataPoints: q.MaxDataPoints,
})
}
pbres, err := tw.TransformPlugin.Transform(ctx, pbQuery, tw.api)
if err != nil {
return nil, err
}
res := &tsdb.Response{
Results: map[string]*tsdb.QueryResult{},
}
for _, r := range pbres.Results {
qr := &tsdb.QueryResult{
RefId: r.RefId,
}
if r.Error != "" {
qr.Error = errors.New(r.Error)
qr.ErrorString = r.Error
}
if r.MetaJson != "" {
metaJSON, err := simplejson.NewJson([]byte(r.MetaJson))
if err != nil {
tw.logger.Error("Error parsing JSON Meta field: " + err.Error())
}
qr.Meta = metaJSON
}
qr.Dataframes = r.Dataframes
res.Results[r.RefId] = qr
}
return res, nil
}
type grafanaAPI struct {
logger log.Logger
}
func (s *grafanaAPI) QueryDatasource(ctx context.Context, req *pluginv2.QueryDatasourceRequest) (*pluginv2.QueryDatasourceResponse, error) {
if len(req.Queries) == 0 {
return nil, fmt.Errorf("zero queries found in datasource request")
}
getDsInfo := &models.GetDataSourceByIdQuery{
Id: req.DatasourceId,
OrgId: req.OrgId,
}
if err := bus.Dispatch(getDsInfo); err != nil {
return nil, fmt.Errorf("Could not find datasource %v", err)
}
// Convert plugin-model (datasource) queries to tsdb queries
queries := make([]*tsdb.Query, len(req.Queries))
for i, query := range req.Queries {
sj, err := simplejson.NewJson([]byte(query.ModelJson))
if err != nil {
return nil, err
}
queries[i] = &tsdb.Query{
RefId: query.RefId,
IntervalMs: query.IntervalMs,
MaxDataPoints: query.MaxDataPoints,
DataSource: getDsInfo.Result,
Model: sj,
}
}
timeRange := tsdb.NewTimeRange(req.TimeRange.FromRaw, req.TimeRange.ToRaw)
tQ := &tsdb.TsdbQuery{
TimeRange: timeRange,
Queries: queries,
}
// Execute the converted queries
tsdbRes, err := tsdb.HandleRequest(ctx, getDsInfo.Result, tQ)
if err != nil {
return nil, err
}
// Convert tsdb results (map) to plugin-model/datasource (slice) results
// Only error and Series responses mapped.
results := make([]*pluginv2.DatasourceQueryResult, len(tsdbRes.Results))
resIdx := 0
for refID, res := range tsdbRes.Results {
qr := &pluginv2.DatasourceQueryResult{
RefId: refID,
}
if res.Error != nil {
qr.Error = res.ErrorString
results[resIdx] = qr
resIdx++
continue
}
encodedFrames := make([][]byte, len(res.Series))
for sIdx, series := range res.Series {
frame, err := tsdb.SeriesToFrame(series)
if err != nil {
return nil, err
}
encodedFrames[sIdx], err = dataframe.MarshalArrow(frame)
if err != nil {
return nil, err
}
}
qr.Dataframes = encodedFrames
results[resIdx] = qr
resIdx++
}
return &pluginv2.QueryDatasourceResponse{Results: results}, nil
}