public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Live: expose HTTP push endpoint that will read influx line protocol and publish to websocket (#32311)

Browse Source 
Co-authored-by: Ryan McKinley <ryantxu@gmail.com>
This commit is contained in:
Alexander Emelin
2021-04-05 19:04:46 +03:00
committed by GitHub
co-authored by Ryan McKinley
parent 7896c6a7b1
commit 54ad791c7e
25 changed files with 576 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/live/channel.go
+2 -1
View File
@@ -7,13 +7,14 @@ import (
// ChannelAddress is the channel ID split by parts.
type ChannelAddress struct {
// Scope is one of available channel scopes:
// like ScopeGrafana, ScopePlugin, ScopeDatasource.
// like ScopeGrafana, ScopePlugin, ScopeDatasource, ScopeStream.
Scope string `json:"scope,omitempty"`
// Namespace meaning depends on the scope.
// * when ScopeGrafana, namespace is a "feature"
// * when ScopePlugin, namespace is the plugin name
// * when ScopeDatasource, namespace is the datasource uid
// * when ScopeStream, namespace is the stream ID.
Namespace string `json:"namespace,omitempty"`
// Within each namespace, the handler can process the path as needed.
pkg/services/live/features/broadcast.go
+6
View File
@@ -4,11 +4,17 @@ import (
"context"
"time"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/models"
)
var (
logger = log.New("live.features") // scoped to all features?
)
// BroadcastRunner will simply broadcast all events to `grafana/broadcast/*` channels
// This assumes that data is a JSON object
type BroadcastRunner struct{}
pkg/services/live/features/measurements.go
-36
View File
@@ -1,36 +0,0 @@
package features
import (
"context"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
)
var (
logger = log.New("live.features") // scoped to all features?
)
// MeasurementsRunner will simply broadcast all events to `grafana/broadcast/*` channels.
// This makes no assumptions about the shape of the data and will broadcast it to anyone listening
type MeasurementsRunner struct {
}
// GetHandlerForPath gets the handler for a path.
// It's called on init.
func (m *MeasurementsRunner) GetHandlerForPath(path string) (models.ChannelHandler, error) {
return m, nil // for now all channels share config
}
// OnSubscribe will let anyone connect to the path
func (m *MeasurementsRunner) OnSubscribe(ctx context.Context, _ *models.SignedInUser, e models.SubscribeEvent) (models.SubscribeReply, backend.SubscribeStreamStatus, error) {
return models.SubscribeReply{}, backend.SubscribeStreamStatusOK, nil
}
// OnPublish is called when a client wants to broadcast on the websocket
// Currently this sends measurements over websocket -- should be replaced with the HTTP interface
func (m *MeasurementsRunner) OnPublish(ctx context.Context, _ *models.SignedInUser, e models.PublishEvent) (models.PublishReply, backend.PublishStreamStatus, error) {
return models.PublishReply{}, backend.PublishStreamStatusOK, nil
}
pkg/services/live/live.go
+38 -21
View File
@@ -9,6 +9,7 @@ import (
"github.com/centrifugal/centrifuge"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/api/dtos"
"github.com/grafana/grafana/pkg/api/response"
@@ -22,6 +23,7 @@ import (
"github.com/grafana/grafana/pkg/services/live/features"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb/cloudwatch"
"github.com/grafana/grafana/pkg/util"
)
var (
@@ -68,6 +70,8 @@ type GrafanaLive struct {
// The core internal features
GrafanaScope CoreGrafanaScope
ManagedStreamRunner *ManagedStreamRunner
contextGetter *pluginContextGetter
streamManager *features.StreamManager
}
@@ -131,7 +135,8 @@ func (g *GrafanaLive) Init() error {
g.GrafanaScope.Dashboards = dash
g.GrafanaScope.Features["dashboard"] = dash
g.GrafanaScope.Features["broadcast"] = &features.BroadcastRunner{}
g.GrafanaScope.Features["measurements"] = &features.MeasurementsRunner{}
g.ManagedStreamRunner = NewManagedStreamRunner(g.Publish)
// Set ConnectHandler called when client successfully connected to Node. Your code
// inside handler must be synchronized since it will be called concurrently from
@@ -349,6 +354,8 @@ func (g *GrafanaLive) GetChannelHandlerFactory(user *models.SignedInUser, scope
return g.handlePluginScope(user, namespace)
case ScopeDatasource:
return g.handleDatasourceScope(user, namespace)
case ScopeStream:
return g.handleStreamScope(user, namespace)
default:
return nil, fmt.Errorf("invalid scope: %q", scope)
}
@@ -382,6 +389,10 @@ func (g *GrafanaLive) handlePluginScope(_ *models.SignedInUser, namespace string
), nil
}
func (g *GrafanaLive) handleStreamScope(_ *models.SignedInUser, namespace string) (models.ChannelHandlerFactory, error) {
return g.ManagedStreamRunner.GetOrCreateStream(namespace)
}
func (g *GrafanaLive) handleDatasourceScope(user *models.SignedInUser, namespace string) (models.ChannelHandlerFactory, error) {
ds, err := g.DatasourceCache.GetDatasourceByUID(namespace, user, false)
if err != nil {
@@ -445,26 +456,32 @@ func (g *GrafanaLive) HandleHTTPPublish(ctx *models.ReqContext, cmd dtos.LivePub
return response.JSON(http.StatusOK, dtos.LivePublishResponse{})
}
// Write to the standard log15 logger
func handleLog(msg centrifuge.LogEntry) {
arr := make([]interface{}, 0)
for k, v := range msg.Fields {
if v == nil {
v = "<nil>"
} else if v == "" {
v = "<empty>"
}
arr = append(arr, k, v)
// HandleListHTTP returns metadata so the UI can build a nice form
func (g *GrafanaLive) HandleListHTTP(_ *models.ReqContext) response.Response {
info := util.DynMap{}
channels := make([]util.DynMap, 0)
for k, v := range g.ManagedStreamRunner.Streams() {
channels = append(channels, v.ListChannels("stream/"+k+"/")...)
}
switch msg.Level {
case centrifuge.LogLevelDebug:
loggerCF.Debug(msg.Message, arr...)
case centrifuge.LogLevelError:
loggerCF.Error(msg.Message, arr...)
case centrifuge.LogLevelInfo:
loggerCF.Info(msg.Message, arr...)
default:
loggerCF.Debug(msg.Message, arr...)
}
// Hardcode sample streams
frame := data.NewFrame("testdata",
data.NewField("Time", nil, make([]time.Time, 0)),
data.NewField("Value", nil, make([]float64, 0)),
data.NewField("Min", nil, make([]float64, 0)),
data.NewField("Max", nil, make([]float64, 0)),
)
channels = append(channels, util.DynMap{
"channel": "plugin/testdata/random-2s-stream",
"data": frame,
}, util.DynMap{
"channel": "plugin/testdata/random-flakey-stream",
"data": frame,
}, util.DynMap{
"channel": "plugin/testdata/random-20Hz-stream",
"data": frame,
})
info["channels"] = channels
return response.JSONStreaming(200, info)
}
pkg/services/live/log.go
+27
View File
@@ -0,0 +1,27 @@
package live
import "github.com/centrifugal/centrifuge"
// Write to the standard log15 logger
func handleLog(msg centrifuge.LogEntry) {
arr := make([]interface{}, 0)
for k, v := range msg.Fields {
if v == nil {
v = "<nil>"
} else if v == "" {
v = "<empty>"
}
arr = append(arr, k, v)
}
switch msg.Level {
case centrifuge.LogLevelDebug:
loggerCF.Debug(msg.Message, arr...)
case centrifuge.LogLevelError:
loggerCF.Error(msg.Message, arr...)
case centrifuge.LogLevelInfo:
loggerCF.Info(msg.Message, arr...)
default:
loggerCF.Debug(msg.Message, arr...)
}
}
pkg/services/live/managed.go
+142
View File
@@ -0,0 +1,142 @@
package live
import (
"context"
"encoding/json"
"fmt"
"sync"
"time"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/util"
)
type ManagedStreamRunner struct {
mu sync.RWMutex
streams map[string]*ManagedStream
publisher models.ChannelPublisher
}
// NewPluginRunner creates new PluginRunner.
func NewManagedStreamRunner(publisher models.ChannelPublisher) *ManagedStreamRunner {
return &ManagedStreamRunner{
publisher: publisher,
streams: map[string]*ManagedStream{},
}
}
// Streams returns map of active managed streams.
func (r *ManagedStreamRunner) Streams() map[string]*ManagedStream {
r.mu.RLock()
defer r.mu.RUnlock()
streams := make(map[string]*ManagedStream, len(r.streams))
for k, v := range r.streams {
streams[k] = v
}
return streams
}
// GetOrCreateStream -- for now this will create new manager for each key.
// Eventually, the stream behavior will need to be configured explicitly
func (r *ManagedStreamRunner) GetOrCreateStream(streamID string) (*ManagedStream, error) {
r.mu.Lock()
defer r.mu.Unlock()
s, ok := r.streams[streamID]
if !ok {
s = NewManagedStream(streamID, r.publisher)
r.streams[streamID] = s
}
return s, nil
}
// ManagedStream holds the state of a managed stream
type ManagedStream struct {
mu sync.RWMutex
id string
start time.Time
last map[string]json.RawMessage
publisher models.ChannelPublisher
}
// NewCache creates new Cache.
func NewManagedStream(id string, publisher models.ChannelPublisher) *ManagedStream {
return &ManagedStream{
id: id,
start: time.Now(),
last: map[string]json.RawMessage{},
publisher: publisher,
}
}
// ListChannels returns info for the UI about this stream.
func (s *ManagedStream) ListChannels(prefix string) []util.DynMap {
s.mu.RLock()
defer s.mu.RUnlock()
info := make([]util.DynMap, 0, len(s.last))
for k, v := range s.last {
ch := util.DynMap{}
ch["channel"] = prefix + k
ch["data"] = v
info = append(info, ch)
}
return info
}
// Push sends data to the stream and optionally processes it.
func (s *ManagedStream) Push(path string, frame *data.Frame) error {
// Keep schema + data for last packet.
frameJSON, err := data.FrameToJSON(frame, true, true)
if err != nil {
logger.Error("Error marshaling Frame to Schema", "error", err)
return err
}
// Locks until we totally finish?
s.mu.Lock()
defer s.mu.Unlock()
_, exists := s.last[path]
s.last[path] = frameJSON
// When the packet already exits, only send the data.
if exists {
frameJSON, err = data.FrameToJSON(frame, false, true)
if err != nil {
logger.Error("Error marshaling Frame to JSON", "error", err)
return err
}
}
// The channel this will be posted into.
channel := fmt.Sprintf("stream/%s/%s", s.id, path)
logger.Debug("Publish data to channel", "channel", channel, "dataLength", len(frameJSON))
return s.publisher(channel, frameJSON)
}
// getLastPacket retrieves schema for a channel.
func (s *ManagedStream) getLastPacket(path string) (json.RawMessage, bool) {
s.mu.RLock()
defer s.mu.RUnlock()
schema, ok := s.last[path]
return schema, ok
}
func (s *ManagedStream) GetHandlerForPath(_ string) (models.ChannelHandler, error) {
return s, nil
}
func (s *ManagedStream) OnSubscribe(_ context.Context, _ *models.SignedInUser, e models.SubscribeEvent) (models.SubscribeReply, backend.SubscribeStreamStatus, error) {
reply := models.SubscribeReply{}
packet, ok := s.getLastPacket(e.Path)
if ok {
reply.Data = packet
}
return reply, backend.SubscribeStreamStatusOK, nil
}
func (s *ManagedStream) OnPublish(_ context.Context, _ *models.SignedInUser, _ models.PublishEvent) (models.PublishReply, backend.PublishStreamStatus, error) {
return models.PublishReply{}, backend.PublishStreamStatusPermissionDenied, nil
}
pkg/services/live/managed_test.go
+30
View File
@@ -0,0 +1,30 @@
package live
import (
"testing"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/stretchr/testify/require"
)
var noopPublisher = func(p string, b []byte) error {
return nil
}
func TestNewManagedStream(t *testing.T) {
c := NewManagedStream("a", noopPublisher)
require.NotNil(t, c)
}
func TestManagedStream_GetLastPacket(t *testing.T) {
c := NewManagedStream("a", noopPublisher)
_, ok := c.getLastPacket("test")
require.False(t, ok)
err := c.Push("test", data.NewFrame("hello"))
require.NoError(t, err)
s, ok := c.getLastPacket("test")
require.NoError(t, err)
require.True(t, ok)
require.Equal(t, `{"schema":{"name":"hello","fields":[]},"data":{"values":[]}}`, string(s))
}
pkg/services/live/plugin_helpers.go
+6 -1
View File
@@ -1,6 +1,8 @@
package live
import (
"fmt"
"github.com/grafana/grafana/pkg/models"
"github.com/centrifugal/centrifuge"
@@ -18,7 +20,10 @@ func newPluginPacketSender(node *centrifuge.Node) *pluginPacketSender {
func (p *pluginPacketSender) Send(channel string, packet *backend.StreamPacket) error {
_, err := p.node.Publish(channel, packet.Data)
return err
if err != nil {
return fmt.Errorf("error publishing %s: %w", string(packet.Data), err)
}
return nil
}
type pluginPresenceGetter struct {
pkg/services/live/push/push.go
+104
View File
@@ -0,0 +1,104 @@
package push
import (
"context"
"net/http"
"github.com/grafana/grafana-live-sdk/telemetry/telegraf"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/registry"
"github.com/grafana/grafana/pkg/services/live"
"github.com/grafana/grafana/pkg/setting"
)
var (
logger = log.New("live_push")
)
func init() {
registry.RegisterServiceWithPriority(&Gateway{}, registry.Low)
}
// Gateway receives data and translates it to Grafana Live publications.
type Gateway struct {
Cfg *setting.Cfg `inject:""`
GrafanaLive *live.GrafanaLive `inject:""`
telegrafConverterWide *telegraf.Converter
telegrafConverterLabelsColumn *telegraf.Converter
}
// Init Gateway.
func (g *Gateway) Init() error {
logger.Info("Telemetry Gateway initialization")
if !g.IsEnabled() {
logger.Debug("Telemetry Gateway not enabled, skipping initialization")
return nil
}
// For now only Telegraf converter (influx format) is supported.
g.telegrafConverterWide = telegraf.NewConverter()
g.telegrafConverterLabelsColumn = telegraf.NewConverter(telegraf.WithUseLabelsColumn(true))
return nil
}
// Run Gateway.
func (g *Gateway) Run(ctx context.Context) error {
if !g.IsEnabled() {
logger.Debug("GrafanaLive feature not enabled, skipping initialization of Telemetry Gateway")
return nil
}
<-ctx.Done()
return ctx.Err()
}
// IsEnabled returns true if the Grafana Live feature is enabled.
func (g *Gateway) IsEnabled() bool {
return g.Cfg.IsLiveEnabled() // turn on when Live on for now.
}
func (g *Gateway) Handle(ctx *models.ReqContext) {
streamID := ctx.Params(":streamId")
stream, err := g.GrafanaLive.ManagedStreamRunner.GetOrCreateStream(streamID)
if err != nil {
logger.Error("Error getting stream", "error", err)
ctx.Resp.WriteHeader(http.StatusInternalServerError)
return
}
// TODO Grafana 8: decide which format to use or keep both.
converter := g.telegrafConverterWide
if ctx.Req.URL.Query().Get("format") == "labels_column" {
converter = g.telegrafConverterLabelsColumn
}
body, err := ctx.Req.Body().Bytes()
if err != nil {
logger.Error("Error reading body", "error", err)
ctx.Resp.WriteHeader(http.StatusInternalServerError)
return
}
logger.Debug("Live Push request body", "streamId", streamID, "bodyLength", len(body))
metricFrames, err := converter.Convert(body)
if err != nil {
logger.Error("Error converting metrics", "error", err)
ctx.Resp.WriteHeader(http.StatusInternalServerError)
return
}
// TODO -- make sure all packets are combined together!
// interval = "1s" vs flush_interval = "5s"
for _, mf := range metricFrames {
err := stream.Push(mf.Key(), mf.Frame())
if err != nil {
ctx.Resp.WriteHeader(http.StatusInternalServerError)
return
}
}
}
pkg/services/live/scope.go
+2
View File
@@ -7,4 +7,6 @@ const (
ScopePlugin = "plugin"
// ScopeDatasource passes control to a datasource plugin.
ScopeDatasource = "ds"
// ScopeStream is a managed data frame stream
ScopeStream = "stream"
)