Live: experimental HA with Redis (#34851)
This commit is contained in:
Alexander Emelin
@@ -3,6 +3,7 @@ package managedstream
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -11,7 +12,6 @@ import (
|
||||
"github.com/grafana/grafana-plugin-sdk-go/live"
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
"github.com/grafana/grafana/pkg/models"
|
||||
"github.com/grafana/grafana/pkg/util"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -20,19 +20,53 @@ var (
|
||||
|
||||
// Runner keeps ManagedStream per streamID.
|
||||
type Runner struct {
|
||||
mu sync.RWMutex
|
||||
streams map[int64]map[string]*ManagedStream
|
||||
publisher models.ChannelPublisher
|
||||
mu sync.RWMutex
|
||||
streams map[int64]map[string]*ManagedStream
|
||||
publisher models.ChannelPublisher
|
||||
frameCache FrameCache
|
||||
}
|
||||
|
||||
// NewRunner creates new Runner.
|
||||
func NewRunner(publisher models.ChannelPublisher) *Runner {
|
||||
func NewRunner(publisher models.ChannelPublisher, frameCache FrameCache) *Runner {
|
||||
return &Runner{
|
||||
publisher: publisher,
|
||||
streams: map[int64]map[string]*ManagedStream{},
|
||||
publisher: publisher,
|
||||
streams: map[int64]map[string]*ManagedStream{},
|
||||
frameCache: frameCache,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Runner) GetManagedChannels(orgID int64) ([]*ManagedChannel, error) {
|
||||
channels := make([]*ManagedChannel, 0)
|
||||
for _, v := range r.Streams(orgID) {
|
||||
streamChannels, err := v.ListChannels(orgID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
channels = append(channels, streamChannels...)
|
||||
}
|
||||
|
||||
// Hardcode sample streams
|
||||
frameJSON, err := data.FrameToJSON(data.NewFrame("testdata",
|
||||
data.NewField("Time", nil, make([]time.Time, 0)),
|
||||
data.NewField("Value", nil, make([]float64, 0)),
|
||||
data.NewField("Min", nil, make([]float64, 0)),
|
||||
data.NewField("Max", nil, make([]float64, 0)),
|
||||
), data.IncludeSchemaOnly)
|
||||
if err == nil {
|
||||
channels = append(channels, &ManagedChannel{
|
||||
Channel: "plugin/testdata/random-2s-stream",
|
||||
Data: frameJSON,
|
||||
}, &ManagedChannel{
|
||||
Channel: "plugin/testdata/random-flakey-stream",
|
||||
Data: frameJSON,
|
||||
}, &ManagedChannel{
|
||||
Channel: "plugin/testdata/random-20Hz-stream",
|
||||
Data: frameJSON,
|
||||
})
|
||||
}
|
||||
return channels, nil
|
||||
}
|
||||
|
||||
// Streams returns a map of active managed streams (per streamID).
|
||||
func (r *Runner) Streams(orgID int64) map[string]*ManagedStream {
|
||||
r.mu.RLock()
|
||||
@@ -58,7 +92,7 @@ func (r *Runner) GetOrCreateStream(orgID int64, streamID string) (*ManagedStream
|
||||
}
|
||||
s, ok := r.streams[orgID][streamID]
|
||||
if !ok {
|
||||
s = NewManagedStream(streamID, r.publisher)
|
||||
s = NewManagedStream(streamID, r.publisher, r.frameCache)
|
||||
r.streams[orgID][streamID] = s
|
||||
}
|
||||
return s, nil
|
||||
@@ -66,112 +100,90 @@ func (r *Runner) GetOrCreateStream(orgID int64, streamID string) (*ManagedStream
|
||||
|
||||
// ManagedStream holds the state of a managed stream.
|
||||
type ManagedStream struct {
|
||||
mu sync.RWMutex
|
||||
id string
|
||||
start time.Time
|
||||
last map[int64]map[string]data.FrameJSONCache
|
||||
publisher models.ChannelPublisher
|
||||
id string
|
||||
start time.Time
|
||||
publisher models.ChannelPublisher
|
||||
frameCache FrameCache
|
||||
}
|
||||
|
||||
// NewManagedStream creates new ManagedStream.
|
||||
func NewManagedStream(id string, publisher models.ChannelPublisher) *ManagedStream {
|
||||
func NewManagedStream(id string, publisher models.ChannelPublisher, schemaUpdater FrameCache) *ManagedStream {
|
||||
return &ManagedStream{
|
||||
id: id,
|
||||
start: time.Now(),
|
||||
last: map[int64]map[string]data.FrameJSONCache{},
|
||||
publisher: publisher,
|
||||
id: id,
|
||||
start: time.Now(),
|
||||
publisher: publisher,
|
||||
frameCache: schemaUpdater,
|
||||
}
|
||||
}
|
||||
|
||||
// ManagedChannel represents a managed stream.
|
||||
type ManagedChannel struct {
|
||||
Channel string `json:"channel"`
|
||||
Data json.RawMessage `json:"data"`
|
||||
}
|
||||
|
||||
// ListChannels returns info for the UI about this stream.
|
||||
func (s *ManagedStream) ListChannels(orgID int64, prefix string) []util.DynMap {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
if _, ok := s.last[orgID]; !ok {
|
||||
return []util.DynMap{}
|
||||
func (s *ManagedStream) ListChannels(orgID int64) ([]*ManagedChannel, error) {
|
||||
paths, err := s.frameCache.GetActiveChannels(orgID)
|
||||
if err != nil {
|
||||
return []*ManagedChannel{}, fmt.Errorf("error getting active managed stream paths: %v", err)
|
||||
}
|
||||
|
||||
info := make([]util.DynMap, 0, len(s.last[orgID]))
|
||||
for k, v := range s.last[orgID] {
|
||||
ch := util.DynMap{}
|
||||
ch["channel"] = prefix + k
|
||||
ch["data"] = json.RawMessage(v.Bytes(data.IncludeSchemaOnly))
|
||||
info = append(info, ch)
|
||||
info := make([]*ManagedChannel, 0, len(paths))
|
||||
for k, v := range paths {
|
||||
managedChannel := &ManagedChannel{
|
||||
Channel: k,
|
||||
Data: v,
|
||||
}
|
||||
info = append(info, managedChannel)
|
||||
}
|
||||
return info
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// Push sends frame to the stream and saves it for later retrieval by subscribers.
|
||||
// unstableSchema flag can be set to disable schema caching for a path.
|
||||
func (s *ManagedStream) Push(orgID int64, path string, frame *data.Frame) error {
|
||||
// Keep schema + data for last packet.
|
||||
msg, err := data.FrameToJSONCache(frame)
|
||||
jsonFrameCache, err := data.FrameToJSONCache(frame)
|
||||
if err != nil {
|
||||
logger.Error("Error marshaling frame with data", "error", err)
|
||||
return err
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
if _, ok := s.last[orgID]; !ok {
|
||||
s.last[orgID] = map[string]data.FrameJSONCache{}
|
||||
}
|
||||
last, exists := s.last[orgID][path]
|
||||
s.last[orgID][path] = msg
|
||||
s.mu.Unlock()
|
||||
|
||||
include := data.IncludeAll
|
||||
if exists && last.SameSchema(&msg) {
|
||||
// When the schema has not changed, just send the data.
|
||||
include = data.IncludeDataOnly
|
||||
}
|
||||
frameJSON := msg.Bytes(include)
|
||||
|
||||
// The channel this will be posted into.
|
||||
channel := live.Channel{Scope: live.ScopeStream, Namespace: s.id, Path: path}.String()
|
||||
|
||||
isUpdated, err := s.frameCache.Update(orgID, channel, jsonFrameCache)
|
||||
if err != nil {
|
||||
logger.Error("Error updating managed stream schema", "error", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// When the schema has not changed, just send the data.
|
||||
include := data.IncludeDataOnly
|
||||
if isUpdated {
|
||||
// When the schema has been changed, send all.
|
||||
include = data.IncludeAll
|
||||
}
|
||||
frameJSON := jsonFrameCache.Bytes(include)
|
||||
|
||||
logger.Debug("Publish data to channel", "channel", channel, "dataLength", len(frameJSON))
|
||||
return s.publisher(orgID, channel, frameJSON)
|
||||
}
|
||||
|
||||
// getLastPacket retrieves last packet channel.
|
||||
func (s *ManagedStream) getLastPacket(orgId int64, path string) (json.RawMessage, bool) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
_, ok := s.last[orgId]
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
msg, ok := s.last[orgId][path]
|
||||
if ok {
|
||||
return msg.Bytes(data.IncludeAll), ok
|
||||
}
|
||||
return nil, ok
|
||||
}
|
||||
|
||||
func (s *ManagedStream) GetHandlerForPath(_ string) (models.ChannelHandler, error) {
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s *ManagedStream) OnSubscribe(_ context.Context, u *models.SignedInUser, e models.SubscribeEvent) (models.SubscribeReply, backend.SubscribeStreamStatus, error) {
|
||||
reply := models.SubscribeReply{}
|
||||
packet, ok := s.getLastPacket(u.OrgId, e.Path)
|
||||
frameJSON, ok, err := s.frameCache.GetFrame(u.OrgId, e.Channel)
|
||||
if err != nil {
|
||||
return reply, 0, err
|
||||
}
|
||||
if ok {
|
||||
reply.Data = packet
|
||||
reply.Data = frameJSON
|
||||
}
|
||||
return reply, backend.SubscribeStreamStatusOK, nil
|
||||
}
|
||||
|
||||
func (s *ManagedStream) OnPublish(_ context.Context, u *models.SignedInUser, evt models.PublishEvent) (models.PublishReply, backend.PublishStreamStatus, error) {
|
||||
var frame data.Frame
|
||||
err := json.Unmarshal(evt.Data, &frame)
|
||||
if err != nil {
|
||||
// Stream scope only deals with data frames.
|
||||
return models.PublishReply{}, 0, err
|
||||
}
|
||||
err = s.Push(u.OrgId, evt.Path, &frame)
|
||||
if err != nil {
|
||||
// Stream scope only deals with data frames.
|
||||
return models.PublishReply{}, 0, err
|
||||
}
|
||||
return models.PublishReply{}, backend.PublishStreamStatusOK, nil
|
||||
func (s *ManagedStream) OnPublish(_ context.Context, _ *models.SignedInUser, _ models.PublishEvent) (models.PublishReply, backend.PublishStreamStatus, error) {
|
||||
return models.PublishReply{}, backend.PublishStreamStatusPermissionDenied, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user