public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Live: experimental HA with Redis (#34851)

Browse Source
This commit is contained in:
Alexander Emelin
2021-06-24 11:07:09 +03:00
committed by GitHub
parent 437faa72a9
commit 98893c0420
20 changed files with 785 additions and 308 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/live/managedstream/cache.go
+17
View File
@@ -0,0 +1,17 @@
package managedstream
import (
"encoding/json"
"github.com/grafana/grafana-plugin-sdk-go/data"
)
// FrameCache allows updating frame schema. Returns true is schema not changed.
type FrameCache interface {
// GetActiveChannels returns active managed stream channels with JSON schema.
GetActiveChannels(orgID int64) (map[string]json.RawMessage, error)
// GetFrame returns full JSON frame for a path.
GetFrame(orgID int64, channel string) (json.RawMessage, bool, error)
// Update updates frame cache and returns true if schema changed.
Update(orgID int64, channel string, frameJson data.FrameJSONCache) (bool, error)
}
pkg/services/live/managedstream/cache_memory.go
+54
View File
@@ -0,0 +1,54 @@
package managedstream
import (
"encoding/json"
"sync"
"github.com/grafana/grafana-plugin-sdk-go/data"
)
// MemoryFrameCache ...
type MemoryFrameCache struct {
mu sync.RWMutex
frames map[int64]map[string]data.FrameJSONCache
}
// NewMemoryFrameCache ...
func NewMemoryFrameCache() *MemoryFrameCache {
return &MemoryFrameCache{
frames: map[int64]map[string]data.FrameJSONCache{},
}
}
func (c *MemoryFrameCache) GetActiveChannels(orgID int64) (map[string]json.RawMessage, error) {
c.mu.RLock()
defer c.mu.RUnlock()
frames, ok := c.frames[orgID]
if !ok {
return nil, nil
}
info := make(map[string]json.RawMessage, len(frames))
for k, v := range frames {
info[k] = v.Bytes(data.IncludeSchemaOnly)
}
return info, nil
}
func (c *MemoryFrameCache) GetFrame(orgID int64, channel string) (json.RawMessage, bool, error) {
c.mu.RLock()
defer c.mu.RUnlock()
cachedFrame, ok := c.frames[orgID][channel]
return cachedFrame.Bytes(data.IncludeAll), ok, nil
}
func (c *MemoryFrameCache) Update(orgID int64, channel string, jsonFrame data.FrameJSONCache) (bool, error) {
c.mu.Lock()
defer c.mu.Unlock()
if _, ok := c.frames[orgID]; !ok {
c.frames[orgID] = map[string]data.FrameJSONCache{}
}
cachedJsonFrame, exists := c.frames[orgID][channel]
schemaUpdated := !exists || !cachedJsonFrame.SameSchema(&jsonFrame)
c.frames[orgID][channel] = jsonFrame
return schemaUpdated, nil
}
pkg/services/live/managedstream/cache_memory_test.go
+69
View File
@@ -0,0 +1,69 @@
package managedstream
import (
"encoding/json"
"testing"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/stretchr/testify/require"
)
func testFrameCache(t *testing.T, c FrameCache) {
// Create new frame and update cache.
frame := data.NewFrame("hello")
frameJsonCache, err := data.FrameToJSONCache(frame)
require.NoError(t, err)
updated, err := c.Update(1, "test", frameJsonCache)
require.NoError(t, err)
require.True(t, updated)
// Make sure channel is active.
channels, err := c.GetActiveChannels(1)
require.NoError(t, err)
schema, ok := channels["test"]
require.True(t, ok)
require.NotZero(t, schema)
// Make sure the same frame does not update schema.
updated, err = c.Update(1, "test", frameJsonCache)
require.NoError(t, err)
require.False(t, updated)
// Now construct new frame with updated schema.
newFrame := data.NewFrame("hello", data.NewField("new_field", nil, []int64{}))
frameJsonCache, err = data.FrameToJSONCache(newFrame)
require.NoError(t, err)
// Make sure schema updated.
updated, err = c.Update(1, "test", frameJsonCache)
require.NoError(t, err)
require.True(t, updated)
// Add the same with another orgID and make sure schema updated.
updated, err = c.Update(2, "test", frameJsonCache)
require.NoError(t, err)
require.True(t, updated)
// Make sure that the last frame successfully saved in cache.
frameJSON, ok, err := c.GetFrame(1, "test")
require.NoError(t, err)
require.True(t, ok)
var f data.Frame
err = json.Unmarshal(frameJSON, &f)
require.NoError(t, err)
require.Equal(t, "new_field", f.Fields[0].Name)
// Make sure channel has updated schema.
channels, err = c.GetActiveChannels(1)
require.NoError(t, err)
require.NotEqual(t, string(channels["test"]), string(schema))
}
func TestMemoryFrameCache(t *testing.T) {
c := NewMemoryFrameCache()
require.NotNil(t, c)
testFrameCache(t, c)
}
pkg/services/live/managedstream/cache_redis.go
+111
View File
@@ -0,0 +1,111 @@
package managedstream
import (
"encoding/json"
"errors"
"sync"
"time"
"github.com/grafana/grafana/pkg/services/live/orgchannel"
"github.com/grafana/grafana-plugin-sdk-go/data"
"gopkg.in/redis.v5"
)
// RedisFrameCache ...
type RedisFrameCache struct {
mu sync.RWMutex
redisClient *redis.Client
frames map[int64]map[string]data.FrameJSONCache
}
// NewRedisFrameCache ...
func NewRedisFrameCache(redisClient *redis.Client) *RedisFrameCache {
return &RedisFrameCache{
frames: map[int64]map[string]data.FrameJSONCache{},
redisClient: redisClient,
}
}
func (c *RedisFrameCache) GetActiveChannels(orgID int64) (map[string]json.RawMessage, error) {
c.mu.RLock()
defer c.mu.RUnlock()
frames, ok := c.frames[orgID]
if !ok {
return nil, nil
}
info := make(map[string]json.RawMessage, len(frames))
for k, v := range frames {
info[k] = v.Bytes(data.IncludeSchemaOnly)
}
return info, nil
}
func (c *RedisFrameCache) GetFrame(orgID int64, channel string) (json.RawMessage, bool, error) {
key := getCacheKey(orgchannel.PrependOrgID(orgID, channel))
cmd := c.redisClient.HGetAll(key)
result, err := cmd.Result()
if err != nil {
return nil, false, err
}
if len(result) == 0 {
return nil, false, nil
}
return json.RawMessage(result["frame"]), true, nil
}
const (
frameCacheTTL = 7 * 24 * time.Hour
)
func (c *RedisFrameCache) Update(orgID int64, channel string, jsonFrame data.FrameJSONCache) (bool, error) {
c.mu.Lock()
if _, ok := c.frames[orgID]; !ok {
c.frames[orgID] = map[string]data.FrameJSONCache{}
}
c.frames[orgID][channel] = jsonFrame
c.mu.Unlock()
stringSchema := string(jsonFrame.Bytes(data.IncludeSchemaOnly))
key := getCacheKey(orgchannel.PrependOrgID(orgID, channel))
pipe := c.redisClient.TxPipeline()
defer func() { _ = pipe.Close() }()
pipe.HGetAll(key)
pipe.HMSet(key, map[string]string{
"schema": stringSchema,
"frame": string(jsonFrame.Bytes(data.IncludeAll)),
})
pipe.Expire(key, frameCacheTTL)
replies, err := pipe.Exec()
if err != nil {
return false, err
}
if len(replies) == 0 {
return false, errors.New("no replies in response")
}
reply := replies[0]
if reply.Err() != nil {
return false, err
}
if mapReply, ok := reply.(*redis.StringStringMapCmd); ok {
result, err := mapReply.Result()
if err != nil {
return false, err
}
if len(result) == 0 {
return true, nil
}
return result["schema"] != stringSchema, nil
}
return true, nil
}
func getCacheKey(channelID string) string {
return "gf_live.managed_stream." + channelID
}
pkg/services/live/managedstream/cache_redis_test.go
+19
View File
@@ -0,0 +1,19 @@
// +build redis
package managedstream
import (
"testing"
"github.com/stretchr/testify/require"
"gopkg.in/redis.v5"
)
func TestRedisCacheStorage(t *testing.T) {
redisClient := redis.NewClient(&redis.Options{
Addr: "localhost:6379",
})
c := NewRedisFrameCache(redisClient)
require.NotNil(t, c)
testFrameCache(t, c)
}
pkg/services/live/managedstream/runner.go
+92 -80
View File
@@ -3,6 +3,7 @@ package managedstream
import (
"context"
"encoding/json"
"fmt"
"sync"
"time"
@@ -11,7 +12,6 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/live"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/util"
)
var (
@@ -20,19 +20,53 @@ var (
// Runner keeps ManagedStream per streamID.
type Runner struct {
mu sync.RWMutex
streams map[int64]map[string]*ManagedStream
publisher models.ChannelPublisher
mu sync.RWMutex
streams map[int64]map[string]*ManagedStream
publisher models.ChannelPublisher
frameCache FrameCache
}
// NewRunner creates new Runner.
func NewRunner(publisher models.ChannelPublisher) *Runner {
func NewRunner(publisher models.ChannelPublisher, frameCache FrameCache) *Runner {
return &Runner{
publisher: publisher,
streams: map[int64]map[string]*ManagedStream{},
publisher: publisher,
streams: map[int64]map[string]*ManagedStream{},
frameCache: frameCache,
}
}
func (r *Runner) GetManagedChannels(orgID int64) ([]*ManagedChannel, error) {
channels := make([]*ManagedChannel, 0)
for _, v := range r.Streams(orgID) {
streamChannels, err := v.ListChannels(orgID)
if err != nil {
return nil, err
}
channels = append(channels, streamChannels...)
}
// Hardcode sample streams
frameJSON, err := data.FrameToJSON(data.NewFrame("testdata",
data.NewField("Time", nil, make([]time.Time, 0)),
data.NewField("Value", nil, make([]float64, 0)),
data.NewField("Min", nil, make([]float64, 0)),
data.NewField("Max", nil, make([]float64, 0)),
), data.IncludeSchemaOnly)
if err == nil {
channels = append(channels, &ManagedChannel{
Channel: "plugin/testdata/random-2s-stream",
Data: frameJSON,
}, &ManagedChannel{
Channel: "plugin/testdata/random-flakey-stream",
Data: frameJSON,
}, &ManagedChannel{
Channel: "plugin/testdata/random-20Hz-stream",
Data: frameJSON,
})
}
return channels, nil
}
// Streams returns a map of active managed streams (per streamID).
func (r *Runner) Streams(orgID int64) map[string]*ManagedStream {
r.mu.RLock()
@@ -58,7 +92,7 @@ func (r *Runner) GetOrCreateStream(orgID int64, streamID string) (*ManagedStream
}
s, ok := r.streams[orgID][streamID]
if !ok {
s = NewManagedStream(streamID, r.publisher)
s = NewManagedStream(streamID, r.publisher, r.frameCache)
r.streams[orgID][streamID] = s
}
return s, nil
@@ -66,112 +100,90 @@ func (r *Runner) GetOrCreateStream(orgID int64, streamID string) (*ManagedStream
// ManagedStream holds the state of a managed stream.
type ManagedStream struct {
mu sync.RWMutex
id string
start time.Time
last map[int64]map[string]data.FrameJSONCache
publisher models.ChannelPublisher
id string
start time.Time
publisher models.ChannelPublisher
frameCache FrameCache
}
// NewManagedStream creates new ManagedStream.
func NewManagedStream(id string, publisher models.ChannelPublisher) *ManagedStream {
func NewManagedStream(id string, publisher models.ChannelPublisher, schemaUpdater FrameCache) *ManagedStream {
return &ManagedStream{
id: id,
start: time.Now(),
last: map[int64]map[string]data.FrameJSONCache{},
publisher: publisher,
id: id,
start: time.Now(),
publisher: publisher,
frameCache: schemaUpdater,
}
}
// ManagedChannel represents a managed stream.
type ManagedChannel struct {
Channel string `json:"channel"`
Data json.RawMessage `json:"data"`
}
// ListChannels returns info for the UI about this stream.
func (s *ManagedStream) ListChannels(orgID int64, prefix string) []util.DynMap {
s.mu.RLock()
defer s.mu.RUnlock()
if _, ok := s.last[orgID]; !ok {
return []util.DynMap{}
func (s *ManagedStream) ListChannels(orgID int64) ([]*ManagedChannel, error) {
paths, err := s.frameCache.GetActiveChannels(orgID)
if err != nil {
return []*ManagedChannel{}, fmt.Errorf("error getting active managed stream paths: %v", err)
}
info := make([]util.DynMap, 0, len(s.last[orgID]))
for k, v := range s.last[orgID] {
ch := util.DynMap{}
ch["channel"] = prefix + k
ch["data"] = json.RawMessage(v.Bytes(data.IncludeSchemaOnly))
info = append(info, ch)
info := make([]*ManagedChannel, 0, len(paths))
for k, v := range paths {
managedChannel := &ManagedChannel{
Channel: k,
Data: v,
}
info = append(info, managedChannel)
}
return info
return info, nil
}
// Push sends frame to the stream and saves it for later retrieval by subscribers.
// unstableSchema flag can be set to disable schema caching for a path.
func (s *ManagedStream) Push(orgID int64, path string, frame *data.Frame) error {
// Keep schema + data for last packet.
msg, err := data.FrameToJSONCache(frame)
jsonFrameCache, err := data.FrameToJSONCache(frame)
if err != nil {
logger.Error("Error marshaling frame with data", "error", err)
return err
}
s.mu.Lock()
if _, ok := s.last[orgID]; !ok {
s.last[orgID] = map[string]data.FrameJSONCache{}
}
last, exists := s.last[orgID][path]
s.last[orgID][path] = msg
s.mu.Unlock()
include := data.IncludeAll
if exists && last.SameSchema(&msg) {
// When the schema has not changed, just send the data.
include = data.IncludeDataOnly
}
frameJSON := msg.Bytes(include)
// The channel this will be posted into.
channel := live.Channel{Scope: live.ScopeStream, Namespace: s.id, Path: path}.String()
isUpdated, err := s.frameCache.Update(orgID, channel, jsonFrameCache)
if err != nil {
logger.Error("Error updating managed stream schema", "error", err)
return err
}
// When the schema has not changed, just send the data.
include := data.IncludeDataOnly
if isUpdated {
// When the schema has been changed, send all.
include = data.IncludeAll
}
frameJSON := jsonFrameCache.Bytes(include)
logger.Debug("Publish data to channel", "channel", channel, "dataLength", len(frameJSON))
return s.publisher(orgID, channel, frameJSON)
}
// getLastPacket retrieves last packet channel.
func (s *ManagedStream) getLastPacket(orgId int64, path string) (json.RawMessage, bool) {
s.mu.RLock()
defer s.mu.RUnlock()
_, ok := s.last[orgId]
if !ok {
return nil, false
}
msg, ok := s.last[orgId][path]
if ok {
return msg.Bytes(data.IncludeAll), ok
}
return nil, ok
}
func (s *ManagedStream) GetHandlerForPath(_ string) (models.ChannelHandler, error) {
return s, nil
}
func (s *ManagedStream) OnSubscribe(_ context.Context, u *models.SignedInUser, e models.SubscribeEvent) (models.SubscribeReply, backend.SubscribeStreamStatus, error) {
reply := models.SubscribeReply{}
packet, ok := s.getLastPacket(u.OrgId, e.Path)
frameJSON, ok, err := s.frameCache.GetFrame(u.OrgId, e.Channel)
if err != nil {
return reply, 0, err
}
if ok {
reply.Data = packet
reply.Data = frameJSON
}
return reply, backend.SubscribeStreamStatusOK, nil
}
func (s *ManagedStream) OnPublish(_ context.Context, u *models.SignedInUser, evt models.PublishEvent) (models.PublishReply, backend.PublishStreamStatus, error) {
var frame data.Frame
err := json.Unmarshal(evt.Data, &frame)
if err != nil {
// Stream scope only deals with data frames.
return models.PublishReply{}, 0, err
}
err = s.Push(u.OrgId, evt.Path, &frame)
if err != nil {
// Stream scope only deals with data frames.
return models.PublishReply{}, 0, err
}
return models.PublishReply{}, backend.PublishStreamStatusOK, nil
func (s *ManagedStream) OnPublish(_ context.Context, _ *models.SignedInUser, _ models.PublishEvent) (models.PublishReply, backend.PublishStreamStatus, error) {
return models.PublishReply{}, backend.PublishStreamStatusPermissionDenied, nil
}
pkg/services/live/managedstream/runner_test.go
+1 -17
View File
@@ -3,7 +3,6 @@ package managedstream
import (
"testing"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/stretchr/testify/require"
)
@@ -19,21 +18,6 @@ func (p *testPublisher) publish(orgID int64, _ string, _ []byte) error {
func TestNewManagedStream(t *testing.T) {
publisher := &testPublisher{orgID: 1, t: t}
c := NewManagedStream("a", publisher.publish)
c := NewManagedStream("a", publisher.publish, NewMemoryFrameCache())
require.NotNil(t, c)
}
func TestManagedStream_GetLastPacket(t *testing.T) {
var orgID int64 = 1
publisher := &testPublisher{orgID: orgID, t: t}
c := NewManagedStream("a", publisher.publish)
_, ok := c.getLastPacket(orgID, "test")
require.False(t, ok)
err := c.Push(orgID, "test", data.NewFrame("hello"))
require.NoError(t, err)
s, ok := c.getLastPacket(orgID, "test")
require.NoError(t, err)
require.True(t, ok)
require.JSONEq(t, `{"schema":{"name":"hello","fields":[]},"data":{"values":[]}}`, string(s))
}