From d4f578966056ea1820186327ca35138df5032fa2 Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 14 Feb 2019 23:13:46 +0100 Subject: [PATCH 01/75] cache: initial version of db cache --- pkg/infra/distcache/database_storage.go | 82 +++++++++++++++++ pkg/infra/distcache/distcache.go | 68 +++++++++++++++ pkg/infra/distcache/distcache_test.go | 87 +++++++++++++++++++ .../sqlstore/migrations/cache_data_mig.go | 17 ++++ .../sqlstore/migrations/migrations.go | 1 + 5 files changed, 255 insertions(+) create mode 100644 pkg/infra/distcache/database_storage.go create mode 100644 pkg/infra/distcache/distcache.go create mode 100644 pkg/infra/distcache/distcache_test.go create mode 100644 pkg/services/sqlstore/migrations/cache_data_mig.go diff --git a/pkg/infra/distcache/database_storage.go b/pkg/infra/distcache/database_storage.go new file mode 100644 index 00000000000..8286f65fea6 --- /dev/null +++ b/pkg/infra/distcache/database_storage.go @@ -0,0 +1,82 @@ +package distcache + +import ( + "time" + + "github.com/grafana/grafana/pkg/services/sqlstore" +) + +type databaseCache struct { + SQLStore *sqlstore.SqlStore +} + +var getTime = time.Now + +func (dc *databaseCache) Get(key string) (interface{}, error) { + //now := getTime().Unix() + + cacheHits := []CacheData{} + err := dc.SQLStore.NewSession().Where(`key = ?`, key).Find(&cacheHits) + if err != nil { + return nil, err + } + + var cacheHit CacheData + if len(cacheHits) == 0 { + return nil, ErrCacheItemNotFound + } + + cacheHit = cacheHits[0] + if cacheHit.Expires > 0 { + if getTime().Unix()-cacheHit.CreatedAt >= cacheHit.Expires { + dc.Delete(key) + return nil, ErrCacheItemNotFound + } + } + + item := &Item{} + if err = DecodeGob(cacheHit.Data, item); err != nil { + return nil, err + } + + return item.Val, nil +} + +type CacheData struct { + Key string + Data []byte + Expires int64 + CreatedAt int64 +} + +func (dc *databaseCache) Put(key string, value interface{}, expire int64) error { + item := &Item{Val: value} + data, err := EncodeGob(item) + if err != nil { + return err + } + + now := getTime().Unix() + + cacheHits := []CacheData{} + err = dc.SQLStore.NewSession().Where(`key = ?`, key).Find(&cacheHits) + if err != nil { + return err + } + + if len(cacheHits) > 0 { + _, err = dc.SQLStore.NewSession().Exec("UPDATE cached_data SET data=?, created=?, expire=? WHERE key=?", data, now, expire, key) + } else { + _, err = dc.SQLStore.NewSession().Exec("INSERT INTO cache_data(key,data,created_at,expires) VALUES(?,?,?,?)", key, data, now, expire) + } + + return err +} + +func (dc *databaseCache) Delete(key string) error { + sql := `DELETE FROM cache_data WHERE key = ?` + + _, err := dc.SQLStore.NewSession().Exec(sql, key) + + return err +} diff --git a/pkg/infra/distcache/distcache.go b/pkg/infra/distcache/distcache.go new file mode 100644 index 00000000000..11efd435de3 --- /dev/null +++ b/pkg/infra/distcache/distcache.go @@ -0,0 +1,68 @@ +package distcache + +import ( + "bytes" + "encoding/gob" + "errors" + + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/services/sqlstore" + + "github.com/grafana/grafana/pkg/registry" +) + +var ( + ErrCacheItemNotFound = errors.New("cache item not found") +) + +func init() { + registry.RegisterService(&DistributedCache{}) +} + +// Init initializes the service +func (ds *DistributedCache) Init() error { + ds.log = log.New("distributed.cache") + + // memory + // redis + // memcache + // database. using SQLSTORE + ds.Client = &databaseCache{SQLStore: ds.SQLStore} + + return nil +} + +// DistributedCache allows Grafana to cache data outside its own process +type DistributedCache struct { + log log.Logger + Client cacheStorage + SQLStore *sqlstore.SqlStore `inject:""` +} + +type Item struct { + Val interface{} + Created int64 + Expire int64 +} + +func EncodeGob(item *Item) ([]byte, error) { + buf := bytes.NewBuffer(nil) + err := gob.NewEncoder(buf).Encode(item) + return buf.Bytes(), err +} + +func DecodeGob(data []byte, out *Item) error { + buf := bytes.NewBuffer(data) + return gob.NewDecoder(buf).Decode(&out) +} + +type cacheStorage interface { + // Get reads object from Cache + Get(key string) (interface{}, error) + + // Puts an object into the cache + Put(key string, value interface{}, expire int64) error + + // Delete object from cache + Delete(key string) error +} diff --git a/pkg/infra/distcache/distcache_test.go b/pkg/infra/distcache/distcache_test.go new file mode 100644 index 00000000000..88066daec7e --- /dev/null +++ b/pkg/infra/distcache/distcache_test.go @@ -0,0 +1,87 @@ +package distcache + +import ( + "encoding/gob" + "testing" + "time" + + "github.com/bmizerany/assert" + + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/services/sqlstore" +) + +type CacheableStruct struct { + String string + Int64 int64 +} + +func init() { + gob.Register(CacheableStruct{}) +} + +func createClient(t *testing.T) cacheStorage { + t.Helper() + + sqlstore := sqlstore.InitTestDB(t) + dc := DistributedCache{log: log.New("test.logger"), SQLStore: sqlstore} + dc.Init() + return dc.Client +} + +func TestCanPutIntoDatabaseStorage(t *testing.T) { + client := createClient(t) + cacheableStruct := CacheableStruct{String: "hej", Int64: 2000} + + err := client.Put("key", cacheableStruct, 1000) + assert.Equal(t, err, nil) + + data, err := client.Get("key") + s, ok := data.(CacheableStruct) + + assert.Equal(t, ok, true) + assert.Equal(t, s.String, "hej") + assert.Equal(t, s.Int64, int64(2000)) + + err = client.Delete("key") + assert.Equal(t, err, nil) + + _, err = client.Get("key") + assert.Equal(t, err, ErrCacheItemNotFound) +} + +func TestCanNotFetchExpiredItems(t *testing.T) { + client := createClient(t) + + cacheableStruct := CacheableStruct{String: "hej", Int64: 2000} + + // insert cache item one day back + getTime = func() time.Time { return time.Now().AddDate(0, 0, -2) } + err := client.Put("key", cacheableStruct, 10000) + assert.Equal(t, err, nil) + + // should not be able to read that value since its expired + getTime = time.Now + _, err = client.Get("key") + assert.Equal(t, err, ErrCacheItemNotFound) +} + +func TestCanSetInfiniteCacheExpiration(t *testing.T) { + client := createClient(t) + + cacheableStruct := CacheableStruct{String: "hej", Int64: 2000} + + // insert cache item one day back + getTime = func() time.Time { return time.Now().AddDate(0, 0, -2) } + err := client.Put("key", cacheableStruct, 0) + assert.Equal(t, err, nil) + + // should not be able to read that value since its expired + getTime = time.Now + data, err := client.Get("key") + s, ok := data.(CacheableStruct) + + assert.Equal(t, ok, true) + assert.Equal(t, s.String, "hej") + assert.Equal(t, s.Int64, int64(2000)) +} diff --git a/pkg/services/sqlstore/migrations/cache_data_mig.go b/pkg/services/sqlstore/migrations/cache_data_mig.go new file mode 100644 index 00000000000..1201b38e337 --- /dev/null +++ b/pkg/services/sqlstore/migrations/cache_data_mig.go @@ -0,0 +1,17 @@ +package migrations + +import . "github.com/grafana/grafana/pkg/services/sqlstore/migrator" + +func addCacheMigration(mg *Migrator) { + var cacheDataV1 = Table{ + Name: "cache_data", + Columns: []*Column{ + {Name: "key", Type: DB_Char, IsPrimaryKey: true, Length: 16}, + {Name: "data", Type: DB_Blob}, + {Name: "expires", Type: DB_Integer, Length: 255, Nullable: false}, + {Name: "created_at", Type: DB_Integer, Length: 255, Nullable: false}, + }, + } + + mg.AddMigration("create cache_data table", NewAddTableMigration(cacheDataV1)) +} diff --git a/pkg/services/sqlstore/migrations/migrations.go b/pkg/services/sqlstore/migrations/migrations.go index 931259ec3ed..3e40c749f37 100644 --- a/pkg/services/sqlstore/migrations/migrations.go +++ b/pkg/services/sqlstore/migrations/migrations.go @@ -33,6 +33,7 @@ func AddMigrations(mg *Migrator) { addUserAuthMigrations(mg) addServerlockMigrations(mg) addUserAuthTokenMigrations(mg) + addCacheMigration(mg) } func addMigrationLogMigrations(mg *Migrator) { From 996d5059b119a9927059812a5384edda7bf2a9d8 Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 15 Feb 2019 09:48:32 +0100 Subject: [PATCH 02/75] test at interface level instead impl --- pkg/infra/distcache/distcache.go | 26 +++++++++++++++++----- pkg/infra/distcache/distcache_test.go | 32 +++++++++++++++------------ 2 files changed, 39 insertions(+), 19 deletions(-) diff --git a/pkg/infra/distcache/distcache.go b/pkg/infra/distcache/distcache.go index 11efd435de3..3a2d553953a 100644 --- a/pkg/infra/distcache/distcache.go +++ b/pkg/infra/distcache/distcache.go @@ -23,15 +23,31 @@ func init() { func (ds *DistributedCache) Init() error { ds.log = log.New("distributed.cache") - // memory - // redis - // memcache - // database. using SQLSTORE - ds.Client = &databaseCache{SQLStore: ds.SQLStore} + ds.Client = createClient(CacheOpts{}, ds.SQLStore) return nil } +type CacheOpts struct { + name string +} + +func createClient(opts CacheOpts, sqlstore *sqlstore.SqlStore) cacheStorage { + if opts.name == "redis" { + return nil + } + + if opts.name == "memcache" { + return nil + } + + if opts.name == "memory" { + return nil + } + + return &databaseCache{SQLStore: sqlstore} +} + // DistributedCache allows Grafana to cache data outside its own process type DistributedCache struct { log log.Logger diff --git a/pkg/infra/distcache/distcache_test.go b/pkg/infra/distcache/distcache_test.go index 88066daec7e..d3009753a14 100644 --- a/pkg/infra/distcache/distcache_test.go +++ b/pkg/infra/distcache/distcache_test.go @@ -7,7 +7,6 @@ import ( "github.com/bmizerany/assert" - "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/services/sqlstore" ) @@ -20,20 +19,29 @@ func init() { gob.Register(CacheableStruct{}) } -func createClient(t *testing.T) cacheStorage { +func createTestClient(t *testing.T, name string) cacheStorage { t.Helper() sqlstore := sqlstore.InitTestDB(t) - dc := DistributedCache{log: log.New("test.logger"), SQLStore: sqlstore} - dc.Init() - return dc.Client + return createClient(CacheOpts{name: name}, sqlstore) } -func TestCanPutIntoDatabaseStorage(t *testing.T) { - client := createClient(t) +func TestAllCacheClients(t *testing.T) { + clients := []string{"database"} // add redis, memcache, memory + + for _, v := range clients { + client := createTestClient(t, v) + + CanPutGetAndDeleteCachedObjects(t, client) + CanNotFetchExpiredItems(t, client) + CanSetInfiniteCacheExpiration(t, client) + } +} + +func CanPutGetAndDeleteCachedObjects(t *testing.T, client cacheStorage) { cacheableStruct := CacheableStruct{String: "hej", Int64: 2000} - err := client.Put("key", cacheableStruct, 1000) + err := client.Put("key", cacheableStruct, 0) assert.Equal(t, err, nil) data, err := client.Get("key") @@ -50,9 +58,7 @@ func TestCanPutIntoDatabaseStorage(t *testing.T) { assert.Equal(t, err, ErrCacheItemNotFound) } -func TestCanNotFetchExpiredItems(t *testing.T) { - client := createClient(t) - +func CanNotFetchExpiredItems(t *testing.T, client cacheStorage) { cacheableStruct := CacheableStruct{String: "hej", Int64: 2000} // insert cache item one day back @@ -66,9 +72,7 @@ func TestCanNotFetchExpiredItems(t *testing.T) { assert.Equal(t, err, ErrCacheItemNotFound) } -func TestCanSetInfiniteCacheExpiration(t *testing.T) { - client := createClient(t) - +func CanSetInfiniteCacheExpiration(t *testing.T, client cacheStorage) { cacheableStruct := CacheableStruct{String: "hej", Int64: 2000} // insert cache item one day back From d99af239462cf015db935d2e34a6fd885f350dc0 Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 15 Feb 2019 14:31:52 +0100 Subject: [PATCH 03/75] add garbage collector for database cache --- pkg/infra/distcache/database_storage.go | 36 +++++++++++-- pkg/infra/distcache/database_storage_test.go | 50 +++++++++++++++++++ .../sqlstore/migrations/cache_data_mig.go | 23 +++++---- 3 files changed, 97 insertions(+), 12 deletions(-) create mode 100644 pkg/infra/distcache/database_storage_test.go diff --git a/pkg/infra/distcache/database_storage.go b/pkg/infra/distcache/database_storage.go index 8286f65fea6..ed55208e18d 100644 --- a/pkg/infra/distcache/database_storage.go +++ b/pkg/infra/distcache/database_storage.go @@ -3,18 +3,48 @@ package distcache import ( "time" + "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/services/sqlstore" ) type databaseCache struct { SQLStore *sqlstore.SqlStore + log log.Logger +} + +func newDatabaseCache(sqlstore *sqlstore.SqlStore) *databaseCache { + dc := &databaseCache{ + SQLStore: sqlstore, + log: log.New("distcache.database"), + } + + go dc.StartGC() + return dc } var getTime = time.Now -func (dc *databaseCache) Get(key string) (interface{}, error) { - //now := getTime().Unix() +func (dc *databaseCache) internalRunGC() { + now := getTime().Unix() + sql := `DELETE FROM cache_data WHERE (? - created) >= expire` + //EXTRACT(EPOCH FROM NOW()) - created >= expire + //UNIX_TIMESTAMP(NOW()) - created >= expire + _, err := dc.SQLStore.NewSession().Exec(sql, now) + if err != nil { + dc.log.Error("failed to run garbage collect", "error", err) + } +} + +func (dc *databaseCache) StartGC() { + dc.internalRunGC() + + time.AfterFunc(time.Second*10, func() { + dc.StartGC() + }) +} + +func (dc *databaseCache) Get(key string) (interface{}, error) { cacheHits := []CacheData{} err := dc.SQLStore.NewSession().Where(`key = ?`, key).Find(&cacheHits) if err != nil { @@ -65,7 +95,7 @@ func (dc *databaseCache) Put(key string, value interface{}, expire int64) error } if len(cacheHits) > 0 { - _, err = dc.SQLStore.NewSession().Exec("UPDATE cached_data SET data=?, created=?, expire=? WHERE key=?", data, now, expire, key) + _, err = dc.SQLStore.NewSession().Exec("UPDATE cache_data SET data=?, created=?, expire=? WHERE key=?", data, now, expire, key) } else { _, err = dc.SQLStore.NewSession().Exec("INSERT INTO cache_data(key,data,created_at,expires) VALUES(?,?,?,?)", key, data, now, expire) } diff --git a/pkg/infra/distcache/database_storage_test.go b/pkg/infra/distcache/database_storage_test.go new file mode 100644 index 00000000000..2e6339c7c32 --- /dev/null +++ b/pkg/infra/distcache/database_storage_test.go @@ -0,0 +1,50 @@ +package distcache + +import ( + "testing" + "time" + + "github.com/bmizerany/assert" + + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/services/sqlstore" +) + +func TestDatabaseStorageGarbageCollection(t *testing.T) { + sqlstore := sqlstore.InitTestDB(t) + + db := &databaseCache{ + SQLStore: sqlstore, + log: log.New("distcache.database"), + } + + obj := &CacheableStruct{String: "foolbar"} + + //set time.now to 2 weeks ago + getTime = func() time.Time { return time.Now().AddDate(0, 0, -2) } + db.Put("key1", obj, 1000) + db.Put("key2", obj, 1000) + db.Put("key3", obj, 1000) + + // insert object that should never expire + db.Put("key4", obj, 0) + + getTime = time.Now + db.Put("key5", obj, 1000) + + //run GC + db.internalRunGC() + + //try to read values + _, err := db.Get("key1") + assert.Equal(t, err, ErrCacheItemNotFound) + _, err = db.Get("key2") + assert.Equal(t, err, ErrCacheItemNotFound) + _, err = db.Get("key3") + assert.Equal(t, err, ErrCacheItemNotFound) + + _, err = db.Get("key4") + assert.Equal(t, err, nil) + _, err = db.Get("key5") + assert.Equal(t, err, nil) +} diff --git a/pkg/services/sqlstore/migrations/cache_data_mig.go b/pkg/services/sqlstore/migrations/cache_data_mig.go index 1201b38e337..f12f7f797c8 100644 --- a/pkg/services/sqlstore/migrations/cache_data_mig.go +++ b/pkg/services/sqlstore/migrations/cache_data_mig.go @@ -1,17 +1,22 @@ package migrations -import . "github.com/grafana/grafana/pkg/services/sqlstore/migrator" +import "github.com/grafana/grafana/pkg/services/sqlstore/migrator" -func addCacheMigration(mg *Migrator) { - var cacheDataV1 = Table{ +func addCacheMigration(mg *migrator.Migrator) { + var cacheDataV1 = migrator.Table{ Name: "cache_data", - Columns: []*Column{ - {Name: "key", Type: DB_Char, IsPrimaryKey: true, Length: 16}, - {Name: "data", Type: DB_Blob}, - {Name: "expires", Type: DB_Integer, Length: 255, Nullable: false}, - {Name: "created_at", Type: DB_Integer, Length: 255, Nullable: false}, + Columns: []*migrator.Column{ + {Name: "key", Type: migrator.DB_NVarchar, IsPrimaryKey: true, Length: 168}, + {Name: "data", Type: migrator.DB_Blob}, + {Name: "expires", Type: migrator.DB_Integer, Length: 255, Nullable: false}, + {Name: "created_at", Type: migrator.DB_Integer, Length: 255, Nullable: false}, + }, + Indices: []*migrator.Index{ + {Cols: []string{"key"}, Type: migrator.UniqueIndex}, }, } - mg.AddMigration("create cache_data table", NewAddTableMigration(cacheDataV1)) + mg.AddMigration("create cache_data table", migrator.NewAddTableMigration(cacheDataV1)) + + mg.AddMigration("add unique index cache_data.key", migrator.NewAddIndexMigration(cacheDataV1, cacheDataV1.Indices[0])) } From 5ced863f7527a1eb366ff8d63df7ff78e6e4b51f Mon Sep 17 00:00:00 2001 From: bergquist Date: Sat, 23 Feb 2019 16:12:37 +0100 Subject: [PATCH 04/75] add support for redis storage --- package.json | 5 -- pkg/infra/distcache/database_storage.go | 13 +++- pkg/infra/distcache/database_storage_test.go | 8 +- pkg/infra/distcache/distcache.go | 7 +- pkg/infra/distcache/distcache_test.go | 20 +++-- pkg/infra/distcache/redis_storage.go | 80 ++++++++++++++++++++ pkg/infra/distcache/redis_storage_test.go | 1 + 7 files changed, 110 insertions(+), 24 deletions(-) create mode 100644 pkg/infra/distcache/redis_storage.go create mode 100644 pkg/infra/distcache/redis_storage_test.go diff --git a/package.json b/package.json index a937ba6f717..af270d47ad0 100644 --- a/package.json +++ b/package.json @@ -142,11 +142,6 @@ "gui:release": "ts-node --project ./scripts/cli/tsconfig.json ./scripts/cli/index.ts gui:release -p", "cli": "ts-node --project ./scripts/cli/tsconfig.json ./scripts/cli/index.ts" }, - "husky": { - "hooks": { - "pre-commit": "lint-staged && grunt precommit" - } - }, "lint-staged": { "*.{ts,tsx,json,scss}": [ "prettier --write", diff --git a/pkg/infra/distcache/database_storage.go b/pkg/infra/distcache/database_storage.go index ed55208e18d..cff5e0fc499 100644 --- a/pkg/infra/distcache/database_storage.go +++ b/pkg/infra/distcache/database_storage.go @@ -18,7 +18,7 @@ func newDatabaseCache(sqlstore *sqlstore.SqlStore) *databaseCache { log: log.New("distcache.database"), } - go dc.StartGC() + //go dc.StartGC() //TODO: start the GC somehow return dc } @@ -79,7 +79,7 @@ type CacheData struct { CreatedAt int64 } -func (dc *databaseCache) Put(key string, value interface{}, expire int64) error { +func (dc *databaseCache) Put(key string, value interface{}, expire time.Duration) error { item := &Item{Val: value} data, err := EncodeGob(item) if err != nil { @@ -94,10 +94,15 @@ func (dc *databaseCache) Put(key string, value interface{}, expire int64) error return err } + var expiresInEpoch int64 + if expire != 0 { + expiresInEpoch = int64(expire) / int64(time.Second) + } + if len(cacheHits) > 0 { - _, err = dc.SQLStore.NewSession().Exec("UPDATE cache_data SET data=?, created=?, expire=? WHERE key=?", data, now, expire, key) + _, err = dc.SQLStore.NewSession().Exec("UPDATE cache_data SET data=?, created=?, expire=? WHERE key=?", data, now, expiresInEpoch, key) } else { - _, err = dc.SQLStore.NewSession().Exec("INSERT INTO cache_data(key,data,created_at,expires) VALUES(?,?,?,?)", key, data, now, expire) + _, err = dc.SQLStore.NewSession().Exec("INSERT INTO cache_data(key,data,created_at,expires) VALUES(?,?,?,?)", key, data, now, expiresInEpoch) } return err diff --git a/pkg/infra/distcache/database_storage_test.go b/pkg/infra/distcache/database_storage_test.go index 2e6339c7c32..931fbc81c7f 100644 --- a/pkg/infra/distcache/database_storage_test.go +++ b/pkg/infra/distcache/database_storage_test.go @@ -22,15 +22,15 @@ func TestDatabaseStorageGarbageCollection(t *testing.T) { //set time.now to 2 weeks ago getTime = func() time.Time { return time.Now().AddDate(0, 0, -2) } - db.Put("key1", obj, 1000) - db.Put("key2", obj, 1000) - db.Put("key3", obj, 1000) + db.Put("key1", obj, 1000*time.Second) + db.Put("key2", obj, 1000*time.Second) + db.Put("key3", obj, 1000*time.Second) // insert object that should never expire db.Put("key4", obj, 0) getTime = time.Now - db.Put("key5", obj, 1000) + db.Put("key5", obj, 1000*time.Second) //run GC db.internalRunGC() diff --git a/pkg/infra/distcache/distcache.go b/pkg/infra/distcache/distcache.go index 3a2d553953a..a60b3d309c2 100644 --- a/pkg/infra/distcache/distcache.go +++ b/pkg/infra/distcache/distcache.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/gob" "errors" + "time" "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/services/sqlstore" @@ -34,7 +35,7 @@ type CacheOpts struct { func createClient(opts CacheOpts, sqlstore *sqlstore.SqlStore) cacheStorage { if opts.name == "redis" { - return nil + return newRedisStorage(nil) } if opts.name == "memcache" { @@ -45,7 +46,7 @@ func createClient(opts CacheOpts, sqlstore *sqlstore.SqlStore) cacheStorage { return nil } - return &databaseCache{SQLStore: sqlstore} + return newDatabaseCache(sqlstore) //&databaseCache{SQLStore: sqlstore} } // DistributedCache allows Grafana to cache data outside its own process @@ -77,7 +78,7 @@ type cacheStorage interface { Get(key string) (interface{}, error) // Puts an object into the cache - Put(key string, value interface{}, expire int64) error + Put(key string, value interface{}, expire time.Duration) error // Delete object from cache Delete(key string) error diff --git a/pkg/infra/distcache/distcache_test.go b/pkg/infra/distcache/distcache_test.go index d3009753a14..dd35744506c 100644 --- a/pkg/infra/distcache/distcache_test.go +++ b/pkg/infra/distcache/distcache_test.go @@ -27,18 +27,18 @@ func createTestClient(t *testing.T, name string) cacheStorage { } func TestAllCacheClients(t *testing.T) { - clients := []string{"database"} // add redis, memcache, memory + clients := []string{"database", "redis"} // add redis, memcache, memory for _, v := range clients { client := createTestClient(t, v) - CanPutGetAndDeleteCachedObjects(t, client) - CanNotFetchExpiredItems(t, client) - CanSetInfiniteCacheExpiration(t, client) + CanPutGetAndDeleteCachedObjects(t, v, client) + CanNotFetchExpiredItems(t, v, client) + CanSetInfiniteCacheExpiration(t, v, client) } } -func CanPutGetAndDeleteCachedObjects(t *testing.T, client cacheStorage) { +func CanPutGetAndDeleteCachedObjects(t *testing.T, name string, client cacheStorage) { cacheableStruct := CacheableStruct{String: "hej", Int64: 2000} err := client.Put("key", cacheableStruct, 0) @@ -58,12 +58,16 @@ func CanPutGetAndDeleteCachedObjects(t *testing.T, client cacheStorage) { assert.Equal(t, err, ErrCacheItemNotFound) } -func CanNotFetchExpiredItems(t *testing.T, client cacheStorage) { +func CanNotFetchExpiredItems(t *testing.T, name string, client cacheStorage) { + if name == "redis" { + t.Skip() //this test does not work with redis since it uses its own getTime fn + } + cacheableStruct := CacheableStruct{String: "hej", Int64: 2000} // insert cache item one day back getTime = func() time.Time { return time.Now().AddDate(0, 0, -2) } - err := client.Put("key", cacheableStruct, 10000) + err := client.Put("key", cacheableStruct, 10000*time.Second) assert.Equal(t, err, nil) // should not be able to read that value since its expired @@ -72,7 +76,7 @@ func CanNotFetchExpiredItems(t *testing.T, client cacheStorage) { assert.Equal(t, err, ErrCacheItemNotFound) } -func CanSetInfiniteCacheExpiration(t *testing.T, client cacheStorage) { +func CanSetInfiniteCacheExpiration(t *testing.T, name string, client cacheStorage) { cacheableStruct := CacheableStruct{String: "hej", Int64: 2000} // insert cache item one day back diff --git a/pkg/infra/distcache/redis_storage.go b/pkg/infra/distcache/redis_storage.go new file mode 100644 index 00000000000..06fc6931758 --- /dev/null +++ b/pkg/infra/distcache/redis_storage.go @@ -0,0 +1,80 @@ +package distcache + +import ( + "time" + + redis "gopkg.in/redis.v2" +) + +type redisStorage struct { + c *redis.Client +} + +func newRedisStorage(c *redis.Client) *redisStorage { + opt := &redis.Options{ + Network: "tcp", + Addr: "localhost:6379", + } + return &redisStorage{ + c: redis.NewClient(opt), + } +} + +// Set sets value to given key in session. +func (s *redisStorage) Put(key string, val interface{}, expires time.Duration) error { + item := &Item{Created: getTime().Unix(), Val: val} + value, err := EncodeGob(item) + if err != nil { + return err + } + + var status *redis.StatusCmd + if expires == 0 { + status = s.c.Set(key, string(value)) + } else { + status = s.c.SetEx(key, expires, string(value)) + } + + return status.Err() +} + +// Get gets value by given key in session. +func (s *redisStorage) Get(key string) (interface{}, error) { + v := s.c.Get(key) + + item := &Item{} + err := DecodeGob([]byte(v.Val()), item) + + if err == nil { + return item.Val, nil + } + + if err.Error() == "EOF" { + return nil, ErrCacheItemNotFound + } + + if err != nil { + return nil, err + } + + return item.Val, nil +} + +// Delete delete a key from session. +func (s *redisStorage) Delete(key string) error { + cmd := s.c.Del(key) + return cmd.Err() +} + +// RedisProvider represents a redis session provider implementation. +type RedisProvider struct { + c *redis.Client + duration time.Duration + prefix string +} + +// Exist returns true if session with given ID exists. +func (p *RedisProvider) Exist(sid string) bool { + has, err := p.c.Exists(p.prefix + sid).Result() + return err == nil && has +} diff --git a/pkg/infra/distcache/redis_storage_test.go b/pkg/infra/distcache/redis_storage_test.go new file mode 100644 index 00000000000..e793fbec4c4 --- /dev/null +++ b/pkg/infra/distcache/redis_storage_test.go @@ -0,0 +1 @@ +package distcache From 11d671c637fea63b74d9082a907b0a97f424e6e0 Mon Sep 17 00:00:00 2001 From: bergquist Date: Sat, 23 Feb 2019 18:28:33 +0100 Subject: [PATCH 05/75] add support for memcached --- pkg/infra/distcache/distcache.go | 8 +-- pkg/infra/distcache/distcache_test.go | 2 +- pkg/infra/distcache/memcached_storage.go | 62 ++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 5 deletions(-) create mode 100644 pkg/infra/distcache/memcached_storage.go diff --git a/pkg/infra/distcache/distcache.go b/pkg/infra/distcache/distcache.go index a60b3d309c2..8a6f7daf90c 100644 --- a/pkg/infra/distcache/distcache.go +++ b/pkg/infra/distcache/distcache.go @@ -39,12 +39,12 @@ func createClient(opts CacheOpts, sqlstore *sqlstore.SqlStore) cacheStorage { } if opts.name == "memcache" { - return nil + return newMemcacheStorage("localhost:9090") } - if opts.name == "memory" { - return nil - } + // if opts.name == "memory" { + // return nil + // } return newDatabaseCache(sqlstore) //&databaseCache{SQLStore: sqlstore} } diff --git a/pkg/infra/distcache/distcache_test.go b/pkg/infra/distcache/distcache_test.go index dd35744506c..a04b5d0228f 100644 --- a/pkg/infra/distcache/distcache_test.go +++ b/pkg/infra/distcache/distcache_test.go @@ -27,7 +27,7 @@ func createTestClient(t *testing.T, name string) cacheStorage { } func TestAllCacheClients(t *testing.T) { - clients := []string{"database", "redis"} // add redis, memcache, memory + clients := []string{"database", "redis", "memcached"} // add redis, memcache, memory for _, v := range clients { client := createTestClient(t, v) diff --git a/pkg/infra/distcache/memcached_storage.go b/pkg/infra/distcache/memcached_storage.go new file mode 100644 index 00000000000..44fbbcc33c6 --- /dev/null +++ b/pkg/infra/distcache/memcached_storage.go @@ -0,0 +1,62 @@ +package distcache + +import ( + "time" + + "github.com/bradfitz/gomemcache/memcache" +) + +type memcacheStorage struct { + c *memcache.Client +} + +func newMemcacheStorage(connStr string) *memcacheStorage { + return &memcacheStorage{ + c: memcache.New(connStr), + } +} + +func NewItem(sid string, data []byte, expire int32) *memcache.Item { + return &memcache.Item{ + Key: sid, + Value: data, + Expiration: expire, + } +} + +// Set sets value to given key in the cache. +func (s *memcacheStorage) Put(key string, val interface{}, expires time.Duration) error { + item := &Item{Val: val} + + bytes, err := EncodeGob(item) + if err != nil { + return err + } + + memcacheItem := NewItem(key, bytes, int32(expires)) + + s.c.Add(memcacheItem) + return nil +} + +// Get gets value by given key in the cache. +func (s *memcacheStorage) Get(key string) (interface{}, error) { + i, err := s.c.Get(key) + if err != nil { + return nil, err + } + + item := &Item{} + + err = DecodeGob(i.Value, item) + if err != nil { + return nil, err + } + + return item.Val, nil +} + +// Delete delete a key from the cache +func (s *memcacheStorage) Delete(key string) error { + return s.c.Delete(key) +} From 3890bd14ebebe9518c33d99727b7267f33e3765b Mon Sep 17 00:00:00 2001 From: bergquist Date: Sat, 23 Feb 2019 22:59:12 +0100 Subject: [PATCH 06/75] fixes typo in redis devenv --- devenv/docker/blocks/redis/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devenv/docker/blocks/redis/docker-compose.yaml b/devenv/docker/blocks/redis/docker-compose.yaml index 65071d4966b..fb56afaac1c 100644 --- a/devenv/docker/blocks/redis/docker-compose.yaml +++ b/devenv/docker/blocks/redis/docker-compose.yaml @@ -1,4 +1,4 @@ - memcached: + redis: image: redis:latest ports: - "6379:6379" From 8029e48588215b5ec4a54c60866ba994bf036cf2 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Fri, 8 Mar 2019 15:15:17 +0100 Subject: [PATCH 07/75] support get user tokens/revoke all user tokens in UserTokenService --- pkg/middleware/middleware_test.go | 67 +++-------------------- pkg/middleware/org_redirect_test.go | 4 +- pkg/middleware/quota_test.go | 5 +- pkg/middleware/recovery_test.go | 3 +- pkg/models/user_token.go | 3 ++ pkg/services/auth/auth_token.go | 51 ++++++++++++++++++ pkg/services/auth/auth_token_test.go | 41 ++++++++++++++ pkg/services/auth/testing.go | 81 ++++++++++++++++++++++++++++ 8 files changed, 190 insertions(+), 65 deletions(-) create mode 100644 pkg/services/auth/testing.go diff --git a/pkg/middleware/middleware_test.go b/pkg/middleware/middleware_test.go index 1fbd303bebd..2fc8e0c456f 100644 --- a/pkg/middleware/middleware_test.go +++ b/pkg/middleware/middleware_test.go @@ -11,6 +11,7 @@ import ( msession "github.com/go-macaron/session" "github.com/grafana/grafana/pkg/bus" m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/auth" "github.com/grafana/grafana/pkg/services/session" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util" @@ -155,7 +156,7 @@ func TestMiddlewareContext(t *testing.T) { return nil }) - sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (*m.UserToken, error) { + sc.userAuthTokenService.LookupTokenProvider = func(unhashedToken string) (*m.UserToken, error) { return &m.UserToken{ UserId: 12, UnhashedToken: unhashedToken, @@ -184,14 +185,14 @@ func TestMiddlewareContext(t *testing.T) { return nil }) - sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (*m.UserToken, error) { + sc.userAuthTokenService.LookupTokenProvider = func(unhashedToken string) (*m.UserToken, error) { return &m.UserToken{ UserId: 12, UnhashedToken: "", }, nil } - sc.userAuthTokenService.tryRotateTokenProvider = func(userToken *m.UserToken, clientIP, userAgent string) (bool, error) { + sc.userAuthTokenService.TryRotateTokenProvider = func(userToken *m.UserToken, clientIP, userAgent string) (bool, error) { userToken.UnhashedToken = "rotated" return true, nil } @@ -226,7 +227,7 @@ func TestMiddlewareContext(t *testing.T) { middlewareScenario("Invalid/expired auth token in cookie", func(sc *scenarioContext) { sc.withTokenSessionCookie("token") - sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (*m.UserToken, error) { + sc.userAuthTokenService.LookupTokenProvider = func(unhashedToken string) (*m.UserToken, error) { return nil, m.ErrUserTokenNotFound } @@ -562,7 +563,7 @@ func middlewareScenario(desc string, fn scenarioFunc) { })) session.Init(&msession.Options{}, 0) - sc.userAuthTokenService = newFakeUserAuthTokenService() + sc.userAuthTokenService = auth.NewFakeUserAuthTokenService() sc.m.Use(GetContextHandler(sc.userAuthTokenService)) // mock out gc goroutine session.StartSessionGC = func() {} @@ -595,7 +596,7 @@ type scenarioContext struct { handlerFunc handlerFunc defaultHandler macaron.Handler url string - userAuthTokenService *fakeUserAuthTokenService + userAuthTokenService *auth.FakeUserAuthTokenService req *http.Request } @@ -676,57 +677,3 @@ func (sc *scenarioContext) exec() { type scenarioFunc func(c *scenarioContext) type handlerFunc func(c *m.ReqContext) - -type fakeUserAuthTokenService struct { - createTokenProvider func(userId int64, clientIP, userAgent string) (*m.UserToken, error) - tryRotateTokenProvider func(token *m.UserToken, clientIP, userAgent string) (bool, error) - lookupTokenProvider func(unhashedToken string) (*m.UserToken, error) - revokeTokenProvider func(token *m.UserToken) error - activeAuthTokenCount func() (int64, error) -} - -func newFakeUserAuthTokenService() *fakeUserAuthTokenService { - return &fakeUserAuthTokenService{ - createTokenProvider: func(userId int64, clientIP, userAgent string) (*m.UserToken, error) { - return &m.UserToken{ - UserId: 0, - UnhashedToken: "", - }, nil - }, - tryRotateTokenProvider: func(token *m.UserToken, clientIP, userAgent string) (bool, error) { - return false, nil - }, - lookupTokenProvider: func(unhashedToken string) (*m.UserToken, error) { - return &m.UserToken{ - UserId: 0, - UnhashedToken: "", - }, nil - }, - revokeTokenProvider: func(token *m.UserToken) error { - return nil - }, - activeAuthTokenCount: func() (int64, error) { - return 10, nil - }, - } -} - -func (s *fakeUserAuthTokenService) CreateToken(userId int64, clientIP, userAgent string) (*m.UserToken, error) { - return s.createTokenProvider(userId, clientIP, userAgent) -} - -func (s *fakeUserAuthTokenService) LookupToken(unhashedToken string) (*m.UserToken, error) { - return s.lookupTokenProvider(unhashedToken) -} - -func (s *fakeUserAuthTokenService) TryRotateToken(token *m.UserToken, clientIP, userAgent string) (bool, error) { - return s.tryRotateTokenProvider(token, clientIP, userAgent) -} - -func (s *fakeUserAuthTokenService) RevokeToken(token *m.UserToken) error { - return s.revokeTokenProvider(token) -} - -func (s *fakeUserAuthTokenService) ActiveTokenCount() (int64, error) { - return s.activeAuthTokenCount() -} diff --git a/pkg/middleware/org_redirect_test.go b/pkg/middleware/org_redirect_test.go index e01d1a68d21..fe5b2736035 100644 --- a/pkg/middleware/org_redirect_test.go +++ b/pkg/middleware/org_redirect_test.go @@ -24,7 +24,7 @@ func TestOrgRedirectMiddleware(t *testing.T) { return nil }) - sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (*m.UserToken, error) { + sc.userAuthTokenService.LookupTokenProvider = func(unhashedToken string) (*m.UserToken, error) { return &m.UserToken{ UserId: 0, UnhashedToken: "", @@ -50,7 +50,7 @@ func TestOrgRedirectMiddleware(t *testing.T) { return nil }) - sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (*m.UserToken, error) { + sc.userAuthTokenService.LookupTokenProvider = func(unhashedToken string) (*m.UserToken, error) { return &m.UserToken{ UserId: 12, UnhashedToken: "", diff --git a/pkg/middleware/quota_test.go b/pkg/middleware/quota_test.go index 52b696cf037..0ba42e708bc 100644 --- a/pkg/middleware/quota_test.go +++ b/pkg/middleware/quota_test.go @@ -3,6 +3,7 @@ package middleware import ( "testing" + "github.com/grafana/grafana/pkg/services/auth" "github.com/grafana/grafana/pkg/services/quota" "github.com/grafana/grafana/pkg/bus" @@ -36,7 +37,7 @@ func TestMiddlewareQuota(t *testing.T) { }, } - fakeAuthTokenService := newFakeUserAuthTokenService() + fakeAuthTokenService := auth.NewFakeUserAuthTokenService() qs := "a.QuotaService{ AuthTokenService: fakeAuthTokenService, } @@ -87,7 +88,7 @@ func TestMiddlewareQuota(t *testing.T) { return nil }) - sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (*m.UserToken, error) { + sc.userAuthTokenService.LookupTokenProvider = func(unhashedToken string) (*m.UserToken, error) { return &m.UserToken{ UserId: 12, UnhashedToken: "", diff --git a/pkg/middleware/recovery_test.go b/pkg/middleware/recovery_test.go index 6736d699a39..00f3b7a3032 100644 --- a/pkg/middleware/recovery_test.go +++ b/pkg/middleware/recovery_test.go @@ -6,6 +6,7 @@ import ( "github.com/grafana/grafana/pkg/bus" m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/auth" "github.com/grafana/grafana/pkg/setting" . "github.com/smartystreets/goconvey/convey" macaron "gopkg.in/macaron.v1" @@ -62,7 +63,7 @@ func recoveryScenario(desc string, url string, fn scenarioFunc) { Delims: macaron.Delims{Left: "[[", Right: "]]"}, })) - sc.userAuthTokenService = newFakeUserAuthTokenService() + sc.userAuthTokenService = auth.NewFakeUserAuthTokenService() sc.m.Use(GetContextHandler(sc.userAuthTokenService)) // mock out gc goroutine sc.m.Use(OrgRedirect()) diff --git a/pkg/models/user_token.go b/pkg/models/user_token.go index 388bc2dd4a2..22f92cb21d2 100644 --- a/pkg/models/user_token.go +++ b/pkg/models/user_token.go @@ -29,5 +29,8 @@ type UserTokenService interface { LookupToken(unhashedToken string) (*UserToken, error) TryRotateToken(token *UserToken, clientIP, userAgent string) (bool, error) RevokeToken(token *UserToken) error + RevokeAllUserTokens(userId int64) error ActiveTokenCount() (int64, error) + GetUserToken(userId, userTokenId int64) (*UserToken, error) + GetUserTokens(userId int64) ([]*UserToken, error) } diff --git a/pkg/services/auth/auth_token.go b/pkg/services/auth/auth_token.go index 648575d54cd..255866a9ba0 100644 --- a/pkg/services/auth/auth_token.go +++ b/pkg/services/auth/auth_token.go @@ -221,6 +221,57 @@ func (s *UserAuthTokenService) RevokeToken(token *models.UserToken) error { return nil } +func (s *UserAuthTokenService) RevokeAllUserTokens(userId int64) error { + sql := `DELETE from user_auth_token WHERE user_id = ?` + res, err := s.SQLStore.NewSession().Exec(sql, userId) + if err != nil { + return err + } + + affected, err := res.RowsAffected() + if err != nil { + return err + } + + s.log.Debug("all user tokens for user revoked", "userId", userId, "count", affected) + + return nil +} + +func (s *UserAuthTokenService) GetUserToken(userId, userTokenId int64) (*models.UserToken, error) { + var token userAuthToken + exists, err := s.SQLStore.NewSession().Where("id = ? AND user_id = ?", userTokenId, userId).Get(&token) + if err != nil { + return nil, err + } + + if !exists { + return nil, models.ErrUserTokenNotFound + } + + var result models.UserToken + token.toUserToken(&result) + + return &result, nil +} + +func (s *UserAuthTokenService) GetUserTokens(userId int64) ([]*models.UserToken, error) { + var tokens []*userAuthToken + err := s.SQLStore.NewSession().Where("user_id = ? AND created_at > ? AND rotated_at > ?", userId, s.createdAfterParam(), s.rotatedAfterParam()).Find(&tokens) + if err != nil { + return nil, err + } + + result := []*models.UserToken{} + for _, token := range tokens { + var userToken models.UserToken + token.toUserToken(&userToken) + result = append(result, &userToken) + } + + return result, nil +} + func (s *UserAuthTokenService) createdAfterParam() int64 { tokenMaxLifetime := time.Duration(s.Cfg.LoginMaxLifetimeDays) * 24 * time.Hour return getTime().Add(-tokenMaxLifetime).Unix() diff --git a/pkg/services/auth/auth_token_test.go b/pkg/services/auth/auth_token_test.go index 49e7acc3a5b..33eb309ad18 100644 --- a/pkg/services/auth/auth_token_test.go +++ b/pkg/services/auth/auth_token_test.go @@ -75,6 +75,47 @@ func TestUserAuthToken(t *testing.T) { err = userAuthTokenService.RevokeToken(userToken) So(err, ShouldEqual, models.ErrUserTokenNotFound) }) + + Convey("When creating an additional token", func() { + userToken2, err := userAuthTokenService.CreateToken(userID, "192.168.10.11:1234", "some user agent") + So(err, ShouldBeNil) + So(userToken2, ShouldNotBeNil) + + Convey("Can get first user token", func() { + token, err := userAuthTokenService.GetUserToken(userID, userToken.Id) + So(err, ShouldBeNil) + So(token, ShouldNotBeNil) + So(token.Id, ShouldEqual, userToken.Id) + }) + + Convey("Can get second user token", func() { + token, err := userAuthTokenService.GetUserToken(userID, userToken2.Id) + So(err, ShouldBeNil) + So(token, ShouldNotBeNil) + So(token.Id, ShouldEqual, userToken2.Id) + }) + + Convey("Can get user tokens", func() { + tokens, err := userAuthTokenService.GetUserTokens(userID) + So(err, ShouldBeNil) + So(tokens, ShouldHaveLength, 2) + So(tokens[0].Id, ShouldEqual, userToken.Id) + So(tokens[1].Id, ShouldEqual, userToken2.Id) + }) + + Convey("Can revoke all user tokens", func() { + err := userAuthTokenService.RevokeAllUserTokens(userID) + So(err, ShouldBeNil) + + model, err := ctx.getAuthTokenByID(userToken.Id) + So(err, ShouldBeNil) + So(model, ShouldBeNil) + + model2, err := ctx.getAuthTokenByID(userToken2.Id) + So(err, ShouldBeNil) + So(model2, ShouldBeNil) + }) + }) }) Convey("expires correctly", func() { diff --git a/pkg/services/auth/testing.go b/pkg/services/auth/testing.go new file mode 100644 index 00000000000..68e65466c3d --- /dev/null +++ b/pkg/services/auth/testing.go @@ -0,0 +1,81 @@ +package auth + +import "github.com/grafana/grafana/pkg/models" + +type FakeUserAuthTokenService struct { + CreateTokenProvider func(userId int64, clientIP, userAgent string) (*models.UserToken, error) + TryRotateTokenProvider func(token *models.UserToken, clientIP, userAgent string) (bool, error) + LookupTokenProvider func(unhashedToken string) (*models.UserToken, error) + RevokeTokenProvider func(token *models.UserToken) error + RevokeAllUserTokensProvider func(userId int64) error + ActiveAuthTokenCount func() (int64, error) + GetUserTokenProvider func(userId, userTokenId int64) (*models.UserToken, error) + GetUserTokensProvider func(userId int64) ([]*models.UserToken, error) +} + +func NewFakeUserAuthTokenService() *FakeUserAuthTokenService { + return &FakeUserAuthTokenService{ + CreateTokenProvider: func(userId int64, clientIP, userAgent string) (*models.UserToken, error) { + return &models.UserToken{ + UserId: 0, + UnhashedToken: "", + }, nil + }, + TryRotateTokenProvider: func(token *models.UserToken, clientIP, userAgent string) (bool, error) { + return false, nil + }, + LookupTokenProvider: func(unhashedToken string) (*models.UserToken, error) { + return &models.UserToken{ + UserId: 0, + UnhashedToken: "", + }, nil + }, + RevokeTokenProvider: func(token *models.UserToken) error { + return nil + }, + RevokeAllUserTokensProvider: func(userId int64) error { + return nil + }, + ActiveAuthTokenCount: func() (int64, error) { + return 10, nil + }, + GetUserTokenProvider: func(userId, userTokenId int64) (*models.UserToken, error) { + return nil, nil + }, + GetUserTokensProvider: func(userId int64) ([]*models.UserToken, error) { + return nil, nil + }, + } +} + +func (s *FakeUserAuthTokenService) CreateToken(userId int64, clientIP, userAgent string) (*models.UserToken, error) { + return s.CreateTokenProvider(userId, clientIP, userAgent) +} + +func (s *FakeUserAuthTokenService) LookupToken(unhashedToken string) (*models.UserToken, error) { + return s.LookupTokenProvider(unhashedToken) +} + +func (s *FakeUserAuthTokenService) TryRotateToken(token *models.UserToken, clientIP, userAgent string) (bool, error) { + return s.TryRotateTokenProvider(token, clientIP, userAgent) +} + +func (s *FakeUserAuthTokenService) RevokeToken(token *models.UserToken) error { + return s.RevokeTokenProvider(token) +} + +func (s *FakeUserAuthTokenService) RevokeAllUserTokens(userId int64) error { + return s.RevokeAllUserTokensProvider(userId) +} + +func (s *FakeUserAuthTokenService) ActiveTokenCount() (int64, error) { + return s.ActiveAuthTokenCount() +} + +func (s *FakeUserAuthTokenService) GetUserToken(userId, userTokenId int64) (*models.UserToken, error) { + return s.GetUserTokenProvider(userId, userTokenId) +} + +func (s *FakeUserAuthTokenService) GetUserTokens(userId int64) ([]*models.UserToken, error) { + return s.GetUserTokensProvider(userId) +} From 0cd5a6772d188fa5bf1ada0c1cb0e7597f3579ac Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Fri, 8 Mar 2019 15:15:38 +0100 Subject: [PATCH 08/75] feat(api): support list/revoke auth token in admin/current user api --- pkg/api/admin_users.go | 23 +++ pkg/api/admin_users_test.go | 138 +++++++++++++++++ pkg/api/api.go | 7 + pkg/api/common_test.go | 16 +- pkg/api/dtos/user_token.go | 12 ++ pkg/api/user_token.go | 110 ++++++++++++++ pkg/api/user_token_test.go | 294 ++++++++++++++++++++++++++++++++++++ pkg/models/user_token.go | 8 +- 8 files changed, 600 insertions(+), 8 deletions(-) create mode 100644 pkg/api/dtos/user_token.go create mode 100644 pkg/api/user_token.go create mode 100644 pkg/api/user_token_test.go diff --git a/pkg/api/admin_users.go b/pkg/api/admin_users.go index c16c2f126f8..4ad8a2b84ab 100644 --- a/pkg/api/admin_users.go +++ b/pkg/api/admin_users.go @@ -110,3 +110,26 @@ func AdminDeleteUser(c *m.ReqContext) { c.JsonOK("User deleted") } + +// POST /api/admin/users/:id/logout +func (server *HTTPServer) AdminLogoutUser(c *m.ReqContext) Response { + userID := c.ParamsInt64(":id") + + if c.UserId == userID { + return Error(400, "You cannot logout yourself", nil) + } + + return server.logoutUserFromAllDevicesInternal(userID) +} + +// GET /api/admin/users/:id/auth-tokens +func (server *HTTPServer) AdminGetUserAuthTokens(c *m.ReqContext) Response { + userID := c.ParamsInt64(":id") + return server.getUserAuthTokensInternal(c, userID) +} + +// POST /api/admin/users/:id/revoke-auth-token +func (server *HTTPServer) AdminRevokeUserAuthToken(c *m.ReqContext, cmd m.RevokeAuthTokenCmd) Response { + userID := c.ParamsInt64(":id") + return server.revokeUserAuthTokenInternal(c, userID, cmd) +} diff --git a/pkg/api/admin_users_test.go b/pkg/api/admin_users_test.go index 0b94a64b3fb..b94f09b0b75 100644 --- a/pkg/api/admin_users_test.go +++ b/pkg/api/admin_users_test.go @@ -6,6 +6,7 @@ import ( "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/bus" m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/auth" . "github.com/smartystreets/goconvey/convey" ) @@ -27,6 +28,62 @@ func TestAdminApiEndpoint(t *testing.T) { So(sc.resp.Code, ShouldEqual, 400) }) }) + + Convey("When a server admin attempts to logout himself from all devices", t, func() { + bus.AddHandler("test", func(cmd *m.GetUserByIdQuery) error { + cmd.Result = &m.User{Id: TestUserID} + return nil + }) + + adminLogoutUserScenario("Should not be allowed when calling POST on", "/api/admin/users/1/logout", "/api/admin/users/:id/logout", func(sc *scenarioContext) { + sc.fakeReqWithParams("POST", sc.url, map[string]string{}).exec() + So(sc.resp.Code, ShouldEqual, 400) + }) + }) + + Convey("When a server admin attempts to logout a non-existing user from all devices", t, func() { + userId := int64(0) + bus.AddHandler("test", func(cmd *m.GetUserByIdQuery) error { + userId = cmd.Id + return m.ErrUserNotFound + }) + + adminLogoutUserScenario("Should return not found when calling POST on", "/api/admin/users/200/logout", "/api/admin/users/:id/logout", func(sc *scenarioContext) { + sc.fakeReqWithParams("POST", sc.url, map[string]string{}).exec() + So(sc.resp.Code, ShouldEqual, 404) + So(userId, ShouldEqual, 200) + }) + }) + + Convey("When a server admin attempts to revoke an auth token for a non-existing user", t, func() { + userId := int64(0) + bus.AddHandler("test", func(cmd *m.GetUserByIdQuery) error { + userId = cmd.Id + return m.ErrUserNotFound + }) + + cmd := m.RevokeAuthTokenCmd{AuthTokenId: 2} + + adminRevokeUserAuthTokenScenario("Should return not found when calling POST on", "/api/admin/users/200/revoke-auth-token", "/api/admin/users/:id/revoke-auth-token", cmd, func(sc *scenarioContext) { + sc.fakeReqWithParams("POST", sc.url, map[string]string{}).exec() + So(sc.resp.Code, ShouldEqual, 404) + So(userId, ShouldEqual, 200) + }) + }) + + Convey("When a server admin gets auth tokens for a non-existing user", t, func() { + userId := int64(0) + bus.AddHandler("test", func(cmd *m.GetUserByIdQuery) error { + userId = cmd.Id + return m.ErrUserNotFound + }) + + adminGetUserAuthTokensScenario("Should return not found when calling GET on", "/api/admin/users/200/auth-tokens", "/api/admin/users/:id/auth-tokens", func(sc *scenarioContext) { + sc.fakeReqWithParams("GET", sc.url, map[string]string{}).exec() + So(sc.resp.Code, ShouldEqual, 404) + So(userId, ShouldEqual, 200) + }) + }) } func putAdminScenario(desc string, url string, routePattern string, role m.RoleType, cmd dtos.AdminUpdateUserPermissionsForm, fn scenarioFunc) { @@ -48,3 +105,84 @@ func putAdminScenario(desc string, url string, routePattern string, role m.RoleT fn(sc) }) } + +func adminLogoutUserScenario(desc string, url string, routePattern string, fn scenarioFunc) { + Convey(desc+" "+url, func() { + defer bus.ClearBusHandlers() + + hs := HTTPServer{ + Bus: bus.GetBus(), + AuthTokenService: auth.NewFakeUserAuthTokenService(), + } + + sc := setupScenarioContext(url) + sc.defaultHandler = Wrap(func(c *m.ReqContext) Response { + sc.context = c + sc.context.UserId = TestUserID + sc.context.OrgId = TestOrgID + sc.context.OrgRole = m.ROLE_ADMIN + + return hs.AdminLogoutUser(c) + }) + + sc.m.Post(routePattern, sc.defaultHandler) + + fn(sc) + }) +} + +func adminRevokeUserAuthTokenScenario(desc string, url string, routePattern string, cmd m.RevokeAuthTokenCmd, fn scenarioFunc) { + Convey(desc+" "+url, func() { + defer bus.ClearBusHandlers() + + fakeAuthTokenService := auth.NewFakeUserAuthTokenService() + + hs := HTTPServer{ + Bus: bus.GetBus(), + AuthTokenService: fakeAuthTokenService, + } + + sc := setupScenarioContext(url) + sc.userAuthTokenService = fakeAuthTokenService + sc.defaultHandler = Wrap(func(c *m.ReqContext) Response { + sc.context = c + sc.context.UserId = TestUserID + sc.context.OrgId = TestOrgID + sc.context.OrgRole = m.ROLE_ADMIN + + return hs.AdminRevokeUserAuthToken(c, cmd) + }) + + sc.m.Post(routePattern, sc.defaultHandler) + + fn(sc) + }) +} + +func adminGetUserAuthTokensScenario(desc string, url string, routePattern string, fn scenarioFunc) { + Convey(desc+" "+url, func() { + defer bus.ClearBusHandlers() + + fakeAuthTokenService := auth.NewFakeUserAuthTokenService() + + hs := HTTPServer{ + Bus: bus.GetBus(), + AuthTokenService: fakeAuthTokenService, + } + + sc := setupScenarioContext(url) + sc.userAuthTokenService = fakeAuthTokenService + sc.defaultHandler = Wrap(func(c *m.ReqContext) Response { + sc.context = c + sc.context.UserId = TestUserID + sc.context.OrgId = TestOrgID + sc.context.OrgRole = m.ROLE_ADMIN + + return hs.AdminGetUserAuthTokens(c) + }) + + sc.m.Get(routePattern, sc.defaultHandler) + + fn(sc) + }) +} diff --git a/pkg/api/api.go b/pkg/api/api.go index 81ea83eae61..f3dc35b6b06 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -133,6 +133,9 @@ func (hs *HTTPServer) registerRoutes() { userRoute.Get("/preferences", Wrap(GetUserPreferences)) userRoute.Put("/preferences", bind(dtos.UpdatePrefsCmd{}), Wrap(UpdateUserPreferences)) + + userRoute.Get("/auth-tokens", Wrap(hs.GetUserAuthTokens)) + userRoute.Post("/revoke-auth-token", bind(m.RevokeAuthTokenCmd{}), Wrap(hs.RevokeUserAuthToken)) }) // users (admin permission required) @@ -375,6 +378,10 @@ func (hs *HTTPServer) registerRoutes() { adminRoute.Put("/users/:id/quotas/:target", bind(m.UpdateUserQuotaCmd{}), Wrap(UpdateUserQuota)) adminRoute.Get("/stats", AdminGetStats) adminRoute.Post("/pause-all-alerts", bind(dtos.PauseAllAlertsCommand{}), Wrap(PauseAllAlerts)) + + adminRoute.Post("/users/:id/logout", Wrap(hs.AdminLogoutUser)) + adminRoute.Get("/users/:id/auth-tokens", Wrap(hs.AdminGetUserAuthTokens)) + adminRoute.Post("/users/:id/revoke-auth-token", bind(m.RevokeAuthTokenCmd{}), Wrap(hs.AdminRevokeUserAuthToken)) }, reqGrafanaAdmin) // rendering diff --git a/pkg/api/common_test.go b/pkg/api/common_test.go index 3f3a50aae69..4e0b0dcd998 100644 --- a/pkg/api/common_test.go +++ b/pkg/api/common_test.go @@ -8,6 +8,7 @@ import ( "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/middleware" m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/auth" "gopkg.in/macaron.v1" . "github.com/smartystreets/goconvey/convey" @@ -94,13 +95,14 @@ func (sc *scenarioContext) fakeReqWithParams(method, url string, queryParams map } type scenarioContext struct { - m *macaron.Macaron - context *m.ReqContext - resp *httptest.ResponseRecorder - handlerFunc handlerFunc - defaultHandler macaron.Handler - req *http.Request - url string + m *macaron.Macaron + context *m.ReqContext + resp *httptest.ResponseRecorder + handlerFunc handlerFunc + defaultHandler macaron.Handler + req *http.Request + url string + userAuthTokenService *auth.FakeUserAuthTokenService } func (sc *scenarioContext) exec() { diff --git a/pkg/api/dtos/user_token.go b/pkg/api/dtos/user_token.go new file mode 100644 index 00000000000..1542421e2f6 --- /dev/null +++ b/pkg/api/dtos/user_token.go @@ -0,0 +1,12 @@ +package dtos + +import "time" + +type UserToken struct { + Id int64 `json:"id"` + IsActive bool `json:"isActive"` + ClientIp string `json:"clientIp"` + UserAgent string `json:"userAgent"` + CreatedAt time.Time `json:"createdAt"` + SeenAt time.Time `json:"seenAt"` +} diff --git a/pkg/api/user_token.go b/pkg/api/user_token.go new file mode 100644 index 00000000000..2f74eedea5d --- /dev/null +++ b/pkg/api/user_token.go @@ -0,0 +1,110 @@ +package api + +import ( + "time" + + "github.com/grafana/grafana/pkg/api/dtos" + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/util" +) + +// GET /api/user/auth-tokens +func (server *HTTPServer) GetUserAuthTokens(c *models.ReqContext) Response { + return server.getUserAuthTokensInternal(c, c.UserId) +} + +// POST /api/user/revoke-auth-token +func (server *HTTPServer) RevokeUserAuthToken(c *models.ReqContext, cmd models.RevokeAuthTokenCmd) Response { + return server.revokeUserAuthTokenInternal(c, c.UserId, cmd) +} + +func (server *HTTPServer) logoutUserFromAllDevicesInternal(userID int64) Response { + userQuery := models.GetUserByIdQuery{Id: userID} + + if err := bus.Dispatch(&userQuery); err != nil { + if err == models.ErrUserNotFound { + return Error(404, "User not found", err) + } + return Error(500, "Could not read user from database", err) + } + + err := server.AuthTokenService.RevokeAllUserTokens(userID) + if err != nil { + return Error(500, "Failed to logout user", err) + } + + return JSON(200, util.DynMap{ + "message": "User logged out", + }) +} + +func (server *HTTPServer) getUserAuthTokensInternal(c *models.ReqContext, userID int64) Response { + userQuery := models.GetUserByIdQuery{Id: userID} + + if err := bus.Dispatch(&userQuery); err != nil { + if err == models.ErrUserNotFound { + return Error(404, "User not found", err) + } + return Error(500, "Failed to get user", err) + } + + tokens, err := server.AuthTokenService.GetUserTokens(userID) + if err != nil { + return Error(500, "Failed to get user auth tokens", err) + } + + result := []*dtos.UserToken{} + for _, token := range tokens { + isActive := false + if c.UserToken != nil && c.UserToken.Id == token.Id { + isActive = true + } + + result = append(result, &dtos.UserToken{ + Id: token.Id, + IsActive: isActive, + ClientIp: token.ClientIp, + UserAgent: token.UserAgent, + CreatedAt: time.Unix(token.CreatedAt, 0), + SeenAt: time.Unix(token.SeenAt, 0), + }) + } + + return JSON(200, result) +} + +func (server *HTTPServer) revokeUserAuthTokenInternal(c *models.ReqContext, userID int64, cmd models.RevokeAuthTokenCmd) Response { + userQuery := models.GetUserByIdQuery{Id: userID} + + if err := bus.Dispatch(&userQuery); err != nil { + if err == models.ErrUserNotFound { + return Error(404, "User not found", err) + } + return Error(500, "Failed to get user", err) + } + + token, err := server.AuthTokenService.GetUserToken(userID, cmd.AuthTokenId) + if err != nil { + if err == models.ErrUserTokenNotFound { + return Error(404, "User auth token not found", err) + } + return Error(500, "Failed to get user auth token", err) + } + + if c.UserToken != nil && c.UserToken.Id == token.Id { + return Error(400, "Cannot revoke active user auth token", nil) + } + + err = server.AuthTokenService.RevokeToken(token) + if err != nil { + if err == models.ErrUserTokenNotFound { + return Error(404, "User auth token not found", err) + } + return Error(500, "Failed to revoke user auth token", err) + } + + return JSON(200, util.DynMap{ + "message": "User auth token revoked", + }) +} diff --git a/pkg/api/user_token_test.go b/pkg/api/user_token_test.go new file mode 100644 index 00000000000..111070dca92 --- /dev/null +++ b/pkg/api/user_token_test.go @@ -0,0 +1,294 @@ +package api + +import ( + "testing" + "time" + + "github.com/grafana/grafana/pkg/bus" + m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/auth" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestUserTokenApiEndpoint(t *testing.T) { + Convey("When current user attempts to revoke an auth token for a non-existing user", t, func() { + userId := int64(0) + bus.AddHandler("test", func(cmd *m.GetUserByIdQuery) error { + userId = cmd.Id + return m.ErrUserNotFound + }) + + cmd := m.RevokeAuthTokenCmd{AuthTokenId: 2} + + revokeUserAuthTokenScenario("Should return not found when calling POST on", "/api/user/revoke-auth-token", "/api/user/revoke-auth-token", cmd, 200, func(sc *scenarioContext) { + sc.fakeReqWithParams("POST", sc.url, map[string]string{}).exec() + So(sc.resp.Code, ShouldEqual, 404) + So(userId, ShouldEqual, 200) + }) + }) + + Convey("When current user gets auth tokens for a non-existing user", t, func() { + userId := int64(0) + bus.AddHandler("test", func(cmd *m.GetUserByIdQuery) error { + userId = cmd.Id + return m.ErrUserNotFound + }) + + getUserAuthTokensScenario("Should return not found when calling GET on", "/api/user/auth-tokens", "/api/user/auth-tokens", 200, func(sc *scenarioContext) { + sc.fakeReqWithParams("GET", sc.url, map[string]string{}).exec() + So(sc.resp.Code, ShouldEqual, 404) + So(userId, ShouldEqual, 200) + }) + }) + + Convey("When logout an existing user from all devices", t, func() { + bus.AddHandler("test", func(cmd *m.GetUserByIdQuery) error { + cmd.Result = &m.User{Id: 200} + return nil + }) + + logoutUserFromAllDevicesInternalScenario("Should be successful", 1, func(sc *scenarioContext) { + sc.fakeReqWithParams("POST", sc.url, map[string]string{}).exec() + So(sc.resp.Code, ShouldEqual, 200) + }) + }) + + Convey("When logout a non-existing user from all devices", t, func() { + bus.AddHandler("test", func(cmd *m.GetUserByIdQuery) error { + return m.ErrUserNotFound + }) + + logoutUserFromAllDevicesInternalScenario("Should return not found", TestUserID, func(sc *scenarioContext) { + sc.fakeReqWithParams("POST", sc.url, map[string]string{}).exec() + So(sc.resp.Code, ShouldEqual, 404) + }) + }) + + Convey("When revoke an auth token for a user", t, func() { + bus.AddHandler("test", func(cmd *m.GetUserByIdQuery) error { + cmd.Result = &m.User{Id: 200} + return nil + }) + + cmd := m.RevokeAuthTokenCmd{AuthTokenId: 2} + token := &m.UserToken{Id: 1} + + revokeUserAuthTokenInternalScenario("Should be successful", cmd, 200, token, func(sc *scenarioContext) { + sc.userAuthTokenService.GetUserTokenProvider = func(userId, userTokenId int64) (*m.UserToken, error) { + return &m.UserToken{Id: 2}, nil + } + sc.fakeReqWithParams("POST", sc.url, map[string]string{}).exec() + So(sc.resp.Code, ShouldEqual, 200) + }) + }) + + Convey("When revoke the active auth token used by himself", t, func() { + bus.AddHandler("test", func(cmd *m.GetUserByIdQuery) error { + cmd.Result = &m.User{Id: TestUserID} + return nil + }) + + cmd := m.RevokeAuthTokenCmd{AuthTokenId: 2} + token := &m.UserToken{Id: 2} + + revokeUserAuthTokenInternalScenario("Should not be successful", cmd, TestUserID, token, func(sc *scenarioContext) { + sc.userAuthTokenService.GetUserTokenProvider = func(userId, userTokenId int64) (*m.UserToken, error) { + return token, nil + } + sc.fakeReqWithParams("POST", sc.url, map[string]string{}).exec() + So(sc.resp.Code, ShouldEqual, 400) + }) + }) + + Convey("When gets auth tokens for a user", t, func() { + bus.AddHandler("test", func(cmd *m.GetUserByIdQuery) error { + cmd.Result = &m.User{Id: TestUserID} + return nil + }) + + currentToken := &m.UserToken{Id: 1} + + getUserAuthTokensInternalScenario("Should be successful", currentToken, func(sc *scenarioContext) { + tokens := []*m.UserToken{ + { + Id: 1, + ClientIp: "127.0.0.1", + UserAgent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36", + CreatedAt: time.Now().Unix(), + SeenAt: time.Now().Unix(), + }, + { + Id: 2, + ClientIp: "127.0.0.2", + UserAgent: "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1", + CreatedAt: time.Now().Unix(), + SeenAt: time.Now().Unix(), + }, + } + sc.userAuthTokenService.GetUserTokensProvider = func(userId int64) ([]*m.UserToken, error) { + return tokens, nil + } + sc.fakeReqWithParams("GET", sc.url, map[string]string{}).exec() + + So(sc.resp.Code, ShouldEqual, 200) + result := sc.ToJSON() + So(result.MustArray(), ShouldHaveLength, 2) + + resultOne := result.GetIndex(0) + So(resultOne.Get("id").MustInt64(), ShouldEqual, tokens[0].Id) + So(resultOne.Get("isActive").MustBool(), ShouldBeTrue) + So(resultOne.Get("clientIp").MustString(), ShouldEqual, "127.0.0.1") + So(resultOne.Get("userAgent").MustString(), ShouldEqual, "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36") + So(resultOne.Get("createdAt").MustString(), ShouldEqual, time.Unix(tokens[0].CreatedAt, 0).Format(time.RFC3339)) + So(resultOne.Get("seenAt").MustString(), ShouldEqual, time.Unix(tokens[0].SeenAt, 0).Format(time.RFC3339)) + + resultTwo := result.GetIndex(1) + So(resultTwo.Get("id").MustInt64(), ShouldEqual, tokens[1].Id) + So(resultTwo.Get("isActive").MustBool(), ShouldBeFalse) + So(resultTwo.Get("clientIp").MustString(), ShouldEqual, "127.0.0.2") + So(resultTwo.Get("userAgent").MustString(), ShouldEqual, "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1") + So(resultTwo.Get("createdAt").MustString(), ShouldEqual, time.Unix(tokens[1].CreatedAt, 0).Format(time.RFC3339)) + So(resultTwo.Get("seenAt").MustString(), ShouldEqual, time.Unix(tokens[1].SeenAt, 0).Format(time.RFC3339)) + }) + }) +} + +func revokeUserAuthTokenScenario(desc string, url string, routePattern string, cmd m.RevokeAuthTokenCmd, userId int64, fn scenarioFunc) { + Convey(desc+" "+url, func() { + defer bus.ClearBusHandlers() + + fakeAuthTokenService := auth.NewFakeUserAuthTokenService() + + hs := HTTPServer{ + Bus: bus.GetBus(), + AuthTokenService: fakeAuthTokenService, + } + + sc := setupScenarioContext(url) + sc.userAuthTokenService = fakeAuthTokenService + sc.defaultHandler = Wrap(func(c *m.ReqContext) Response { + sc.context = c + sc.context.UserId = userId + sc.context.OrgId = TestOrgID + sc.context.OrgRole = m.ROLE_ADMIN + + return hs.RevokeUserAuthToken(c, cmd) + }) + + sc.m.Post(routePattern, sc.defaultHandler) + + fn(sc) + }) +} + +func getUserAuthTokensScenario(desc string, url string, routePattern string, userId int64, fn scenarioFunc) { + Convey(desc+" "+url, func() { + defer bus.ClearBusHandlers() + + fakeAuthTokenService := auth.NewFakeUserAuthTokenService() + + hs := HTTPServer{ + Bus: bus.GetBus(), + AuthTokenService: fakeAuthTokenService, + } + + sc := setupScenarioContext(url) + sc.userAuthTokenService = fakeAuthTokenService + sc.defaultHandler = Wrap(func(c *m.ReqContext) Response { + sc.context = c + sc.context.UserId = userId + sc.context.OrgId = TestOrgID + sc.context.OrgRole = m.ROLE_ADMIN + + return hs.GetUserAuthTokens(c) + }) + + sc.m.Get(routePattern, sc.defaultHandler) + + fn(sc) + }) +} + +func logoutUserFromAllDevicesInternalScenario(desc string, userId int64, fn scenarioFunc) { + Convey(desc, func() { + defer bus.ClearBusHandlers() + + hs := HTTPServer{ + Bus: bus.GetBus(), + AuthTokenService: auth.NewFakeUserAuthTokenService(), + } + + sc := setupScenarioContext("/") + sc.defaultHandler = Wrap(func(c *m.ReqContext) Response { + sc.context = c + sc.context.UserId = TestUserID + sc.context.OrgId = TestOrgID + sc.context.OrgRole = m.ROLE_ADMIN + + return hs.logoutUserFromAllDevicesInternal(userId) + }) + + sc.m.Post("/", sc.defaultHandler) + + fn(sc) + }) +} + +func revokeUserAuthTokenInternalScenario(desc string, cmd m.RevokeAuthTokenCmd, userId int64, token *m.UserToken, fn scenarioFunc) { + Convey(desc, func() { + defer bus.ClearBusHandlers() + + fakeAuthTokenService := auth.NewFakeUserAuthTokenService() + + hs := HTTPServer{ + Bus: bus.GetBus(), + AuthTokenService: fakeAuthTokenService, + } + + sc := setupScenarioContext("/") + sc.userAuthTokenService = fakeAuthTokenService + sc.defaultHandler = Wrap(func(c *m.ReqContext) Response { + sc.context = c + sc.context.UserId = TestUserID + sc.context.OrgId = TestOrgID + sc.context.OrgRole = m.ROLE_ADMIN + sc.context.UserToken = token + + return hs.revokeUserAuthTokenInternal(c, userId, cmd) + }) + + sc.m.Post("/", sc.defaultHandler) + + fn(sc) + }) +} + +func getUserAuthTokensInternalScenario(desc string, token *m.UserToken, fn scenarioFunc) { + Convey(desc, func() { + defer bus.ClearBusHandlers() + + fakeAuthTokenService := auth.NewFakeUserAuthTokenService() + + hs := HTTPServer{ + Bus: bus.GetBus(), + AuthTokenService: fakeAuthTokenService, + } + + sc := setupScenarioContext("/") + sc.userAuthTokenService = fakeAuthTokenService + sc.defaultHandler = Wrap(func(c *m.ReqContext) Response { + sc.context = c + sc.context.UserId = TestUserID + sc.context.OrgId = TestOrgID + sc.context.OrgRole = m.ROLE_ADMIN + sc.context.UserToken = token + + return hs.getUserAuthTokensInternal(c, TestUserID) + }) + + sc.m.Get("/", sc.defaultHandler) + + fn(sc) + }) +} diff --git a/pkg/models/user_token.go b/pkg/models/user_token.go index 22f92cb21d2..8c3e7985995 100644 --- a/pkg/models/user_token.go +++ b/pkg/models/user_token.go @@ -1,6 +1,8 @@ package models -import "errors" +import ( + "errors" +) // Typed errors var ( @@ -23,6 +25,10 @@ type UserToken struct { UnhashedToken string } +type RevokeAuthTokenCmd struct { + AuthTokenId int64 `json:"authTokenId"` +} + // UserTokenService are used for generating and validating user tokens type UserTokenService interface { CreateToken(userId int64, clientIP, userAgent string) (*UserToken, error) From 80ce11a4a433a755a66b9b7782892d2b5e1436cd Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Fri, 8 Mar 2019 15:15:57 +0100 Subject: [PATCH 09/75] docs: update admin and user http api documentation --- docs/sources/http_api/admin.md | 102 +++++++++++++++++++++++++++++++++ docs/sources/http_api/user.md | 72 +++++++++++++++++++++++ 2 files changed, 174 insertions(+) diff --git a/docs/sources/http_api/admin.md b/docs/sources/http_api/admin.md index a27fd2aac14..c2d540c452b 100644 --- a/docs/sources/http_api/admin.md +++ b/docs/sources/http_api/admin.md @@ -341,3 +341,105 @@ Content-Type: application/json {"state": "new state", "message": "alerts pause/un paused", "alertsAffected": 100} ``` + +## Auth tokens for User + +`GET /api/admin/users/:id/auth-tokens` + +Return a list of all auth tokens (devices) that the user currently have logged in from. + +Only works with Basic Authentication (username and password). See [introduction](http://docs.grafana.org/http_api/admin/#admin-api) for an explanation. + +**Example Request**: + +```http +GET /api/admin/users/1/auth-tokens HTTP/1.1 +Accept: application/json +Content-Type: application/json +``` + +**Example Response**: + +```http +HTTP/1.1 200 +Content-Type: application/json + +[ + { + "id": 361, + "isActive": false, + "clientIp": "127.0.0.1", + "userAgent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36", + "createdAt": "2019-03-05T21:22:54+01:00", + "seenAt": "2019-03-06T19:41:06+01:00" + }, + { + "id": 364, + "isActive": false, + "clientIp": "127.0.0.1", + "userAgent": "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1", + "createdAt": "2019-03-06T19:41:19+01:00", + "seenAt": "2019-03-06T19:41:21+01:00" + } +] +``` + +## Revoke auth token for User + +`POST /api/admin/users/:id/revoke-auth-token` + +Revokes the given auth token (device) for the user. User of issued auth token (device) will no longer be logged in +and will be required to authenticate again upon next activity. + +Only works with Basic Authentication (username and password). See [introduction](http://docs.grafana.org/http_api/admin/#admin-api) for an explanation. + +**Example Request**: + +```http +POST /api/admin/users/1/revoke-auth-token HTTP/1.1 +Accept: application/json +Content-Type: application/json + +{ + "authTokenId": 364 +} +``` + +**Example Response**: + +```http +HTTP/1.1 200 +Content-Type: application/json + +{ + "message": "User auth token revoked" +} +``` + +## Logout User + +`POST /api/admin/users/:id/logout` + +Logout user revokes all auth tokens (devices) for the user. User of issued auth tokens (devices) will no longer be logged in +and will be required to authenticate again upon next activity. + +Only works with Basic Authentication (username and password). See [introduction](http://docs.grafana.org/http_api/admin/#admin-api) for an explanation. + +**Example Request**: + +```http +POST /api/admin/users/1/logout HTTP/1.1 +Accept: application/json +Content-Type: application/json +``` + +**Example Response**: + +```http +HTTP/1.1 200 +Content-Type: application/json + +{ + "message": "User auth token revoked" +} +``` diff --git a/docs/sources/http_api/user.md b/docs/sources/http_api/user.md index 669e8003247..a81f608c2f5 100644 --- a/docs/sources/http_api/user.md +++ b/docs/sources/http_api/user.md @@ -478,3 +478,75 @@ Content-Type: application/json {"message":"Dashboard unstarred"} ``` + +## Auth tokens of the actual User + +`GET /api/user/auth-tokens` + +Return a list of all auth tokens (devices) that the actual user currently have logged in from. + +**Example Request**: + +```http +GET /api/user/auth-tokens HTTP/1.1 +Accept: application/json +Content-Type: application/json +Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk +``` + +**Example Response**: + +```http +HTTP/1.1 200 +Content-Type: application/json + +[ + { + "id": 361, + "isActive": true, + "clientIp": "127.0.0.1", + "userAgent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36", + "createdAt": "2019-03-05T21:22:54+01:00", + "seenAt": "2019-03-06T19:41:06+01:00" + }, + { + "id": 364, + "isActive": false, + "clientIp": "127.0.0.1", + "userAgent": "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1", + "createdAt": "2019-03-06T19:41:19+01:00", + "seenAt": "2019-03-06T19:41:21+01:00" + } +] +``` + +## Revoke an auth token of the actual User + +`POST /api/user/revoke-auth-token` + +Revokes the given auth token (device) for the actual user. User of issued auth token (device) will no longer be logged in +and will be required to authenticate again upon next activity. + +**Example Request**: + +```http +POST /api/user/revoke-auth-token HTTP/1.1 +Accept: application/json +Content-Type: application/json +Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk + +{ + "authTokenId": 364 +} +``` + +**Example Response**: + +```http +HTTP/1.1 200 +Content-Type: application/json + +{ + "message": "User auth token revoked" +} +``` From c8ff698d9094dc43192f825874ccb5ea8f27bd83 Mon Sep 17 00:00:00 2001 From: bergquist Date: Sat, 23 Feb 2019 22:59:34 +0100 Subject: [PATCH 10/75] avoid exposing internal structs and functions --- pkg/infra/distcache/database_storage.go | 16 ++++++++-------- pkg/infra/distcache/distcache.go | 12 +++++------- pkg/infra/distcache/distcache_test.go | 14 +++++--------- pkg/infra/distcache/memcached_storage.go | 12 ++++++------ pkg/infra/distcache/redis_storage.go | 21 ++++----------------- 5 files changed, 28 insertions(+), 47 deletions(-) diff --git a/pkg/infra/distcache/database_storage.go b/pkg/infra/distcache/database_storage.go index cff5e0fc499..f4365383b82 100644 --- a/pkg/infra/distcache/database_storage.go +++ b/pkg/infra/distcache/database_storage.go @@ -45,13 +45,13 @@ func (dc *databaseCache) StartGC() { } func (dc *databaseCache) Get(key string) (interface{}, error) { - cacheHits := []CacheData{} + cacheHits := []cacheData{} err := dc.SQLStore.NewSession().Where(`key = ?`, key).Find(&cacheHits) if err != nil { return nil, err } - var cacheHit CacheData + var cacheHit cacheData if len(cacheHits) == 0 { return nil, ErrCacheItemNotFound } @@ -64,15 +64,15 @@ func (dc *databaseCache) Get(key string) (interface{}, error) { } } - item := &Item{} - if err = DecodeGob(cacheHit.Data, item); err != nil { + item := &cachedItem{} + if err = decodeGob(cacheHit.Data, item); err != nil { return nil, err } return item.Val, nil } -type CacheData struct { +type cacheData struct { Key string Data []byte Expires int64 @@ -80,15 +80,15 @@ type CacheData struct { } func (dc *databaseCache) Put(key string, value interface{}, expire time.Duration) error { - item := &Item{Val: value} - data, err := EncodeGob(item) + item := &cachedItem{Val: value} + data, err := encodeGob(item) if err != nil { return err } now := getTime().Unix() - cacheHits := []CacheData{} + cacheHits := []cacheData{} err = dc.SQLStore.NewSession().Where(`key = ?`, key).Find(&cacheHits) if err != nil { return err diff --git a/pkg/infra/distcache/distcache.go b/pkg/infra/distcache/distcache.go index 8a6f7daf90c..8ba1a306a3f 100644 --- a/pkg/infra/distcache/distcache.go +++ b/pkg/infra/distcache/distcache.go @@ -46,7 +46,7 @@ func createClient(opts CacheOpts, sqlstore *sqlstore.SqlStore) cacheStorage { // return nil // } - return newDatabaseCache(sqlstore) //&databaseCache{SQLStore: sqlstore} + return newDatabaseCache(sqlstore) } // DistributedCache allows Grafana to cache data outside its own process @@ -56,19 +56,17 @@ type DistributedCache struct { SQLStore *sqlstore.SqlStore `inject:""` } -type Item struct { - Val interface{} - Created int64 - Expire int64 +type cachedItem struct { + Val interface{} } -func EncodeGob(item *Item) ([]byte, error) { +func encodeGob(item *cachedItem) ([]byte, error) { buf := bytes.NewBuffer(nil) err := gob.NewEncoder(buf).Encode(item) return buf.Bytes(), err } -func DecodeGob(data []byte, out *Item) error { +func decodeGob(data []byte, out *cachedItem) error { buf := bytes.NewBuffer(data) return gob.NewDecoder(buf).Decode(&out) } diff --git a/pkg/infra/distcache/distcache_test.go b/pkg/infra/distcache/distcache_test.go index a04b5d0228f..6f59c40f0e9 100644 --- a/pkg/infra/distcache/distcache_test.go +++ b/pkg/infra/distcache/distcache_test.go @@ -27,7 +27,7 @@ func createTestClient(t *testing.T, name string) cacheStorage { } func TestAllCacheClients(t *testing.T) { - clients := []string{"database", "redis", "memcached"} // add redis, memcache, memory + clients := []string{"database", "redis"} // add redis, memcache, memory for _, v := range clients { client := createTestClient(t, v) @@ -59,19 +59,15 @@ func CanPutGetAndDeleteCachedObjects(t *testing.T, name string, client cacheStor } func CanNotFetchExpiredItems(t *testing.T, name string, client cacheStorage) { - if name == "redis" { - t.Skip() //this test does not work with redis since it uses its own getTime fn - } - cacheableStruct := CacheableStruct{String: "hej", Int64: 2000} - // insert cache item one day back - getTime = func() time.Time { return time.Now().AddDate(0, 0, -2) } - err := client.Put("key", cacheableStruct, 10000*time.Second) + err := client.Put("key", cacheableStruct, time.Second) assert.Equal(t, err, nil) + //not sure how this can be avoided when testing redis/memcached :/ + <-time.After(time.Second + time.Millisecond) + // should not be able to read that value since its expired - getTime = time.Now _, err = client.Get("key") assert.Equal(t, err, ErrCacheItemNotFound) } diff --git a/pkg/infra/distcache/memcached_storage.go b/pkg/infra/distcache/memcached_storage.go index 44fbbcc33c6..71e037cf196 100644 --- a/pkg/infra/distcache/memcached_storage.go +++ b/pkg/infra/distcache/memcached_storage.go @@ -16,7 +16,7 @@ func newMemcacheStorage(connStr string) *memcacheStorage { } } -func NewItem(sid string, data []byte, expire int32) *memcache.Item { +func newItem(sid string, data []byte, expire int32) *memcache.Item { return &memcache.Item{ Key: sid, Value: data, @@ -26,14 +26,14 @@ func NewItem(sid string, data []byte, expire int32) *memcache.Item { // Set sets value to given key in the cache. func (s *memcacheStorage) Put(key string, val interface{}, expires time.Duration) error { - item := &Item{Val: val} + item := &cachedItem{Val: val} - bytes, err := EncodeGob(item) + bytes, err := encodeGob(item) if err != nil { return err } - memcacheItem := NewItem(key, bytes, int32(expires)) + memcacheItem := newItem(key, bytes, int32(expires)) s.c.Add(memcacheItem) return nil @@ -46,9 +46,9 @@ func (s *memcacheStorage) Get(key string) (interface{}, error) { return nil, err } - item := &Item{} + item := &cachedItem{} - err = DecodeGob(i.Value, item) + err = decodeGob(i.Value, item) if err != nil { return nil, err } diff --git a/pkg/infra/distcache/redis_storage.go b/pkg/infra/distcache/redis_storage.go index 06fc6931758..49055fd8356 100644 --- a/pkg/infra/distcache/redis_storage.go +++ b/pkg/infra/distcache/redis_storage.go @@ -22,8 +22,8 @@ func newRedisStorage(c *redis.Client) *redisStorage { // Set sets value to given key in session. func (s *redisStorage) Put(key string, val interface{}, expires time.Duration) error { - item := &Item{Created: getTime().Unix(), Val: val} - value, err := EncodeGob(item) + item := &cachedItem{Val: val} + value, err := encodeGob(item) if err != nil { return err } @@ -42,8 +42,8 @@ func (s *redisStorage) Put(key string, val interface{}, expires time.Duration) e func (s *redisStorage) Get(key string) (interface{}, error) { v := s.c.Get(key) - item := &Item{} - err := DecodeGob([]byte(v.Val()), item) + item := &cachedItem{} + err := decodeGob([]byte(v.Val()), item) if err == nil { return item.Val, nil @@ -65,16 +65,3 @@ func (s *redisStorage) Delete(key string) error { cmd := s.c.Del(key) return cmd.Err() } - -// RedisProvider represents a redis session provider implementation. -type RedisProvider struct { - c *redis.Client - duration time.Duration - prefix string -} - -// Exist returns true if session with given ID exists. -func (p *RedisProvider) Exist(sid string) bool { - has, err := p.c.Exists(p.prefix + sid).Result() - return err == nil && has -} From a60bb83a70376639ac3460ba5b0d51b2e3fdc6dd Mon Sep 17 00:00:00 2001 From: bergquist Date: Sun, 3 Mar 2019 04:42:11 +0100 Subject: [PATCH 11/75] extract tests into seperate files --- pkg/infra/distcache/distcache.go | 10 ++++++++-- pkg/infra/distcache/distcache_test.go | 15 ++++++++------- pkg/infra/distcache/memcached_storage.go | 10 +++++++--- pkg/infra/distcache/redis_storage.go | 8 +------- pkg/infra/distcache/redis_storage_test.go | 11 +++++++++++ 5 files changed, 35 insertions(+), 19 deletions(-) diff --git a/pkg/infra/distcache/distcache.go b/pkg/infra/distcache/distcache.go index 8ba1a306a3f..87a6da45029 100644 --- a/pkg/infra/distcache/distcache.go +++ b/pkg/infra/distcache/distcache.go @@ -8,6 +8,7 @@ import ( "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/services/sqlstore" + redis "gopkg.in/redis.v2" "github.com/grafana/grafana/pkg/registry" ) @@ -35,11 +36,16 @@ type CacheOpts struct { func createClient(opts CacheOpts, sqlstore *sqlstore.SqlStore) cacheStorage { if opts.name == "redis" { - return newRedisStorage(nil) + opt := &redis.Options{ + Network: "tcp", + Addr: "localhost:6379", + } + + return newRedisStorage(redis.NewClient(opt)) } if opts.name == "memcache" { - return newMemcacheStorage("localhost:9090") + return newMemcacheStorage("localhost:11211") } // if opts.name == "memory" { diff --git a/pkg/infra/distcache/distcache_test.go b/pkg/infra/distcache/distcache_test.go index 6f59c40f0e9..af6f426e1c0 100644 --- a/pkg/infra/distcache/distcache_test.go +++ b/pkg/infra/distcache/distcache_test.go @@ -27,18 +27,19 @@ func createTestClient(t *testing.T, name string) cacheStorage { } func TestAllCacheClients(t *testing.T) { - clients := []string{"database", "redis"} // add redis, memcache, memory + //clients := []string{"database", "redis", "memcache"} // add redis, memcache, memory + clients := []string{} // add redis, memcache, memory for _, v := range clients { client := createTestClient(t, v) - CanPutGetAndDeleteCachedObjects(t, v, client) - CanNotFetchExpiredItems(t, v, client) - CanSetInfiniteCacheExpiration(t, v, client) + CanPutGetAndDeleteCachedObjects(t, client) + CanNotFetchExpiredItems(t, client) + CanSetInfiniteCacheExpiration(t, client) } } -func CanPutGetAndDeleteCachedObjects(t *testing.T, name string, client cacheStorage) { +func CanPutGetAndDeleteCachedObjects(t *testing.T, client cacheStorage) { cacheableStruct := CacheableStruct{String: "hej", Int64: 2000} err := client.Put("key", cacheableStruct, 0) @@ -58,7 +59,7 @@ func CanPutGetAndDeleteCachedObjects(t *testing.T, name string, client cacheStor assert.Equal(t, err, ErrCacheItemNotFound) } -func CanNotFetchExpiredItems(t *testing.T, name string, client cacheStorage) { +func CanNotFetchExpiredItems(t *testing.T, client cacheStorage) { cacheableStruct := CacheableStruct{String: "hej", Int64: 2000} err := client.Put("key", cacheableStruct, time.Second) @@ -72,7 +73,7 @@ func CanNotFetchExpiredItems(t *testing.T, name string, client cacheStorage) { assert.Equal(t, err, ErrCacheItemNotFound) } -func CanSetInfiniteCacheExpiration(t *testing.T, name string, client cacheStorage) { +func CanSetInfiniteCacheExpiration(t *testing.T, client cacheStorage) { cacheableStruct := CacheableStruct{String: "hej", Int64: 2000} // insert cache item one day back diff --git a/pkg/infra/distcache/memcached_storage.go b/pkg/infra/distcache/memcached_storage.go index 71e037cf196..1186bef626b 100644 --- a/pkg/infra/distcache/memcached_storage.go +++ b/pkg/infra/distcache/memcached_storage.go @@ -24,7 +24,7 @@ func newItem(sid string, data []byte, expire int32) *memcache.Item { } } -// Set sets value to given key in the cache. +// Put sets value to given key in the cache. func (s *memcacheStorage) Put(key string, val interface{}, expires time.Duration) error { item := &cachedItem{Val: val} @@ -35,13 +35,17 @@ func (s *memcacheStorage) Put(key string, val interface{}, expires time.Duration memcacheItem := newItem(key, bytes, int32(expires)) - s.c.Add(memcacheItem) - return nil + return s.c.Add(memcacheItem) } // Get gets value by given key in the cache. func (s *memcacheStorage) Get(key string) (interface{}, error) { i, err := s.c.Get(key) + + if err != nil && err.Error() == "memcache: cache miss" { + return nil, ErrCacheItemNotFound + } + if err != nil { return nil, err } diff --git a/pkg/infra/distcache/redis_storage.go b/pkg/infra/distcache/redis_storage.go index 49055fd8356..bb21b26473e 100644 --- a/pkg/infra/distcache/redis_storage.go +++ b/pkg/infra/distcache/redis_storage.go @@ -11,13 +11,7 @@ type redisStorage struct { } func newRedisStorage(c *redis.Client) *redisStorage { - opt := &redis.Options{ - Network: "tcp", - Addr: "localhost:6379", - } - return &redisStorage{ - c: redis.NewClient(opt), - } + return &redisStorage{c: c} } // Set sets value to given key in session. diff --git a/pkg/infra/distcache/redis_storage_test.go b/pkg/infra/distcache/redis_storage_test.go index e793fbec4c4..39d39d41b12 100644 --- a/pkg/infra/distcache/redis_storage_test.go +++ b/pkg/infra/distcache/redis_storage_test.go @@ -1 +1,12 @@ package distcache + +import "testing" + +func TestRedisCacheStorage(t *testing.T) { + + client := createTestClient(t, "redis") + + CanPutGetAndDeleteCachedObjects(t, client) + CanNotFetchExpiredItems(t, client) + CanSetInfiniteCacheExpiration(t, client) +} From 8db2864feef388f9ee1894c84783e5d05ff61a60 Mon Sep 17 00:00:00 2001 From: bergquist Date: Sun, 3 Mar 2019 05:25:17 +0100 Subject: [PATCH 12/75] adds memory as dist storage alt --- .../database_storage_integration_test.go | 12 +++++++ pkg/infra/distcache/distcache.go | 6 ++-- pkg/infra/distcache/distcache_test.go | 3 +- pkg/infra/distcache/memcached_storage_test.go | 12 +++++++ pkg/infra/distcache/memory_storage.go | 35 +++++++++++++++++++ 5 files changed, 63 insertions(+), 5 deletions(-) create mode 100644 pkg/infra/distcache/database_storage_integration_test.go create mode 100644 pkg/infra/distcache/memcached_storage_test.go create mode 100644 pkg/infra/distcache/memory_storage.go diff --git a/pkg/infra/distcache/database_storage_integration_test.go b/pkg/infra/distcache/database_storage_integration_test.go new file mode 100644 index 00000000000..e305759983d --- /dev/null +++ b/pkg/infra/distcache/database_storage_integration_test.go @@ -0,0 +1,12 @@ +package distcache + +import "testing" + +func TestIntegrationDatabaseCacheStorage(t *testing.T) { + + client := createTestClient(t, "database") + + CanPutGetAndDeleteCachedObjects(t, client) + CanNotFetchExpiredItems(t, client) + CanSetInfiniteCacheExpiration(t, client) +} diff --git a/pkg/infra/distcache/distcache.go b/pkg/infra/distcache/distcache.go index 87a6da45029..d21ada1e6a3 100644 --- a/pkg/infra/distcache/distcache.go +++ b/pkg/infra/distcache/distcache.go @@ -48,9 +48,9 @@ func createClient(opts CacheOpts, sqlstore *sqlstore.SqlStore) cacheStorage { return newMemcacheStorage("localhost:11211") } - // if opts.name == "memory" { - // return nil - // } + if opts.name == "memory" { + return newMemoryStorage() + } return newDatabaseCache(sqlstore) } diff --git a/pkg/infra/distcache/distcache_test.go b/pkg/infra/distcache/distcache_test.go index af6f426e1c0..ec778b0c335 100644 --- a/pkg/infra/distcache/distcache_test.go +++ b/pkg/infra/distcache/distcache_test.go @@ -27,8 +27,7 @@ func createTestClient(t *testing.T, name string) cacheStorage { } func TestAllCacheClients(t *testing.T) { - //clients := []string{"database", "redis", "memcache"} // add redis, memcache, memory - clients := []string{} // add redis, memcache, memory + clients := []string{"memory"} // add redis, memcache, memory for _, v := range clients { client := createTestClient(t, v) diff --git a/pkg/infra/distcache/memcached_storage_test.go b/pkg/infra/distcache/memcached_storage_test.go new file mode 100644 index 00000000000..b02f67f062f --- /dev/null +++ b/pkg/infra/distcache/memcached_storage_test.go @@ -0,0 +1,12 @@ +package distcache + +import "testing" + +func TestMemcachedCacheStorage(t *testing.T) { + + client := createTestClient(t, "memcache") + + CanPutGetAndDeleteCachedObjects(t, client) + CanNotFetchExpiredItems(t, client) + CanSetInfiniteCacheExpiration(t, client) +} diff --git a/pkg/infra/distcache/memory_storage.go b/pkg/infra/distcache/memory_storage.go new file mode 100644 index 00000000000..a1203cabe75 --- /dev/null +++ b/pkg/infra/distcache/memory_storage.go @@ -0,0 +1,35 @@ +package distcache + +import ( + "time" + + gocache "github.com/patrickmn/go-cache" +) + +type memoryStorage struct { + c *gocache.Cache +} + +func newMemoryStorage() *memoryStorage { + return &memoryStorage{ + c: gocache.New(time.Minute*30, time.Minute*30), + } +} + +func (s *memoryStorage) Put(key string, val interface{}, expires time.Duration) error { + return s.c.Add(key, val, expires) +} + +func (s *memoryStorage) Get(key string) (interface{}, error) { + val, exist := s.c.Get(key) + if !exist { + return nil, ErrCacheItemNotFound + } + + return val, nil +} + +func (s *memoryStorage) Delete(key string) error { + s.c.Delete(key) + return nil +} From 33935b09f0e543d0fc9583fabd2810685ca2c0cb Mon Sep 17 00:00:00 2001 From: bergquist Date: Sun, 3 Mar 2019 12:34:41 +0100 Subject: [PATCH 13/75] uses set instead of add for memcache set always sets the value regardless. --- .../database_storage_integration_test.go | 6 +----- pkg/infra/distcache/distcache_test.go | 16 ++++++++-------- pkg/infra/distcache/memcached_storage.go | 2 +- pkg/infra/distcache/memcached_storage_test.go | 7 +------ pkg/infra/distcache/redis_storage_test.go | 7 +------ 5 files changed, 12 insertions(+), 26 deletions(-) diff --git a/pkg/infra/distcache/database_storage_integration_test.go b/pkg/infra/distcache/database_storage_integration_test.go index e305759983d..b8f564f9710 100644 --- a/pkg/infra/distcache/database_storage_integration_test.go +++ b/pkg/infra/distcache/database_storage_integration_test.go @@ -4,9 +4,5 @@ import "testing" func TestIntegrationDatabaseCacheStorage(t *testing.T) { - client := createTestClient(t, "database") - - CanPutGetAndDeleteCachedObjects(t, client) - CanNotFetchExpiredItems(t, client) - CanSetInfiniteCacheExpiration(t, client) + RunTestsForClient(t, createTestClient(t, "database")) } diff --git a/pkg/infra/distcache/distcache_test.go b/pkg/infra/distcache/distcache_test.go index ec778b0c335..a40066b788f 100644 --- a/pkg/infra/distcache/distcache_test.go +++ b/pkg/infra/distcache/distcache_test.go @@ -26,16 +26,16 @@ func createTestClient(t *testing.T, name string) cacheStorage { return createClient(CacheOpts{name: name}, sqlstore) } -func TestAllCacheClients(t *testing.T) { - clients := []string{"memory"} // add redis, memcache, memory +func TestMemoryStorageClient(t *testing.T) { - for _, v := range clients { - client := createTestClient(t, v) + client := createTestClient(t, "memory") + RunTestsForClient(t, client) +} - CanPutGetAndDeleteCachedObjects(t, client) - CanNotFetchExpiredItems(t, client) - CanSetInfiniteCacheExpiration(t, client) - } +func RunTestsForClient(t *testing.T, client cacheStorage) { + CanPutGetAndDeleteCachedObjects(t, client) + CanNotFetchExpiredItems(t, client) + CanSetInfiniteCacheExpiration(t, client) } func CanPutGetAndDeleteCachedObjects(t *testing.T, client cacheStorage) { diff --git a/pkg/infra/distcache/memcached_storage.go b/pkg/infra/distcache/memcached_storage.go index 1186bef626b..7f97a043628 100644 --- a/pkg/infra/distcache/memcached_storage.go +++ b/pkg/infra/distcache/memcached_storage.go @@ -35,7 +35,7 @@ func (s *memcacheStorage) Put(key string, val interface{}, expires time.Duration memcacheItem := newItem(key, bytes, int32(expires)) - return s.c.Add(memcacheItem) + return s.c.Set(memcacheItem) } // Get gets value by given key in the cache. diff --git a/pkg/infra/distcache/memcached_storage_test.go b/pkg/infra/distcache/memcached_storage_test.go index b02f67f062f..524a4fcea10 100644 --- a/pkg/infra/distcache/memcached_storage_test.go +++ b/pkg/infra/distcache/memcached_storage_test.go @@ -3,10 +3,5 @@ package distcache import "testing" func TestMemcachedCacheStorage(t *testing.T) { - - client := createTestClient(t, "memcache") - - CanPutGetAndDeleteCachedObjects(t, client) - CanNotFetchExpiredItems(t, client) - CanSetInfiniteCacheExpiration(t, client) + RunTestsForClient(t, createTestClient(t, "memcache")) } diff --git a/pkg/infra/distcache/redis_storage_test.go b/pkg/infra/distcache/redis_storage_test.go index 39d39d41b12..6ba093a205c 100644 --- a/pkg/infra/distcache/redis_storage_test.go +++ b/pkg/infra/distcache/redis_storage_test.go @@ -3,10 +3,5 @@ package distcache import "testing" func TestRedisCacheStorage(t *testing.T) { - - client := createTestClient(t, "redis") - - CanPutGetAndDeleteCachedObjects(t, client) - CanNotFetchExpiredItems(t, client) - CanSetInfiniteCacheExpiration(t, client) + RunTestsForClient(t, createTestClient(t, "redis")) } From f9f2d9fcf3074123d96750ff1b428d2cf3c09911 Mon Sep 17 00:00:00 2001 From: bergquist Date: Sun, 3 Mar 2019 12:41:38 +0100 Subject: [PATCH 14/75] avoid exporting test helpers --- .../database_storage_integration_test.go | 3 +-- pkg/infra/distcache/distcache_test.go | 20 +++++++------------ pkg/infra/distcache/memcached_storage_test.go | 2 +- pkg/infra/distcache/memory_storage_test.go | 7 +++++++ pkg/infra/distcache/redis_storage_test.go | 2 +- 5 files changed, 17 insertions(+), 17 deletions(-) create mode 100644 pkg/infra/distcache/memory_storage_test.go diff --git a/pkg/infra/distcache/database_storage_integration_test.go b/pkg/infra/distcache/database_storage_integration_test.go index b8f564f9710..fac430e7e8d 100644 --- a/pkg/infra/distcache/database_storage_integration_test.go +++ b/pkg/infra/distcache/database_storage_integration_test.go @@ -3,6 +3,5 @@ package distcache import "testing" func TestIntegrationDatabaseCacheStorage(t *testing.T) { - - RunTestsForClient(t, createTestClient(t, "database")) + runTestsForClient(t, createTestClient(t, "database")) } diff --git a/pkg/infra/distcache/distcache_test.go b/pkg/infra/distcache/distcache_test.go index a40066b788f..33a6d2c9c7b 100644 --- a/pkg/infra/distcache/distcache_test.go +++ b/pkg/infra/distcache/distcache_test.go @@ -26,19 +26,13 @@ func createTestClient(t *testing.T, name string) cacheStorage { return createClient(CacheOpts{name: name}, sqlstore) } -func TestMemoryStorageClient(t *testing.T) { - - client := createTestClient(t, "memory") - RunTestsForClient(t, client) +func runTestsForClient(t *testing.T, client cacheStorage) { + canPutGetAndDeleteCachedObjects(t, client) + canNotFetchExpiredItems(t, client) + canSetInfiniteCacheExpiration(t, client) } -func RunTestsForClient(t *testing.T, client cacheStorage) { - CanPutGetAndDeleteCachedObjects(t, client) - CanNotFetchExpiredItems(t, client) - CanSetInfiniteCacheExpiration(t, client) -} - -func CanPutGetAndDeleteCachedObjects(t *testing.T, client cacheStorage) { +func canPutGetAndDeleteCachedObjects(t *testing.T, client cacheStorage) { cacheableStruct := CacheableStruct{String: "hej", Int64: 2000} err := client.Put("key", cacheableStruct, 0) @@ -58,7 +52,7 @@ func CanPutGetAndDeleteCachedObjects(t *testing.T, client cacheStorage) { assert.Equal(t, err, ErrCacheItemNotFound) } -func CanNotFetchExpiredItems(t *testing.T, client cacheStorage) { +func canNotFetchExpiredItems(t *testing.T, client cacheStorage) { cacheableStruct := CacheableStruct{String: "hej", Int64: 2000} err := client.Put("key", cacheableStruct, time.Second) @@ -72,7 +66,7 @@ func CanNotFetchExpiredItems(t *testing.T, client cacheStorage) { assert.Equal(t, err, ErrCacheItemNotFound) } -func CanSetInfiniteCacheExpiration(t *testing.T, client cacheStorage) { +func canSetInfiniteCacheExpiration(t *testing.T, client cacheStorage) { cacheableStruct := CacheableStruct{String: "hej", Int64: 2000} // insert cache item one day back diff --git a/pkg/infra/distcache/memcached_storage_test.go b/pkg/infra/distcache/memcached_storage_test.go index 524a4fcea10..de784730e4a 100644 --- a/pkg/infra/distcache/memcached_storage_test.go +++ b/pkg/infra/distcache/memcached_storage_test.go @@ -3,5 +3,5 @@ package distcache import "testing" func TestMemcachedCacheStorage(t *testing.T) { - RunTestsForClient(t, createTestClient(t, "memcache")) + runTestsForClient(t, createTestClient(t, "memcache")) } diff --git a/pkg/infra/distcache/memory_storage_test.go b/pkg/infra/distcache/memory_storage_test.go new file mode 100644 index 00000000000..cbf4c3790af --- /dev/null +++ b/pkg/infra/distcache/memory_storage_test.go @@ -0,0 +1,7 @@ +package distcache + +import "testing" + +func TestMemoryCacheStorage(t *testing.T) { + runTestsForClient(t, createTestClient(t, "memory")) +} diff --git a/pkg/infra/distcache/redis_storage_test.go b/pkg/infra/distcache/redis_storage_test.go index 6ba093a205c..b33d2b22e53 100644 --- a/pkg/infra/distcache/redis_storage_test.go +++ b/pkg/infra/distcache/redis_storage_test.go @@ -3,5 +3,5 @@ package distcache import "testing" func TestRedisCacheStorage(t *testing.T) { - RunTestsForClient(t, createTestClient(t, "redis")) + runTestsForClient(t, createTestClient(t, "redis")) } From 196cdf97106f1ed8c3d20d11eca17e2286a6a70a Mon Sep 17 00:00:00 2001 From: bergquist Date: Sun, 3 Mar 2019 21:48:00 +0100 Subject: [PATCH 15/75] adds config to default settings --- conf/defaults.ini | 12 ++++++++ .../database_storage_integration_test.go | 7 ----- pkg/infra/distcache/distcache.go | 27 +++++++---------- pkg/infra/distcache/distcache_test.go | 30 +++++++++++++++++-- pkg/infra/distcache/memcached_storage.go | 5 ++-- pkg/infra/distcache/memcached_storage_test.go | 9 ++++-- pkg/infra/distcache/memory_storage_test.go | 9 ++++-- pkg/infra/distcache/redis_storage.go | 9 ++++-- pkg/infra/distcache/redis_storage_test.go | 10 +++++-- pkg/setting/setting.go | 16 ++++++++++ 10 files changed, 97 insertions(+), 37 deletions(-) delete mode 100644 pkg/infra/distcache/database_storage_integration_test.go diff --git a/conf/defaults.ini b/conf/defaults.ini index df02e01235b..d77f980f806 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -106,6 +106,18 @@ path = grafana.db # For "sqlite3" only. cache mode setting used for connecting to the database cache_mode = private +#################################### Cache server ############################# +[cache_server] +# Either "memory", "redis", "memcache" or "database" default is "database" +type = database + +# cache connectionstring options +# memory: no config required. Should only be used on single install grafana. +# database: will use Grafana primary database. +# redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=grafana` +# memcache: 127.0.0.1:11211 +connstr = + #################################### Session ############################# [session] # Either "memory", "file", "redis", "mysql", "postgres", "memcache", default is "file" diff --git a/pkg/infra/distcache/database_storage_integration_test.go b/pkg/infra/distcache/database_storage_integration_test.go deleted file mode 100644 index fac430e7e8d..00000000000 --- a/pkg/infra/distcache/database_storage_integration_test.go +++ /dev/null @@ -1,7 +0,0 @@ -package distcache - -import "testing" - -func TestIntegrationDatabaseCacheStorage(t *testing.T) { - runTestsForClient(t, createTestClient(t, "database")) -} diff --git a/pkg/infra/distcache/distcache.go b/pkg/infra/distcache/distcache.go index d21ada1e6a3..ee824ae4c52 100644 --- a/pkg/infra/distcache/distcache.go +++ b/pkg/infra/distcache/distcache.go @@ -6,9 +6,10 @@ import ( "errors" "time" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/services/sqlstore" - redis "gopkg.in/redis.v2" "github.com/grafana/grafana/pkg/registry" ) @@ -25,30 +26,21 @@ func init() { func (ds *DistributedCache) Init() error { ds.log = log.New("distributed.cache") - ds.Client = createClient(CacheOpts{}, ds.SQLStore) + ds.Client = createClient(ds.Cfg.CacheOptions, ds.SQLStore) return nil } -type CacheOpts struct { - name string -} - -func createClient(opts CacheOpts, sqlstore *sqlstore.SqlStore) cacheStorage { - if opts.name == "redis" { - opt := &redis.Options{ - Network: "tcp", - Addr: "localhost:6379", - } - - return newRedisStorage(redis.NewClient(opt)) +func createClient(opts *setting.CacheOpts, sqlstore *sqlstore.SqlStore) cacheStorage { + if opts.Name == "redis" { + return newRedisStorage(opts) } - if opts.name == "memcache" { - return newMemcacheStorage("localhost:11211") + if opts.Name == "memcache" { + return newMemcacheStorage(opts) } - if opts.name == "memory" { + if opts.Name == "memory" { return newMemoryStorage() } @@ -60,6 +52,7 @@ type DistributedCache struct { log log.Logger Client cacheStorage SQLStore *sqlstore.SqlStore `inject:""` + Cfg *setting.Cfg `inject:""` } type cachedItem struct { diff --git a/pkg/infra/distcache/distcache_test.go b/pkg/infra/distcache/distcache_test.go index 33a6d2c9c7b..f6ed13d4f06 100644 --- a/pkg/infra/distcache/distcache_test.go +++ b/pkg/infra/distcache/distcache_test.go @@ -8,6 +8,7 @@ import ( "github.com/bmizerany/assert" "github.com/grafana/grafana/pkg/services/sqlstore" + "github.com/grafana/grafana/pkg/setting" ) type CacheableStruct struct { @@ -19,11 +20,34 @@ func init() { gob.Register(CacheableStruct{}) } -func createTestClient(t *testing.T, name string) cacheStorage { +func createTestClient(t *testing.T, opts *setting.CacheOpts, sqlstore *sqlstore.SqlStore) cacheStorage { t.Helper() - sqlstore := sqlstore.InitTestDB(t) - return createClient(CacheOpts{name: name}, sqlstore) + dc := &DistributedCache{ + SQLStore: sqlstore, + Cfg: &setting.Cfg{ + CacheOptions: opts, + }, + } + + err := dc.Init() + if err != nil { + t.Fatalf("failed to init client for test. error: %v", err) + } + + return dc.Client +} + +func TestCachedBasedOnConfig(t *testing.T) { + + cfg := setting.NewCfg() + cfg.Load(&setting.CommandLineArgs{ + HomePath: "../../../", + }) + + client := createTestClient(t, cfg.CacheOptions, sqlstore.InitTestDB(t)) + + runTestsForClient(t, client) } func runTestsForClient(t *testing.T, client cacheStorage) { diff --git a/pkg/infra/distcache/memcached_storage.go b/pkg/infra/distcache/memcached_storage.go index 7f97a043628..df1346bf350 100644 --- a/pkg/infra/distcache/memcached_storage.go +++ b/pkg/infra/distcache/memcached_storage.go @@ -4,15 +4,16 @@ import ( "time" "github.com/bradfitz/gomemcache/memcache" + "github.com/grafana/grafana/pkg/setting" ) type memcacheStorage struct { c *memcache.Client } -func newMemcacheStorage(connStr string) *memcacheStorage { +func newMemcacheStorage(opts *setting.CacheOpts) *memcacheStorage { return &memcacheStorage{ - c: memcache.New(connStr), + c: memcache.New(opts.ConnStr), } } diff --git a/pkg/infra/distcache/memcached_storage_test.go b/pkg/infra/distcache/memcached_storage_test.go index de784730e4a..3f885700cb4 100644 --- a/pkg/infra/distcache/memcached_storage_test.go +++ b/pkg/infra/distcache/memcached_storage_test.go @@ -1,7 +1,12 @@ package distcache -import "testing" +import ( + "testing" + + "github.com/grafana/grafana/pkg/setting" +) func TestMemcachedCacheStorage(t *testing.T) { - runTestsForClient(t, createTestClient(t, "memcache")) + opts := &setting.CacheOpts{Name: "memcache", ConnStr: "localhost:11211"} + runTestsForClient(t, createTestClient(t, opts, nil)) } diff --git a/pkg/infra/distcache/memory_storage_test.go b/pkg/infra/distcache/memory_storage_test.go index cbf4c3790af..5318b7c19b8 100644 --- a/pkg/infra/distcache/memory_storage_test.go +++ b/pkg/infra/distcache/memory_storage_test.go @@ -1,7 +1,12 @@ package distcache -import "testing" +import ( + "testing" + + "github.com/grafana/grafana/pkg/setting" +) func TestMemoryCacheStorage(t *testing.T) { - runTestsForClient(t, createTestClient(t, "memory")) + opts := &setting.CacheOpts{Name: "memory"} + runTestsForClient(t, createTestClient(t, opts, nil)) } diff --git a/pkg/infra/distcache/redis_storage.go b/pkg/infra/distcache/redis_storage.go index bb21b26473e..4e6a8b6d325 100644 --- a/pkg/infra/distcache/redis_storage.go +++ b/pkg/infra/distcache/redis_storage.go @@ -3,6 +3,7 @@ package distcache import ( "time" + "github.com/grafana/grafana/pkg/setting" redis "gopkg.in/redis.v2" ) @@ -10,8 +11,12 @@ type redisStorage struct { c *redis.Client } -func newRedisStorage(c *redis.Client) *redisStorage { - return &redisStorage{c: c} +func newRedisStorage(opts *setting.CacheOpts) *redisStorage { + opt := &redis.Options{ + Network: "tcp", + Addr: opts.ConnStr, + } + return &redisStorage{c: redis.NewClient(opt)} } // Set sets value to given key in session. diff --git a/pkg/infra/distcache/redis_storage_test.go b/pkg/infra/distcache/redis_storage_test.go index b33d2b22e53..7c63ce46b38 100644 --- a/pkg/infra/distcache/redis_storage_test.go +++ b/pkg/infra/distcache/redis_storage_test.go @@ -1,7 +1,13 @@ package distcache -import "testing" +import ( + "testing" + + "github.com/grafana/grafana/pkg/setting" +) func TestRedisCacheStorage(t *testing.T) { - runTestsForClient(t, createTestClient(t, "redis")) + + opts := &setting.CacheOpts{Name: "redis", ConnStr: "localhost:6379"} + runTestsForClient(t, createTestClient(t, opts, nil)) } diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index 5d44a3585dc..f25f2211b40 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -240,6 +240,9 @@ type Cfg struct { // User EditorsCanOwn bool + + // DistributedCache + CacheOptions *CacheOpts } type CommandLineArgs struct { @@ -779,9 +782,22 @@ func (cfg *Cfg) Load(args *CommandLineArgs) error { enterprise := iniFile.Section("enterprise") cfg.EnterpriseLicensePath = enterprise.Key("license_path").MustString(filepath.Join(cfg.DataPath, "license.jwt")) + cacheServer := iniFile.Section("cache_server") + //cfg.DistCacheType = cacheServer.Key("type").MustString("database") + //cfg.DistCacheConnStr = cacheServer.Key("connstr").MustString("") + cfg.CacheOptions = &CacheOpts{ + Name: cacheServer.Key("type").MustString("database"), + ConnStr: cacheServer.Key("connstr").MustString(""), + } + return nil } +type CacheOpts struct { + Name string + ConnStr string +} + func (cfg *Cfg) readSessionConfig() { sec := cfg.Raw.Section("session") SessionOptions = session.Options{} From b933b4efc8a9dcd9f73e00d063b806d3d429a640 Mon Sep 17 00:00:00 2001 From: bergquist Date: Sun, 3 Mar 2019 22:04:11 +0100 Subject: [PATCH 16/75] test redis and memcached during integration tests --- ...ed_storage_test.go => memcached_storage_integration_test.go} | 2 ++ ...{redis_storage_test.go => redis_storage_integration_test.go} | 2 ++ 2 files changed, 4 insertions(+) rename pkg/infra/distcache/{memcached_storage_test.go => memcached_storage_integration_test.go} (92%) rename pkg/infra/distcache/{redis_storage_test.go => redis_storage_integration_test.go} (93%) diff --git a/pkg/infra/distcache/memcached_storage_test.go b/pkg/infra/distcache/memcached_storage_integration_test.go similarity index 92% rename from pkg/infra/distcache/memcached_storage_test.go rename to pkg/infra/distcache/memcached_storage_integration_test.go index 3f885700cb4..128abb6923f 100644 --- a/pkg/infra/distcache/memcached_storage_test.go +++ b/pkg/infra/distcache/memcached_storage_integration_test.go @@ -1,3 +1,5 @@ +// +build memcached + package distcache import ( diff --git a/pkg/infra/distcache/redis_storage_test.go b/pkg/infra/distcache/redis_storage_integration_test.go similarity index 93% rename from pkg/infra/distcache/redis_storage_test.go rename to pkg/infra/distcache/redis_storage_integration_test.go index 7c63ce46b38..289a3ff4e2d 100644 --- a/pkg/infra/distcache/redis_storage_test.go +++ b/pkg/infra/distcache/redis_storage_integration_test.go @@ -1,3 +1,5 @@ +// +build redis + package distcache import ( From 995647be2c99224ffa60cb5f572e649b11ad0530 Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 5 Mar 2019 14:22:22 +0100 Subject: [PATCH 17/75] removes memory as distcache option if database caching is to expensive if should not use distcache in the first place. --- pkg/infra/distcache/distcache.go | 4 --- pkg/infra/distcache/memory_storage.go | 35 ---------------------- pkg/infra/distcache/memory_storage_test.go | 12 -------- 3 files changed, 51 deletions(-) delete mode 100644 pkg/infra/distcache/memory_storage.go delete mode 100644 pkg/infra/distcache/memory_storage_test.go diff --git a/pkg/infra/distcache/distcache.go b/pkg/infra/distcache/distcache.go index ee824ae4c52..44ab2e08583 100644 --- a/pkg/infra/distcache/distcache.go +++ b/pkg/infra/distcache/distcache.go @@ -40,10 +40,6 @@ func createClient(opts *setting.CacheOpts, sqlstore *sqlstore.SqlStore) cacheSto return newMemcacheStorage(opts) } - if opts.Name == "memory" { - return newMemoryStorage() - } - return newDatabaseCache(sqlstore) } diff --git a/pkg/infra/distcache/memory_storage.go b/pkg/infra/distcache/memory_storage.go deleted file mode 100644 index a1203cabe75..00000000000 --- a/pkg/infra/distcache/memory_storage.go +++ /dev/null @@ -1,35 +0,0 @@ -package distcache - -import ( - "time" - - gocache "github.com/patrickmn/go-cache" -) - -type memoryStorage struct { - c *gocache.Cache -} - -func newMemoryStorage() *memoryStorage { - return &memoryStorage{ - c: gocache.New(time.Minute*30, time.Minute*30), - } -} - -func (s *memoryStorage) Put(key string, val interface{}, expires time.Duration) error { - return s.c.Add(key, val, expires) -} - -func (s *memoryStorage) Get(key string) (interface{}, error) { - val, exist := s.c.Get(key) - if !exist { - return nil, ErrCacheItemNotFound - } - - return val, nil -} - -func (s *memoryStorage) Delete(key string) error { - s.c.Delete(key) - return nil -} diff --git a/pkg/infra/distcache/memory_storage_test.go b/pkg/infra/distcache/memory_storage_test.go deleted file mode 100644 index 5318b7c19b8..00000000000 --- a/pkg/infra/distcache/memory_storage_test.go +++ /dev/null @@ -1,12 +0,0 @@ -package distcache - -import ( - "testing" - - "github.com/grafana/grafana/pkg/setting" -) - -func TestMemoryCacheStorage(t *testing.T) { - opts := &setting.CacheOpts{Name: "memory"} - runTestsForClient(t, createTestClient(t, opts, nil)) -} From 98f54326595f861867aca27f44c7af997f653b72 Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 5 Mar 2019 14:35:36 +0100 Subject: [PATCH 18/75] `memcache` -> `memcached` https://github.com/memcached/memcached --- conf/defaults.ini | 2 +- pkg/infra/distcache/distcache.go | 4 ++-- pkg/infra/distcache/memcached_storage.go | 12 ++++++------ .../distcache/memcached_storage_integration_test.go | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/conf/defaults.ini b/conf/defaults.ini index d77f980f806..3386e552b8a 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -108,7 +108,7 @@ cache_mode = private #################################### Cache server ############################# [cache_server] -# Either "memory", "redis", "memcache" or "database" default is "database" +# Either "memory", "redis", "memcached" or "database" default is "database" type = database # cache connectionstring options diff --git a/pkg/infra/distcache/distcache.go b/pkg/infra/distcache/distcache.go index 44ab2e08583..c293b62f608 100644 --- a/pkg/infra/distcache/distcache.go +++ b/pkg/infra/distcache/distcache.go @@ -36,8 +36,8 @@ func createClient(opts *setting.CacheOpts, sqlstore *sqlstore.SqlStore) cacheSto return newRedisStorage(opts) } - if opts.Name == "memcache" { - return newMemcacheStorage(opts) + if opts.Name == "memcached" { + return newMemcachedStorage(opts) } return newDatabaseCache(sqlstore) diff --git a/pkg/infra/distcache/memcached_storage.go b/pkg/infra/distcache/memcached_storage.go index df1346bf350..ea326d759b7 100644 --- a/pkg/infra/distcache/memcached_storage.go +++ b/pkg/infra/distcache/memcached_storage.go @@ -7,12 +7,12 @@ import ( "github.com/grafana/grafana/pkg/setting" ) -type memcacheStorage struct { +type memcachedStorage struct { c *memcache.Client } -func newMemcacheStorage(opts *setting.CacheOpts) *memcacheStorage { - return &memcacheStorage{ +func newMemcachedStorage(opts *setting.CacheOpts) *memcachedStorage { + return &memcachedStorage{ c: memcache.New(opts.ConnStr), } } @@ -26,7 +26,7 @@ func newItem(sid string, data []byte, expire int32) *memcache.Item { } // Put sets value to given key in the cache. -func (s *memcacheStorage) Put(key string, val interface{}, expires time.Duration) error { +func (s *memcachedStorage) Put(key string, val interface{}, expires time.Duration) error { item := &cachedItem{Val: val} bytes, err := encodeGob(item) @@ -40,7 +40,7 @@ func (s *memcacheStorage) Put(key string, val interface{}, expires time.Duration } // Get gets value by given key in the cache. -func (s *memcacheStorage) Get(key string) (interface{}, error) { +func (s *memcachedStorage) Get(key string) (interface{}, error) { i, err := s.c.Get(key) if err != nil && err.Error() == "memcache: cache miss" { @@ -62,6 +62,6 @@ func (s *memcacheStorage) Get(key string) (interface{}, error) { } // Delete delete a key from the cache -func (s *memcacheStorage) Delete(key string) error { +func (s *memcachedStorage) Delete(key string) error { return s.c.Delete(key) } diff --git a/pkg/infra/distcache/memcached_storage_integration_test.go b/pkg/infra/distcache/memcached_storage_integration_test.go index 128abb6923f..125bf8d2bf1 100644 --- a/pkg/infra/distcache/memcached_storage_integration_test.go +++ b/pkg/infra/distcache/memcached_storage_integration_test.go @@ -9,6 +9,6 @@ import ( ) func TestMemcachedCacheStorage(t *testing.T) { - opts := &setting.CacheOpts{Name: "memcache", ConnStr: "localhost:11211"} + opts := &setting.CacheOpts{Name: "memcached", ConnStr: "localhost:11211"} runTestsForClient(t, createTestClient(t, opts, nil)) } From 6231095f72b0305a50b8d7e926b17db0df7a69eb Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 5 Mar 2019 14:57:45 +0100 Subject: [PATCH 19/75] reverts package.json I made during the flight >.> --- package.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/package.json b/package.json index af270d47ad0..a937ba6f717 100644 --- a/package.json +++ b/package.json @@ -142,6 +142,11 @@ "gui:release": "ts-node --project ./scripts/cli/tsconfig.json ./scripts/cli/index.ts gui:release -p", "cli": "ts-node --project ./scripts/cli/tsconfig.json ./scripts/cli/index.ts" }, + "husky": { + "hooks": { + "pre-commit": "lint-staged && grunt precommit" + } + }, "lint-staged": { "*.{ts,tsx,json,scss}": [ "prettier --write", From 9a78c231653bd3b4fb6b412ffa8d9dc6de06778a Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 5 Mar 2019 15:15:05 +0100 Subject: [PATCH 20/75] rename put -> set --- pkg/infra/distcache/database_storage.go | 2 +- pkg/infra/distcache/database_storage_test.go | 10 +++++----- pkg/infra/distcache/distcache.go | 14 +++++++++----- pkg/infra/distcache/distcache_test.go | 16 ++++++++-------- pkg/infra/distcache/memcached_storage.go | 4 ++-- pkg/infra/distcache/redis_storage.go | 2 +- 6 files changed, 26 insertions(+), 22 deletions(-) diff --git a/pkg/infra/distcache/database_storage.go b/pkg/infra/distcache/database_storage.go index f4365383b82..0cf613471db 100644 --- a/pkg/infra/distcache/database_storage.go +++ b/pkg/infra/distcache/database_storage.go @@ -79,7 +79,7 @@ type cacheData struct { CreatedAt int64 } -func (dc *databaseCache) Put(key string, value interface{}, expire time.Duration) error { +func (dc *databaseCache) Set(key string, value interface{}, expire time.Duration) error { item := &cachedItem{Val: value} data, err := encodeGob(item) if err != nil { diff --git a/pkg/infra/distcache/database_storage_test.go b/pkg/infra/distcache/database_storage_test.go index 931fbc81c7f..24d8cea16bb 100644 --- a/pkg/infra/distcache/database_storage_test.go +++ b/pkg/infra/distcache/database_storage_test.go @@ -22,15 +22,15 @@ func TestDatabaseStorageGarbageCollection(t *testing.T) { //set time.now to 2 weeks ago getTime = func() time.Time { return time.Now().AddDate(0, 0, -2) } - db.Put("key1", obj, 1000*time.Second) - db.Put("key2", obj, 1000*time.Second) - db.Put("key3", obj, 1000*time.Second) + db.Set("key1", obj, 1000*time.Second) + db.Set("key2", obj, 1000*time.Second) + db.Set("key3", obj, 1000*time.Second) // insert object that should never expire - db.Put("key4", obj, 0) + db.Set("key4", obj, 0) getTime = time.Now - db.Put("key5", obj, 1000*time.Second) + db.Set("key5", obj, 1000*time.Second) //run GC db.internalRunGC() diff --git a/pkg/infra/distcache/distcache.go b/pkg/infra/distcache/distcache.go index c293b62f608..549774b848b 100644 --- a/pkg/infra/distcache/distcache.go +++ b/pkg/infra/distcache/distcache.go @@ -31,7 +31,7 @@ func (ds *DistributedCache) Init() error { return nil } -func createClient(opts *setting.CacheOpts, sqlstore *sqlstore.SqlStore) cacheStorage { +func createClient(opts *setting.CacheOpts, sqlstore *sqlstore.SqlStore) CacheStorage { if opts.Name == "redis" { return newRedisStorage(opts) } @@ -46,7 +46,7 @@ func createClient(opts *setting.CacheOpts, sqlstore *sqlstore.SqlStore) cacheSto // DistributedCache allows Grafana to cache data outside its own process type DistributedCache struct { log log.Logger - Client cacheStorage + Client CacheStorage SQLStore *sqlstore.SqlStore `inject:""` Cfg *setting.Cfg `inject:""` } @@ -66,12 +66,16 @@ func decodeGob(data []byte, out *cachedItem) error { return gob.NewDecoder(buf).Decode(&out) } -type cacheStorage interface { +// CacheStorage allows the caller to set, get and delete items in the cache. +// Cached items are stored as byte arrays and marshalled using "encoding/gob" +// so any struct added to the cache needs to be registred with `gob.Register` +// ex `gob.Register(CacheableStruct{})`` +type CacheStorage interface { // Get reads object from Cache Get(key string) (interface{}, error) - // Puts an object into the cache - Put(key string, value interface{}, expire time.Duration) error + // Set sets an object into the cache + Set(key string, value interface{}, expire time.Duration) error // Delete object from cache Delete(key string) error diff --git a/pkg/infra/distcache/distcache_test.go b/pkg/infra/distcache/distcache_test.go index f6ed13d4f06..a4a596fd930 100644 --- a/pkg/infra/distcache/distcache_test.go +++ b/pkg/infra/distcache/distcache_test.go @@ -20,7 +20,7 @@ func init() { gob.Register(CacheableStruct{}) } -func createTestClient(t *testing.T, opts *setting.CacheOpts, sqlstore *sqlstore.SqlStore) cacheStorage { +func createTestClient(t *testing.T, opts *setting.CacheOpts, sqlstore *sqlstore.SqlStore) CacheStorage { t.Helper() dc := &DistributedCache{ @@ -50,16 +50,16 @@ func TestCachedBasedOnConfig(t *testing.T) { runTestsForClient(t, client) } -func runTestsForClient(t *testing.T, client cacheStorage) { +func runTestsForClient(t *testing.T, client CacheStorage) { canPutGetAndDeleteCachedObjects(t, client) canNotFetchExpiredItems(t, client) canSetInfiniteCacheExpiration(t, client) } -func canPutGetAndDeleteCachedObjects(t *testing.T, client cacheStorage) { +func canPutGetAndDeleteCachedObjects(t *testing.T, client CacheStorage) { cacheableStruct := CacheableStruct{String: "hej", Int64: 2000} - err := client.Put("key", cacheableStruct, 0) + err := client.Set("key", cacheableStruct, 0) assert.Equal(t, err, nil) data, err := client.Get("key") @@ -76,10 +76,10 @@ func canPutGetAndDeleteCachedObjects(t *testing.T, client cacheStorage) { assert.Equal(t, err, ErrCacheItemNotFound) } -func canNotFetchExpiredItems(t *testing.T, client cacheStorage) { +func canNotFetchExpiredItems(t *testing.T, client CacheStorage) { cacheableStruct := CacheableStruct{String: "hej", Int64: 2000} - err := client.Put("key", cacheableStruct, time.Second) + err := client.Set("key", cacheableStruct, time.Second) assert.Equal(t, err, nil) //not sure how this can be avoided when testing redis/memcached :/ @@ -90,12 +90,12 @@ func canNotFetchExpiredItems(t *testing.T, client cacheStorage) { assert.Equal(t, err, ErrCacheItemNotFound) } -func canSetInfiniteCacheExpiration(t *testing.T, client cacheStorage) { +func canSetInfiniteCacheExpiration(t *testing.T, client CacheStorage) { cacheableStruct := CacheableStruct{String: "hej", Int64: 2000} // insert cache item one day back getTime = func() time.Time { return time.Now().AddDate(0, 0, -2) } - err := client.Put("key", cacheableStruct, 0) + err := client.Set("key", cacheableStruct, 0) assert.Equal(t, err, nil) // should not be able to read that value since its expired diff --git a/pkg/infra/distcache/memcached_storage.go b/pkg/infra/distcache/memcached_storage.go index ea326d759b7..7a29eec0e5d 100644 --- a/pkg/infra/distcache/memcached_storage.go +++ b/pkg/infra/distcache/memcached_storage.go @@ -25,8 +25,8 @@ func newItem(sid string, data []byte, expire int32) *memcache.Item { } } -// Put sets value to given key in the cache. -func (s *memcachedStorage) Put(key string, val interface{}, expires time.Duration) error { +// Set sets value to given key in the cache. +func (s *memcachedStorage) Set(key string, val interface{}, expires time.Duration) error { item := &cachedItem{Val: val} bytes, err := encodeGob(item) diff --git a/pkg/infra/distcache/redis_storage.go b/pkg/infra/distcache/redis_storage.go index 4e6a8b6d325..1414671f05b 100644 --- a/pkg/infra/distcache/redis_storage.go +++ b/pkg/infra/distcache/redis_storage.go @@ -20,7 +20,7 @@ func newRedisStorage(opts *setting.CacheOpts) *redisStorage { } // Set sets value to given key in session. -func (s *redisStorage) Put(key string, val interface{}, expires time.Duration) error { +func (s *redisStorage) Set(key string, val interface{}, expires time.Duration) error { item := &cachedItem{Val: val} value, err := encodeGob(item) if err != nil { From daa3b17951f3c149ecb8434a61a86b4422749589 Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 5 Mar 2019 15:34:51 +0100 Subject: [PATCH 21/75] code layouts and comments --- conf/defaults.ini | 3 +- pkg/cmd/grafana-server/server.go | 1 + pkg/infra/distcache/database_storage.go | 56 +++++++++++---------- pkg/infra/distcache/distcache.go | 63 ++++++++++++++++-------- pkg/infra/distcache/distcache_test.go | 3 +- pkg/infra/distcache/memcached_storage.go | 11 ++--- pkg/setting/setting.go | 2 - 7 files changed, 79 insertions(+), 60 deletions(-) diff --git a/conf/defaults.ini b/conf/defaults.ini index 3386e552b8a..91a58243c04 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -108,11 +108,10 @@ cache_mode = private #################################### Cache server ############################# [cache_server] -# Either "memory", "redis", "memcached" or "database" default is "database" +# Either "redis", "memcached" or "database" default is "database" type = database # cache connectionstring options -# memory: no config required. Should only be used on single install grafana. # database: will use Grafana primary database. # redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=grafana` # memcache: 127.0.0.1:11211 diff --git a/pkg/cmd/grafana-server/server.go b/pkg/cmd/grafana-server/server.go index 53218147ae0..d2852e0b8ca 100644 --- a/pkg/cmd/grafana-server/server.go +++ b/pkg/cmd/grafana-server/server.go @@ -28,6 +28,7 @@ import ( // self registering services _ "github.com/grafana/grafana/pkg/extensions" + _ "github.com/grafana/grafana/pkg/infra/distcache" _ "github.com/grafana/grafana/pkg/infra/metrics" _ "github.com/grafana/grafana/pkg/infra/serverlock" _ "github.com/grafana/grafana/pkg/infra/tracing" diff --git a/pkg/infra/distcache/database_storage.go b/pkg/infra/distcache/database_storage.go index 0cf613471db..6a357005a21 100644 --- a/pkg/infra/distcache/database_storage.go +++ b/pkg/infra/distcache/database_storage.go @@ -1,6 +1,7 @@ package distcache import ( + "context" "time" "github.com/grafana/grafana/pkg/log" @@ -18,32 +19,33 @@ func newDatabaseCache(sqlstore *sqlstore.SqlStore) *databaseCache { log: log.New("distcache.database"), } - //go dc.StartGC() //TODO: start the GC somehow return dc } +func (dc *databaseCache) Run(ctx context.Context) error { + ticker := time.NewTicker(time.Minute * 10) + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + dc.internalRunGC() + } + } +} + var getTime = time.Now func (dc *databaseCache) internalRunGC() { now := getTime().Unix() - sql := `DELETE FROM cache_data WHERE (? - created) >= expire` + sql := `DELETE FROM cache_data WHERE (? - created_at) >= expires AND expires <> 0` - //EXTRACT(EPOCH FROM NOW()) - created >= expire - //UNIX_TIMESTAMP(NOW()) - created >= expire _, err := dc.SQLStore.NewSession().Exec(sql, now) if err != nil { dc.log.Error("failed to run garbage collect", "error", err) } } -func (dc *databaseCache) StartGC() { - dc.internalRunGC() - - time.AfterFunc(time.Second*10, func() { - dc.StartGC() - }) -} - func (dc *databaseCache) Get(key string) (interface{}, error) { cacheHits := []cacheData{} err := dc.SQLStore.NewSession().Where(`key = ?`, key).Find(&cacheHits) @@ -57,8 +59,10 @@ func (dc *databaseCache) Get(key string) (interface{}, error) { } cacheHit = cacheHits[0] + // if Expires is set. Make sure its still valid. if cacheHit.Expires > 0 { - if getTime().Unix()-cacheHit.CreatedAt >= cacheHit.Expires { + existedButExpired := getTime().Unix()-cacheHit.CreatedAt >= cacheHit.Expires + if existedButExpired { dc.Delete(key) return nil, ErrCacheItemNotFound } @@ -72,13 +76,6 @@ func (dc *databaseCache) Get(key string) (interface{}, error) { return item.Val, nil } -type cacheData struct { - Key string - Data []byte - Expires int64 - CreatedAt int64 -} - func (dc *databaseCache) Set(key string, value interface{}, expire time.Duration) error { item := &cachedItem{Val: value} data, err := encodeGob(item) @@ -87,22 +84,23 @@ func (dc *databaseCache) Set(key string, value interface{}, expire time.Duration } now := getTime().Unix() - cacheHits := []cacheData{} err = dc.SQLStore.NewSession().Where(`key = ?`, key).Find(&cacheHits) if err != nil { return err } - var expiresInEpoch int64 + var expiresAtEpoch int64 if expire != 0 { - expiresInEpoch = int64(expire) / int64(time.Second) + expiresAtEpoch = int64(expire) / int64(time.Second) } + session := dc.SQLStore.NewSession() + // insert or update depending on if item already exist if len(cacheHits) > 0 { - _, err = dc.SQLStore.NewSession().Exec("UPDATE cache_data SET data=?, created=?, expire=? WHERE key=?", data, now, expiresInEpoch, key) + _, err = session.Exec("UPDATE cache_data SET data=?, created=?, expire=? WHERE key=?", data, now, expiresAtEpoch, key) } else { - _, err = dc.SQLStore.NewSession().Exec("INSERT INTO cache_data(key,data,created_at,expires) VALUES(?,?,?,?)", key, data, now, expiresInEpoch) + _, err = session.Exec("INSERT INTO cache_data(key,data,created_at,expires) VALUES(?,?,?,?)", key, data, now, expiresAtEpoch) } return err @@ -110,8 +108,14 @@ func (dc *databaseCache) Set(key string, value interface{}, expire time.Duration func (dc *databaseCache) Delete(key string) error { sql := `DELETE FROM cache_data WHERE key = ?` - _, err := dc.SQLStore.NewSession().Exec(sql, key) return err } + +type cacheData struct { + Key string + Data []byte + Expires int64 + CreatedAt int64 +} diff --git a/pkg/infra/distcache/distcache.go b/pkg/infra/distcache/distcache.go index 549774b848b..a8f12adaa27 100644 --- a/pkg/infra/distcache/distcache.go +++ b/pkg/infra/distcache/distcache.go @@ -2,6 +2,7 @@ package distcache import ( "bytes" + "context" "encoding/gob" "errors" "time" @@ -22,6 +23,29 @@ func init() { registry.RegisterService(&DistributedCache{}) } +// CacheStorage allows the caller to set, get and delete items in the cache. +// Cached items are stored as byte arrays and marshalled using "encoding/gob" +// so any struct added to the cache needs to be registred with `distcache.Register` +// ex `distcache.Register(CacheableStruct{})`` +type CacheStorage interface { + // Get reads object from Cache + Get(key string) (interface{}, error) + + // Set sets an object into the cache + Set(key string, value interface{}, expire time.Duration) error + + // Delete object from cache + Delete(key string) error +} + +// DistributedCache allows Grafana to cache data outside its own process +type DistributedCache struct { + log log.Logger + Client CacheStorage + SQLStore *sqlstore.SqlStore `inject:""` + Cfg *setting.Cfg `inject:""` +} + // Init initializes the service func (ds *DistributedCache) Init() error { ds.log = log.New("distributed.cache") @@ -31,6 +55,16 @@ func (ds *DistributedCache) Init() error { return nil } +func (ds *DistributedCache) Run(ctx context.Context) error { + backgroundjob, ok := ds.Client.(registry.BackgroundService) + if ok { + return backgroundjob.Run(ctx) + } + + <-ctx.Done() + return ctx.Err() +} + func createClient(opts *setting.CacheOpts, sqlstore *sqlstore.SqlStore) CacheStorage { if opts.Name == "redis" { return newRedisStorage(opts) @@ -43,12 +77,14 @@ func createClient(opts *setting.CacheOpts, sqlstore *sqlstore.SqlStore) CacheSto return newDatabaseCache(sqlstore) } -// DistributedCache allows Grafana to cache data outside its own process -type DistributedCache struct { - log log.Logger - Client CacheStorage - SQLStore *sqlstore.SqlStore `inject:""` - Cfg *setting.Cfg `inject:""` +// Register records a type, identified by a value for that type, under its +// internal type name. That name will identify the concrete type of a value +// sent or received as an interface variable. Only types that will be +// transferred as implementations of interface values need to be registered. +// Expecting to be used only during initialization, it panics if the mapping +// between types and names is not a bijection. +func Register(value interface{}) { + gob.Register(value) } type cachedItem struct { @@ -65,18 +101,3 @@ func decodeGob(data []byte, out *cachedItem) error { buf := bytes.NewBuffer(data) return gob.NewDecoder(buf).Decode(&out) } - -// CacheStorage allows the caller to set, get and delete items in the cache. -// Cached items are stored as byte arrays and marshalled using "encoding/gob" -// so any struct added to the cache needs to be registred with `gob.Register` -// ex `gob.Register(CacheableStruct{})`` -type CacheStorage interface { - // Get reads object from Cache - Get(key string) (interface{}, error) - - // Set sets an object into the cache - Set(key string, value interface{}, expire time.Duration) error - - // Delete object from cache - Delete(key string) error -} diff --git a/pkg/infra/distcache/distcache_test.go b/pkg/infra/distcache/distcache_test.go index a4a596fd930..b631a6283ac 100644 --- a/pkg/infra/distcache/distcache_test.go +++ b/pkg/infra/distcache/distcache_test.go @@ -1,7 +1,6 @@ package distcache import ( - "encoding/gob" "testing" "time" @@ -17,7 +16,7 @@ type CacheableStruct struct { } func init() { - gob.Register(CacheableStruct{}) + Register(CacheableStruct{}) } func createTestClient(t *testing.T, opts *setting.CacheOpts, sqlstore *sqlstore.SqlStore) CacheStorage { diff --git a/pkg/infra/distcache/memcached_storage.go b/pkg/infra/distcache/memcached_storage.go index 7a29eec0e5d..998d05621c9 100644 --- a/pkg/infra/distcache/memcached_storage.go +++ b/pkg/infra/distcache/memcached_storage.go @@ -28,21 +28,18 @@ func newItem(sid string, data []byte, expire int32) *memcache.Item { // Set sets value to given key in the cache. func (s *memcachedStorage) Set(key string, val interface{}, expires time.Duration) error { item := &cachedItem{Val: val} - bytes, err := encodeGob(item) if err != nil { return err } - memcacheItem := newItem(key, bytes, int32(expires)) - - return s.c.Set(memcacheItem) + memcachedItem := newItem(key, bytes, int32(expires)) + return s.c.Set(memcachedItem) } // Get gets value by given key in the cache. func (s *memcachedStorage) Get(key string) (interface{}, error) { - i, err := s.c.Get(key) - + memcachedItem, err := s.c.Get(key) if err != nil && err.Error() == "memcache: cache miss" { return nil, ErrCacheItemNotFound } @@ -53,7 +50,7 @@ func (s *memcachedStorage) Get(key string) (interface{}, error) { item := &cachedItem{} - err = decodeGob(i.Value, item) + err = decodeGob(memcachedItem.Value, item) if err != nil { return nil, err } diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index f25f2211b40..864c29fb382 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -783,8 +783,6 @@ func (cfg *Cfg) Load(args *CommandLineArgs) error { cfg.EnterpriseLicensePath = enterprise.Key("license_path").MustString(filepath.Join(cfg.DataPath, "license.jwt")) cacheServer := iniFile.Section("cache_server") - //cfg.DistCacheType = cacheServer.Key("type").MustString("database") - //cfg.DistCacheConnStr = cacheServer.Key("connstr").MustString("") cfg.CacheOptions = &CacheOpts{ Name: cacheServer.Key("type").MustString("database"), ConnStr: cacheServer.Key("connstr").MustString(""), From dbc1315d6f69bb6ce154e5b907d1077d5301c7f7 Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 7 Mar 2019 17:16:08 +0100 Subject: [PATCH 22/75] build steps for cache servers --- .circleci/config.yml | 18 ++++++++++++++++++ scripts/circle-test-cache-servers.sh | 17 +++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100755 scripts/circle-test-cache-servers.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 69cea87dccd..9ec8b9dc05d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -56,6 +56,23 @@ jobs: name: postgres integration tests command: './scripts/circle-test-postgres.sh' + cache-server-test: + docker: + - image: circleci/golang:1.11.5 + - image: circleci/redis:4-alpine + - image: memcached + working_directory: /go/src/github.com/grafana/grafana + steps: + - checkout + #- run: sudo apt update + #- run: sudo apt install -y postgresql-client + - run: dockerize -wait tcp://127.0.0.1:11211 -timeout 120s + - run: dockerize -wait tcp://127.0.0.1:6739 -timeout 120s + #- run: 'PGPASSWORD=grafanatest psql -p 5432 -h 127.0.0.1 -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql' + - run: + name: cache server tests + command: './scripts/circle-test-cache-servers.sh' + codespell: docker: - image: circleci/python @@ -554,4 +571,5 @@ workflows: - gometalinter - mysql-integration-test - postgres-integration-test + - cache-server-test filters: *filter-not-release-or-master diff --git a/scripts/circle-test-cache-servers.sh b/scripts/circle-test-cache-servers.sh new file mode 100755 index 00000000000..6b29be15f42 --- /dev/null +++ b/scripts/circle-test-cache-servers.sh @@ -0,0 +1,17 @@ +#!/bin/bash +function exit_if_fail { + command=$@ + echo "Executing '$command'" + eval $command + rc=$? + if [ $rc -ne 0 ]; then + echo "'$command' returned $rc." + exit $rc + fi +} + +echo "running redis and memcache tests" +#set -e +#time for d in $(go list ./pkg/...); do +time exit_if_fail go test -tags="redis memcached" ./pkg/infra/distcache/... +#done From 66e71b66dd94d6a6ccafae16f8c0cb8fc1da8603 Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 7 Mar 2019 19:07:11 +0100 Subject: [PATCH 23/75] renames key to cache_key apparently key is a reserved keyword in mysql. and the error messages doesnt mention that. can I please have 6h back? --- .circleci/config.yml | 7 ++-- pkg/infra/distcache/database_storage.go | 37 ++++++++++--------- pkg/infra/distcache/database_storage_test.go | 16 +++++--- pkg/infra/distcache/distcache_test.go | 18 ++++----- .../sqlstore/migrations/cache_data_mig.go | 6 +-- scripts/circle-test-cache-servers.sh | 3 +- 6 files changed, 48 insertions(+), 39 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9ec8b9dc05d..da0e0665285 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -64,11 +64,8 @@ jobs: working_directory: /go/src/github.com/grafana/grafana steps: - checkout - #- run: sudo apt update - #- run: sudo apt install -y postgresql-client - run: dockerize -wait tcp://127.0.0.1:11211 -timeout 120s - - run: dockerize -wait tcp://127.0.0.1:6739 -timeout 120s - #- run: 'PGPASSWORD=grafanatest psql -p 5432 -h 127.0.0.1 -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql' + - run: dockerize -wait tcp://127.0.0.1:6379 -timeout 120s - run: name: cache server tests command: './scripts/circle-test-cache-servers.sh' @@ -562,6 +559,8 @@ workflows: filters: *filter-not-release-or-master - postgres-integration-test: filters: *filter-not-release-or-master + - cache-server-test: + filters: *filter-not-release-or-master - grafana-docker-pr: requires: - build diff --git a/pkg/infra/distcache/database_storage.go b/pkg/infra/distcache/database_storage.go index 6a357005a21..9883751569f 100644 --- a/pkg/infra/distcache/database_storage.go +++ b/pkg/infra/distcache/database_storage.go @@ -8,6 +8,8 @@ import ( "github.com/grafana/grafana/pkg/services/sqlstore" ) +var getTime = time.Now + type databaseCache struct { SQLStore *sqlstore.SqlStore log log.Logger @@ -34,8 +36,6 @@ func (dc *databaseCache) Run(ctx context.Context) error { } } -var getTime = time.Now - func (dc *databaseCache) internalRunGC() { now := getTime().Unix() sql := `DELETE FROM cache_data WHERE (? - created_at) >= expires AND expires <> 0` @@ -47,19 +47,20 @@ func (dc *databaseCache) internalRunGC() { } func (dc *databaseCache) Get(key string) (interface{}, error) { - cacheHits := []cacheData{} - err := dc.SQLStore.NewSession().Where(`key = ?`, key).Find(&cacheHits) + cacheHits := []CacheData{} + sess := dc.SQLStore.NewSession() + defer sess.Close() + err := sess.Where("cache_key= ?", key).Find(&cacheHits) + if err != nil { return nil, err } - var cacheHit cacheData if len(cacheHits) == 0 { return nil, ErrCacheItemNotFound } - cacheHit = cacheHits[0] - // if Expires is set. Make sure its still valid. + cacheHit := cacheHits[0] if cacheHit.Expires > 0 { existedButExpired := getTime().Unix()-cacheHit.CreatedAt >= cacheHit.Expires if existedButExpired { @@ -83,9 +84,10 @@ func (dc *databaseCache) Set(key string, value interface{}, expire time.Duration return err } - now := getTime().Unix() - cacheHits := []cacheData{} - err = dc.SQLStore.NewSession().Where(`key = ?`, key).Find(&cacheHits) + session := dc.SQLStore.NewSession() + + var cacheHit CacheData + has, err := session.Where("cache_key = ?", key).Get(&cacheHit) if err != nil { return err } @@ -95,27 +97,28 @@ func (dc *databaseCache) Set(key string, value interface{}, expire time.Duration expiresAtEpoch = int64(expire) / int64(time.Second) } - session := dc.SQLStore.NewSession() // insert or update depending on if item already exist - if len(cacheHits) > 0 { - _, err = session.Exec("UPDATE cache_data SET data=?, created=?, expire=? WHERE key=?", data, now, expiresAtEpoch, key) + if has { + _, err = session.Exec(`UPDATE cache_data SET data=?, created=?, expire=? WHERE cache_key='?'`, data, getTime().Unix(), expiresAtEpoch, key) } else { - _, err = session.Exec("INSERT INTO cache_data(key,data,created_at,expires) VALUES(?,?,?,?)", key, data, now, expiresAtEpoch) + _, err = session.Exec(`INSERT INTO cache_data (cache_key,data,created_at,expires) VALUES(?,?,?,?)`, key, data, getTime().Unix(), expiresAtEpoch) } return err } func (dc *databaseCache) Delete(key string) error { - sql := `DELETE FROM cache_data WHERE key = ?` + sql := "DELETE FROM cache_data WHERE cache_key=?" _, err := dc.SQLStore.NewSession().Exec(sql, key) return err } -type cacheData struct { - Key string +type CacheData struct { + CacheKey string Data []byte Expires int64 CreatedAt int64 } + +// func (cd CacheData) TableName() string { return "cache_data" } diff --git a/pkg/infra/distcache/database_storage_test.go b/pkg/infra/distcache/database_storage_test.go index 24d8cea16bb..fc526996c89 100644 --- a/pkg/infra/distcache/database_storage_test.go +++ b/pkg/infra/distcache/database_storage_test.go @@ -21,10 +21,16 @@ func TestDatabaseStorageGarbageCollection(t *testing.T) { obj := &CacheableStruct{String: "foolbar"} //set time.now to 2 weeks ago + var err error getTime = func() time.Time { return time.Now().AddDate(0, 0, -2) } - db.Set("key1", obj, 1000*time.Second) - db.Set("key2", obj, 1000*time.Second) - db.Set("key3", obj, 1000*time.Second) + err = db.Set("key1", obj, 1000*time.Second) + assert.Equal(t, err, nil) + + err = db.Set("key2", obj, 1000*time.Second) + assert.Equal(t, err, nil) + + err = db.Set("key3", obj, 1000*time.Second) + assert.Equal(t, err, nil) // insert object that should never expire db.Set("key4", obj, 0) @@ -36,8 +42,8 @@ func TestDatabaseStorageGarbageCollection(t *testing.T) { db.internalRunGC() //try to read values - _, err := db.Get("key1") - assert.Equal(t, err, ErrCacheItemNotFound) + _, err = db.Get("key1") + assert.Equal(t, err, ErrCacheItemNotFound, "expected cache item not found. got: ", err) _, err = db.Get("key2") assert.Equal(t, err, ErrCacheItemNotFound) _, err = db.Get("key3") diff --git a/pkg/infra/distcache/distcache_test.go b/pkg/infra/distcache/distcache_test.go index b631a6283ac..62b07027a05 100644 --- a/pkg/infra/distcache/distcache_test.go +++ b/pkg/infra/distcache/distcache_test.go @@ -58,34 +58,34 @@ func runTestsForClient(t *testing.T, client CacheStorage) { func canPutGetAndDeleteCachedObjects(t *testing.T, client CacheStorage) { cacheableStruct := CacheableStruct{String: "hej", Int64: 2000} - err := client.Set("key", cacheableStruct, 0) - assert.Equal(t, err, nil) + err := client.Set("key1", cacheableStruct, 0) + assert.Equal(t, err, nil, "expected nil. got: ", err) - data, err := client.Get("key") + data, err := client.Get("key1") s, ok := data.(CacheableStruct) assert.Equal(t, ok, true) assert.Equal(t, s.String, "hej") assert.Equal(t, s.Int64, int64(2000)) - err = client.Delete("key") + err = client.Delete("key1") assert.Equal(t, err, nil) - _, err = client.Get("key") + _, err = client.Get("key1") assert.Equal(t, err, ErrCacheItemNotFound) } func canNotFetchExpiredItems(t *testing.T, client CacheStorage) { cacheableStruct := CacheableStruct{String: "hej", Int64: 2000} - err := client.Set("key", cacheableStruct, time.Second) + err := client.Set("key1", cacheableStruct, time.Second) assert.Equal(t, err, nil) //not sure how this can be avoided when testing redis/memcached :/ <-time.After(time.Second + time.Millisecond) // should not be able to read that value since its expired - _, err = client.Get("key") + _, err = client.Get("key1") assert.Equal(t, err, ErrCacheItemNotFound) } @@ -94,12 +94,12 @@ func canSetInfiniteCacheExpiration(t *testing.T, client CacheStorage) { // insert cache item one day back getTime = func() time.Time { return time.Now().AddDate(0, 0, -2) } - err := client.Set("key", cacheableStruct, 0) + err := client.Set("key1", cacheableStruct, 0) assert.Equal(t, err, nil) // should not be able to read that value since its expired getTime = time.Now - data, err := client.Get("key") + data, err := client.Get("key1") s, ok := data.(CacheableStruct) assert.Equal(t, ok, true) diff --git a/pkg/services/sqlstore/migrations/cache_data_mig.go b/pkg/services/sqlstore/migrations/cache_data_mig.go index f12f7f797c8..3467b88962b 100644 --- a/pkg/services/sqlstore/migrations/cache_data_mig.go +++ b/pkg/services/sqlstore/migrations/cache_data_mig.go @@ -6,17 +6,17 @@ func addCacheMigration(mg *migrator.Migrator) { var cacheDataV1 = migrator.Table{ Name: "cache_data", Columns: []*migrator.Column{ - {Name: "key", Type: migrator.DB_NVarchar, IsPrimaryKey: true, Length: 168}, + {Name: "cache_key", Type: migrator.DB_NVarchar, IsPrimaryKey: true, Length: 168}, {Name: "data", Type: migrator.DB_Blob}, {Name: "expires", Type: migrator.DB_Integer, Length: 255, Nullable: false}, {Name: "created_at", Type: migrator.DB_Integer, Length: 255, Nullable: false}, }, Indices: []*migrator.Index{ - {Cols: []string{"key"}, Type: migrator.UniqueIndex}, + {Cols: []string{"cache_key"}, Type: migrator.UniqueIndex}, }, } mg.AddMigration("create cache_data table", migrator.NewAddTableMigration(cacheDataV1)) - mg.AddMigration("add unique index cache_data.key", migrator.NewAddIndexMigration(cacheDataV1, cacheDataV1.Indices[0])) + mg.AddMigration("add unique index cache_data.cache_key", migrator.NewAddIndexMigration(cacheDataV1, cacheDataV1.Indices[0])) } diff --git a/scripts/circle-test-cache-servers.sh b/scripts/circle-test-cache-servers.sh index 6b29be15f42..a75b7235763 100755 --- a/scripts/circle-test-cache-servers.sh +++ b/scripts/circle-test-cache-servers.sh @@ -13,5 +13,6 @@ function exit_if_fail { echo "running redis and memcache tests" #set -e #time for d in $(go list ./pkg/...); do -time exit_if_fail go test -tags="redis memcached" ./pkg/infra/distcache/... +time exit_if_fail go test -tags=redis ./pkg/infra/distcache/... +time exit_if_fail go test -tags=memcached ./pkg/infra/distcache/... #done From 7e7427637cf67e385934a3cc11f04aa641179139 Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 8 Mar 2019 20:49:16 +0100 Subject: [PATCH 24/75] renames distcache -> remotecache --- conf/defaults.ini | 2 +- conf/sample.ini | 11 ++++++++++ pkg/cmd/grafana-server/server.go | 2 +- .../database_storage.go | 2 +- .../database_storage_test.go | 2 +- .../memcached_storage.go | 4 ++-- .../memcached_storage_integration_test.go | 4 ++-- .../redis_storage.go | 4 ++-- .../redis_storage_integration_test.go | 4 ++-- .../remotecache.go} | 20 ++++++++++--------- .../remotecache_test.go} | 10 +++++----- pkg/setting/setting.go | 8 ++++---- scripts/circle-test-cache-servers.sh | 4 ++-- 13 files changed, 45 insertions(+), 32 deletions(-) rename pkg/infra/{distcache => remotecache}/database_storage.go (99%) rename pkg/infra/{distcache => remotecache}/database_storage_test.go (98%) rename pkg/infra/{distcache => remotecache}/memcached_storage.go (92%) rename pkg/infra/{distcache => remotecache}/memcached_storage_integration_test.go (64%) rename pkg/infra/{distcache => remotecache}/redis_storage.go (92%) rename pkg/infra/{distcache => remotecache}/redis_storage_integration_test.go (65%) rename pkg/infra/{distcache/distcache.go => remotecache/remotecache.go} (79%) rename pkg/infra/{distcache/distcache_test.go => remotecache/remotecache_test.go} (90%) diff --git a/conf/defaults.ini b/conf/defaults.ini index 91a58243c04..74bb8b057ad 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -107,7 +107,7 @@ path = grafana.db cache_mode = private #################################### Cache server ############################# -[cache_server] +[remote_cache] # Either "redis", "memcached" or "database" default is "database" type = database diff --git a/conf/sample.ini b/conf/sample.ini index 57ff82181de..860efab0140 100644 --- a/conf/sample.ini +++ b/conf/sample.ini @@ -102,6 +102,17 @@ log_queries = # For "sqlite3" only. cache mode setting used for connecting to the database. (private, shared) ;cache_mode = private +#################################### Cache server ############################# +[remote_cache] +# Either "redis", "memcached" or "database" default is "database" +;type = database + +# cache connectionstring options +# database: will use Grafana primary database. +# redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=grafana` +# memcache: 127.0.0.1:11211 +;connstr = + #################################### Session #################################### [session] # Either "memory", "file", "redis", "mysql", "postgres", default is "file" diff --git a/pkg/cmd/grafana-server/server.go b/pkg/cmd/grafana-server/server.go index d2852e0b8ca..c10212329cf 100644 --- a/pkg/cmd/grafana-server/server.go +++ b/pkg/cmd/grafana-server/server.go @@ -28,8 +28,8 @@ import ( // self registering services _ "github.com/grafana/grafana/pkg/extensions" - _ "github.com/grafana/grafana/pkg/infra/distcache" _ "github.com/grafana/grafana/pkg/infra/metrics" + _ "github.com/grafana/grafana/pkg/infra/remotecache" _ "github.com/grafana/grafana/pkg/infra/serverlock" _ "github.com/grafana/grafana/pkg/infra/tracing" _ "github.com/grafana/grafana/pkg/infra/usagestats" diff --git a/pkg/infra/distcache/database_storage.go b/pkg/infra/remotecache/database_storage.go similarity index 99% rename from pkg/infra/distcache/database_storage.go rename to pkg/infra/remotecache/database_storage.go index 9883751569f..cb6c95ce157 100644 --- a/pkg/infra/distcache/database_storage.go +++ b/pkg/infra/remotecache/database_storage.go @@ -1,4 +1,4 @@ -package distcache +package remotecache import ( "context" diff --git a/pkg/infra/distcache/database_storage_test.go b/pkg/infra/remotecache/database_storage_test.go similarity index 98% rename from pkg/infra/distcache/database_storage_test.go rename to pkg/infra/remotecache/database_storage_test.go index fc526996c89..7fde3d325e5 100644 --- a/pkg/infra/distcache/database_storage_test.go +++ b/pkg/infra/remotecache/database_storage_test.go @@ -1,4 +1,4 @@ -package distcache +package remotecache import ( "testing" diff --git a/pkg/infra/distcache/memcached_storage.go b/pkg/infra/remotecache/memcached_storage.go similarity index 92% rename from pkg/infra/distcache/memcached_storage.go rename to pkg/infra/remotecache/memcached_storage.go index 998d05621c9..7356849c1ef 100644 --- a/pkg/infra/distcache/memcached_storage.go +++ b/pkg/infra/remotecache/memcached_storage.go @@ -1,4 +1,4 @@ -package distcache +package remotecache import ( "time" @@ -11,7 +11,7 @@ type memcachedStorage struct { c *memcache.Client } -func newMemcachedStorage(opts *setting.CacheOpts) *memcachedStorage { +func newMemcachedStorage(opts *setting.RemoteCacheOptions) *memcachedStorage { return &memcachedStorage{ c: memcache.New(opts.ConnStr), } diff --git a/pkg/infra/distcache/memcached_storage_integration_test.go b/pkg/infra/remotecache/memcached_storage_integration_test.go similarity index 64% rename from pkg/infra/distcache/memcached_storage_integration_test.go rename to pkg/infra/remotecache/memcached_storage_integration_test.go index 125bf8d2bf1..d55d78ff482 100644 --- a/pkg/infra/distcache/memcached_storage_integration_test.go +++ b/pkg/infra/remotecache/memcached_storage_integration_test.go @@ -1,6 +1,6 @@ // +build memcached -package distcache +package remotecache import ( "testing" @@ -9,6 +9,6 @@ import ( ) func TestMemcachedCacheStorage(t *testing.T) { - opts := &setting.CacheOpts{Name: "memcached", ConnStr: "localhost:11211"} + opts := &setting.RemoteCacheOptions{Name: "memcached", ConnStr: "localhost:11211"} runTestsForClient(t, createTestClient(t, opts, nil)) } diff --git a/pkg/infra/distcache/redis_storage.go b/pkg/infra/remotecache/redis_storage.go similarity index 92% rename from pkg/infra/distcache/redis_storage.go rename to pkg/infra/remotecache/redis_storage.go index 1414671f05b..9d54020fe79 100644 --- a/pkg/infra/distcache/redis_storage.go +++ b/pkg/infra/remotecache/redis_storage.go @@ -1,4 +1,4 @@ -package distcache +package remotecache import ( "time" @@ -11,7 +11,7 @@ type redisStorage struct { c *redis.Client } -func newRedisStorage(opts *setting.CacheOpts) *redisStorage { +func newRedisStorage(opts *setting.RemoteCacheOptions) *redisStorage { opt := &redis.Options{ Network: "tcp", Addr: opts.ConnStr, diff --git a/pkg/infra/distcache/redis_storage_integration_test.go b/pkg/infra/remotecache/redis_storage_integration_test.go similarity index 65% rename from pkg/infra/distcache/redis_storage_integration_test.go rename to pkg/infra/remotecache/redis_storage_integration_test.go index 289a3ff4e2d..bd834fb89ff 100644 --- a/pkg/infra/distcache/redis_storage_integration_test.go +++ b/pkg/infra/remotecache/redis_storage_integration_test.go @@ -1,6 +1,6 @@ // +build redis -package distcache +package remotecache import ( "testing" @@ -10,6 +10,6 @@ import ( func TestRedisCacheStorage(t *testing.T) { - opts := &setting.CacheOpts{Name: "redis", ConnStr: "localhost:6379"} + opts := &setting.RemoteCacheOptions{Name: "redis", ConnStr: "localhost:6379"} runTestsForClient(t, createTestClient(t, opts, nil)) } diff --git a/pkg/infra/distcache/distcache.go b/pkg/infra/remotecache/remotecache.go similarity index 79% rename from pkg/infra/distcache/distcache.go rename to pkg/infra/remotecache/remotecache.go index a8f12adaa27..761a2b3d337 100644 --- a/pkg/infra/distcache/distcache.go +++ b/pkg/infra/remotecache/remotecache.go @@ -1,4 +1,4 @@ -package distcache +package remotecache import ( "bytes" @@ -20,7 +20,7 @@ var ( ) func init() { - registry.RegisterService(&DistributedCache{}) + registry.RegisterService(&RemoteCache{}) } // CacheStorage allows the caller to set, get and delete items in the cache. @@ -38,8 +38,8 @@ type CacheStorage interface { Delete(key string) error } -// DistributedCache allows Grafana to cache data outside its own process -type DistributedCache struct { +// RemoteCache allows Grafana to cache data outside its own process +type RemoteCache struct { log log.Logger Client CacheStorage SQLStore *sqlstore.SqlStore `inject:""` @@ -47,15 +47,17 @@ type DistributedCache struct { } // Init initializes the service -func (ds *DistributedCache) Init() error { - ds.log = log.New("distributed.cache") +func (ds *RemoteCache) Init() error { + ds.log = log.New("cache.remote") - ds.Client = createClient(ds.Cfg.CacheOptions, ds.SQLStore) + ds.Client = createClient(ds.Cfg.RemoteCacheOptions, ds.SQLStore) return nil } -func (ds *DistributedCache) Run(ctx context.Context) error { +// Run start the backend processes for cache clients +func (ds *RemoteCache) Run(ctx context.Context) error { + //create new interface if more clients need GC jobs backgroundjob, ok := ds.Client.(registry.BackgroundService) if ok { return backgroundjob.Run(ctx) @@ -65,7 +67,7 @@ func (ds *DistributedCache) Run(ctx context.Context) error { return ctx.Err() } -func createClient(opts *setting.CacheOpts, sqlstore *sqlstore.SqlStore) CacheStorage { +func createClient(opts *setting.RemoteCacheOptions, sqlstore *sqlstore.SqlStore) CacheStorage { if opts.Name == "redis" { return newRedisStorage(opts) } diff --git a/pkg/infra/distcache/distcache_test.go b/pkg/infra/remotecache/remotecache_test.go similarity index 90% rename from pkg/infra/distcache/distcache_test.go rename to pkg/infra/remotecache/remotecache_test.go index 62b07027a05..8887686c3a1 100644 --- a/pkg/infra/distcache/distcache_test.go +++ b/pkg/infra/remotecache/remotecache_test.go @@ -1,4 +1,4 @@ -package distcache +package remotecache import ( "testing" @@ -19,13 +19,13 @@ func init() { Register(CacheableStruct{}) } -func createTestClient(t *testing.T, opts *setting.CacheOpts, sqlstore *sqlstore.SqlStore) CacheStorage { +func createTestClient(t *testing.T, opts *setting.RemoteCacheOptions, sqlstore *sqlstore.SqlStore) CacheStorage { t.Helper() - dc := &DistributedCache{ + dc := &RemoteCache{ SQLStore: sqlstore, Cfg: &setting.Cfg{ - CacheOptions: opts, + RemoteCacheOptions: opts, }, } @@ -44,7 +44,7 @@ func TestCachedBasedOnConfig(t *testing.T) { HomePath: "../../../", }) - client := createTestClient(t, cfg.CacheOptions, sqlstore.InitTestDB(t)) + client := createTestClient(t, cfg.RemoteCacheOptions, sqlstore.InitTestDB(t)) runTestsForClient(t, client) } diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index 864c29fb382..9d135ca3aae 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -242,7 +242,7 @@ type Cfg struct { EditorsCanOwn bool // DistributedCache - CacheOptions *CacheOpts + RemoteCacheOptions *RemoteCacheOptions } type CommandLineArgs struct { @@ -782,8 +782,8 @@ func (cfg *Cfg) Load(args *CommandLineArgs) error { enterprise := iniFile.Section("enterprise") cfg.EnterpriseLicensePath = enterprise.Key("license_path").MustString(filepath.Join(cfg.DataPath, "license.jwt")) - cacheServer := iniFile.Section("cache_server") - cfg.CacheOptions = &CacheOpts{ + cacheServer := iniFile.Section("remote_cache") + cfg.RemoteCacheOptions = &RemoteCacheOptions{ Name: cacheServer.Key("type").MustString("database"), ConnStr: cacheServer.Key("connstr").MustString(""), } @@ -791,7 +791,7 @@ func (cfg *Cfg) Load(args *CommandLineArgs) error { return nil } -type CacheOpts struct { +type RemoteCacheOptions struct { Name string ConnStr string } diff --git a/scripts/circle-test-cache-servers.sh b/scripts/circle-test-cache-servers.sh index a75b7235763..3ec5dbf1069 100755 --- a/scripts/circle-test-cache-servers.sh +++ b/scripts/circle-test-cache-servers.sh @@ -13,6 +13,6 @@ function exit_if_fail { echo "running redis and memcache tests" #set -e #time for d in $(go list ./pkg/...); do -time exit_if_fail go test -tags=redis ./pkg/infra/distcache/... -time exit_if_fail go test -tags=memcached ./pkg/infra/distcache/... +time exit_if_fail go test -tags=redis ./pkg/infra/remotecache/... +time exit_if_fail go test -tags=memcached ./pkg/infra/remotecache/... #done From 085b63109945b5ae43d71f9ee194e1c3f4285f99 Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 11 Mar 2019 09:20:30 +0100 Subject: [PATCH 25/75] add docs about remote cache settings --- docs/sources/installation/configuration.md | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/docs/sources/installation/configuration.md b/docs/sources/installation/configuration.md index f0418ad31a6..9705dd2001c 100644 --- a/docs/sources/installation/configuration.md +++ b/docs/sources/installation/configuration.md @@ -162,9 +162,9 @@ executed with working directory set to the installation path. ### enable_gzip -Set this option to `true` to enable HTTP compression, this can improve -transfer speed and bandwidth utilization. It is recommended that most -users set it to `true`. By default it is set to `false` for compatibility +Set this option to `true` to enable HTTP compression, this can improve +transfer speed and bandwidth utilization. It is recommended that most +users set it to `true`. By default it is set to `false` for compatibility reasons. ### cert_file @@ -179,7 +179,6 @@ Path to the certificate key file (if `protocol` is set to `https`). Set to true for Grafana to log all HTTP requests (not just errors). These are logged as Info level events to grafana log. -
@@ -262,6 +261,19 @@ Set to `true` to log the sql calls and execution times. For "sqlite3" only. [Shared cache](https://www.sqlite.org/sharedcache.html) setting used for connecting to the database. (private, shared) Defaults to private. +
+ +## [remote_cache] + +### type + +Either `redis`, `memcached` or `database` default is `database` + +### connstr + +The remote cache connection string. Leave empty when using `database` since it will use the primary database. +Redis example config: `addr=127.0.0.1:6379,pool_size=100,db=grafana` +Memcache example: `127.0.0.1:11211`
From b2967fbb3747a32a4548eebc4a6fba580f2fa7d3 Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 11 Mar 2019 10:44:16 +0100 Subject: [PATCH 26/75] avoid exposing cache client directly --- pkg/infra/remotecache/remotecache.go | 20 ++++++++++++++++---- pkg/infra/remotecache/remotecache_test.go | 2 +- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/pkg/infra/remotecache/remotecache.go b/pkg/infra/remotecache/remotecache.go index 761a2b3d337..1b9d67b9358 100644 --- a/pkg/infra/remotecache/remotecache.go +++ b/pkg/infra/remotecache/remotecache.go @@ -31,7 +31,7 @@ type CacheStorage interface { // Get reads object from Cache Get(key string) (interface{}, error) - // Set sets an object into the cache + // Set sets an object into the cache. if `expire` is set to zero it never expires. Set(key string, value interface{}, expire time.Duration) error // Delete object from cache @@ -41,16 +41,28 @@ type CacheStorage interface { // RemoteCache allows Grafana to cache data outside its own process type RemoteCache struct { log log.Logger - Client CacheStorage + client CacheStorage SQLStore *sqlstore.SqlStore `inject:""` Cfg *setting.Cfg `inject:""` } +func (ds *RemoteCache) Get(key string) (interface{}, error) { + return ds.client.Get(key) +} + +func (ds *RemoteCache) Set(key string, value interface{}, expire time.Duration) error { + return ds.client.Set(key, value, expire) +} + +func (ds *RemoteCache) Delete(key string) error { + return ds.client.Delete(key) +} + // Init initializes the service func (ds *RemoteCache) Init() error { ds.log = log.New("cache.remote") - ds.Client = createClient(ds.Cfg.RemoteCacheOptions, ds.SQLStore) + ds.client = createClient(ds.Cfg.RemoteCacheOptions, ds.SQLStore) return nil } @@ -58,7 +70,7 @@ func (ds *RemoteCache) Init() error { // Run start the backend processes for cache clients func (ds *RemoteCache) Run(ctx context.Context) error { //create new interface if more clients need GC jobs - backgroundjob, ok := ds.Client.(registry.BackgroundService) + backgroundjob, ok := ds.client.(registry.BackgroundService) if ok { return backgroundjob.Run(ctx) } diff --git a/pkg/infra/remotecache/remotecache_test.go b/pkg/infra/remotecache/remotecache_test.go index 8887686c3a1..ac22607ee70 100644 --- a/pkg/infra/remotecache/remotecache_test.go +++ b/pkg/infra/remotecache/remotecache_test.go @@ -34,7 +34,7 @@ func createTestClient(t *testing.T, opts *setting.RemoteCacheOptions, sqlstore * t.Fatalf("failed to init client for test. error: %v", err) } - return dc.Client + return dc.client } func TestCachedBasedOnConfig(t *testing.T) { From 7aeab0a235a515accca2cb7eaae6061cba97a51c Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 11 Mar 2019 10:59:55 +0100 Subject: [PATCH 27/75] use `Get` instead of `Find` --- pkg/infra/remotecache/database_storage.go | 22 +++++++++++----------- scripts/circle-test-cache-servers.sh | 4 +--- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/pkg/infra/remotecache/database_storage.go b/pkg/infra/remotecache/database_storage.go index cb6c95ce157..2e34ecd1c73 100644 --- a/pkg/infra/remotecache/database_storage.go +++ b/pkg/infra/remotecache/database_storage.go @@ -47,24 +47,24 @@ func (dc *databaseCache) internalRunGC() { } func (dc *databaseCache) Get(key string) (interface{}, error) { - cacheHits := []CacheData{} - sess := dc.SQLStore.NewSession() - defer sess.Close() - err := sess.Where("cache_key= ?", key).Find(&cacheHits) + cacheHit := CacheData{} + session := dc.SQLStore.NewSession() + defer session.Close() + + exist, err := session.Where("cache_key= ?", key).Get(&cacheHit) if err != nil { return nil, err } - if len(cacheHits) == 0 { + if !exist { return nil, ErrCacheItemNotFound } - cacheHit := cacheHits[0] if cacheHit.Expires > 0 { existedButExpired := getTime().Unix()-cacheHit.CreatedAt >= cacheHit.Expires if existedButExpired { - dc.Delete(key) + _ = dc.Delete(key) //ignore this error since we will return `ErrCacheItemNotFound` anyway return nil, ErrCacheItemNotFound } } @@ -99,9 +99,11 @@ func (dc *databaseCache) Set(key string, value interface{}, expire time.Duration // insert or update depending on if item already exist if has { - _, err = session.Exec(`UPDATE cache_data SET data=?, created=?, expire=? WHERE cache_key='?'`, data, getTime().Unix(), expiresAtEpoch, key) + sql := `UPDATE cache_data SET data=?, created=?, expire=? WHERE cache_key='?'` + _, err = session.Exec(sql, data, getTime().Unix(), expiresAtEpoch, key) } else { - _, err = session.Exec(`INSERT INTO cache_data (cache_key,data,created_at,expires) VALUES(?,?,?,?)`, key, data, getTime().Unix(), expiresAtEpoch) + sql := `INSERT INTO cache_data (cache_key,data,created_at,expires) VALUES(?,?,?,?)` + _, err = session.Exec(sql, key, data, getTime().Unix(), expiresAtEpoch) } return err @@ -120,5 +122,3 @@ type CacheData struct { Expires int64 CreatedAt int64 } - -// func (cd CacheData) TableName() string { return "cache_data" } diff --git a/scripts/circle-test-cache-servers.sh b/scripts/circle-test-cache-servers.sh index 3ec5dbf1069..bacd9928362 100755 --- a/scripts/circle-test-cache-servers.sh +++ b/scripts/circle-test-cache-servers.sh @@ -11,8 +11,6 @@ function exit_if_fail { } echo "running redis and memcache tests" -#set -e -#time for d in $(go list ./pkg/...); do + time exit_if_fail go test -tags=redis ./pkg/infra/remotecache/... time exit_if_fail go test -tags=memcached ./pkg/infra/remotecache/... -#done From 8cd54c94e99e9221c0901d15bdcf74c51764c246 Mon Sep 17 00:00:00 2001 From: ryan Date: Mon, 11 Mar 2019 14:47:54 -0700 Subject: [PATCH 28/75] make value processing reusable --- .../src/components/Gauge/Gauge.test.tsx | 92 +-------------- .../grafana-ui/src/components/Gauge/Gauge.tsx | 77 ++----------- .../src/utils/valueProcessor.test.ts | 107 ++++++++++++++++++ .../grafana-ui/src/utils/valueProcessor.ts | 97 ++++++++++++++++ .../panel/gauge/DisplayValueEditor.tsx | 64 +++++++++++ public/app/plugins/panel/gauge/GaugePanel.tsx | 42 ++++--- .../plugins/panel/gauge/GaugePanelEditor.tsx | 13 ++- .../panel/gauge/SingleStatValueEditor.tsx | 49 +------- public/app/plugins/panel/gauge/types.ts | 22 ++-- 9 files changed, 342 insertions(+), 221 deletions(-) create mode 100644 packages/grafana-ui/src/utils/valueProcessor.test.ts create mode 100644 packages/grafana-ui/src/utils/valueProcessor.ts create mode 100644 public/app/plugins/panel/gauge/DisplayValueEditor.tsx diff --git a/packages/grafana-ui/src/components/Gauge/Gauge.test.tsx b/packages/grafana-ui/src/components/Gauge/Gauge.test.tsx index 70e29abc221..c6a49eb5b55 100644 --- a/packages/grafana-ui/src/components/Gauge/Gauge.test.tsx +++ b/packages/grafana-ui/src/components/Gauge/Gauge.test.tsx @@ -2,7 +2,6 @@ import React from 'react'; import { shallow } from 'enzyme'; import { Gauge, Props } from './Gauge'; -import { ValueMapping, MappingType } from '../../types'; import { getTheme } from '../../themes'; jest.mock('jquery', () => ({ @@ -12,19 +11,16 @@ jest.mock('jquery', () => ({ const setup = (propOverrides?: object) => { const props: Props = { maxValue: 100, - valueMappings: [], minValue: 0, - prefix: '', showThresholdMarkers: true, showThresholdLabels: false, - suffix: '', thresholds: [{ index: 0, value: -Infinity, color: '#7EB26D' }], - unit: 'none', - stat: 'avg', height: 300, width: 300, - value: 25, - decimals: 0, + value: { + text: '25', + numeric: 25, + }, theme: getTheme(), }; @@ -39,38 +35,6 @@ const setup = (propOverrides?: object) => { }; }; -describe('Get font color', () => { - it('should get first threshold color when only one threshold', () => { - const { instance } = setup({ thresholds: [{ index: 0, value: -Infinity, color: '#7EB26D' }] }); - - expect(instance.getFontColor(49)).toEqual('#7EB26D'); - }); - - it('should get the threshold color if value is same as a threshold', () => { - const { instance } = setup({ - thresholds: [ - { index: 2, value: 75, color: '#6ED0E0' }, - { index: 1, value: 50, color: '#EAB839' }, - { index: 0, value: -Infinity, color: '#7EB26D' }, - ], - }); - - expect(instance.getFontColor(50)).toEqual('#EAB839'); - }); - - it('should get the nearest threshold color between thresholds', () => { - const { instance } = setup({ - thresholds: [ - { index: 2, value: 75, color: '#6ED0E0' }, - { index: 1, value: 50, color: '#EAB839' }, - { index: 0, value: -Infinity, color: '#7EB26D' }, - ], - }); - - expect(instance.getFontColor(55)).toEqual('#EAB839'); - }); -}); - describe('Get thresholds formatted', () => { it('should return first thresholds color for min and max', () => { const { instance } = setup({ thresholds: [{ index: 0, value: -Infinity, color: '#7EB26D' }] }); @@ -98,51 +62,3 @@ describe('Get thresholds formatted', () => { ]); }); }); - -describe('Format value', () => { - it('should return if value isNaN', () => { - const valueMappings: ValueMapping[] = []; - const value = 'N/A'; - const { instance } = setup({ valueMappings }); - - const result = instance.formatValue(value); - - expect(result).toEqual('N/A'); - }); - - it('should return formatted value if there are no value mappings', () => { - const valueMappings: ValueMapping[] = []; - const value = '6'; - const { instance } = setup({ valueMappings, decimals: 1 }); - - const result = instance.formatValue(value); - - expect(result).toEqual('6.0'); - }); - - it('should return formatted value if there are no matching value mappings', () => { - const valueMappings: ValueMapping[] = [ - { id: 0, operator: '', text: 'elva', type: MappingType.ValueToText, value: '11' }, - { id: 1, operator: '', text: '1-9', type: MappingType.RangeToText, from: '1', to: '9' }, - ]; - const value = '10'; - const { instance } = setup({ valueMappings, decimals: 1 }); - - const result = instance.formatValue(value); - - expect(result).toEqual('10.0'); - }); - - it('should return mapped value if there are matching value mappings', () => { - const valueMappings: ValueMapping[] = [ - { id: 0, operator: '', text: '1-20', type: MappingType.RangeToText, from: '1', to: '20' }, - { id: 1, operator: '', text: 'elva', type: MappingType.ValueToText, value: '11' }, - ]; - const value = '11'; - const { instance } = setup({ valueMappings, decimals: 1 }); - - const result = instance.formatValue(value); - - expect(result).toEqual('1-20'); - }); -}); diff --git a/packages/grafana-ui/src/components/Gauge/Gauge.tsx b/packages/grafana-ui/src/components/Gauge/Gauge.tsx index d04daae3dab..460547a4d7e 100644 --- a/packages/grafana-ui/src/components/Gauge/Gauge.tsx +++ b/packages/grafana-ui/src/components/Gauge/Gauge.tsx @@ -1,28 +1,20 @@ import React, { PureComponent } from 'react'; import $ from 'jquery'; -import { getMappedValue } from '../../utils/valueMappings'; import { getColorFromHexRgbOrName } from '../../utils/namedColorsPalette'; import { Themeable, GrafanaThemeType } from '../../types/theme'; -import { ValueMapping, Threshold, BasicGaugeColor } from '../../types/panel'; -import { getValueFormat } from '../../utils/valueFormats/valueFormats'; - -type TimeSeriesValue = string | number | null; +import { Threshold, BasicGaugeColor } from '../../types/panel'; +import { DisplayValue } from '../../utils/valueProcessor'; export interface Props extends Themeable { - decimals?: number | null; + width: number; height: number; - valueMappings: ValueMapping[]; maxValue: number; minValue: number; - prefix: string; thresholds: Threshold[]; showThresholdMarkers: boolean; showThresholdLabels: boolean; - stat: string; - suffix: string; - unit: string; - width: number; - value: number; + + value: DisplayValue; } const FONT_SCALE = 1; @@ -32,15 +24,10 @@ export class Gauge extends PureComponent { static defaultProps = { maxValue: 100, - valueMappings: [], minValue: 0, - prefix: '', showThresholdMarkers: true, showThresholdLabels: false, - suffix: '', thresholds: [], - unit: 'none', - stat: 'avg', theme: GrafanaThemeType.Dark, }; @@ -52,49 +39,6 @@ export class Gauge extends PureComponent { this.draw(); } - formatValue(value: TimeSeriesValue) { - const { decimals, valueMappings, prefix, suffix, unit } = this.props; - - if (isNaN(value as number)) { - return value; - } - - if (valueMappings.length > 0) { - const valueMappedValue = getMappedValue(valueMappings, value); - if (valueMappedValue) { - return `${prefix && prefix + ' '}${valueMappedValue.text}${suffix && ' ' + suffix}`; - } - } - - const formatFunc = getValueFormat(unit); - const formattedValue = formatFunc(value as number, decimals); - const handleNoValueValue = formattedValue || 'no value'; - - return `${prefix && prefix + ' '}${handleNoValueValue}${suffix && ' ' + suffix}`; - } - - getFontColor(value: TimeSeriesValue) { - const { thresholds, theme } = this.props; - - if (thresholds.length === 1) { - return getColorFromHexRgbOrName(thresholds[0].color, theme.type); - } - - const atThreshold = thresholds.filter(threshold => (value as number) === threshold.value)[0]; - if (atThreshold) { - return getColorFromHexRgbOrName(atThreshold.color, theme.type); - } - - const belowThreshold = thresholds.filter(threshold => (value as number) > threshold.value); - - if (belowThreshold.length > 0) { - const nearestThreshold = belowThreshold.sort((t1, t2) => t2.value - t1.value)[0]; - return getColorFromHexRgbOrName(nearestThreshold.color, theme.type); - } - - return BasicGaugeColor.Red; - } - getFormattedThresholds() { const { maxValue, minValue, thresholds, theme } = this.props; @@ -123,15 +67,13 @@ export class Gauge extends PureComponent { draw() { const { maxValue, minValue, showThresholdLabels, showThresholdMarkers, width, height, theme, value } = this.props; - const formattedValue = this.formatValue(value) as string; const dimension = Math.min(width, height * 1.3); const backgroundColor = theme.type === GrafanaThemeType.Light ? 'rgb(230,230,230)' : theme.colors.dark3; const gaugeWidthReduceRatio = showThresholdLabels ? 1.5 : 1; const gaugeWidth = Math.min(dimension / 6, 60) / gaugeWidthReduceRatio; const thresholdMarkersWidth = gaugeWidth / 5; - const fontSize = - Math.min(dimension / 5, 100) * (formattedValue !== null ? this.getFontScale(formattedValue.length) : 1); + const fontSize = Math.min(dimension / 5, 100) * this.getFontScale(value.text.length); const thresholdLabelFontSize = fontSize / 2.5; const options = { @@ -160,9 +102,9 @@ export class Gauge extends PureComponent { width: thresholdMarkersWidth, }, value: { - color: this.getFontColor(value), + color: value.color ? value.color : BasicGaugeColor.Red, formatter: () => { - return formattedValue; + return value.text; }, font: { size: fontSize, family: '"Helvetica Neue", Helvetica, Arial, sans-serif' }, }, @@ -171,7 +113,8 @@ export class Gauge extends PureComponent { }, }; - const plotSeries = { data: [[0, value]] }; + const numeric = value.numeric !== null ? value.numeric : 0; + const plotSeries = { data: [[0, numeric]] }; try { $.plot(this.canvasElement, [plotSeries], options); diff --git a/packages/grafana-ui/src/utils/valueProcessor.test.ts b/packages/grafana-ui/src/utils/valueProcessor.test.ts new file mode 100644 index 00000000000..76c18f9e93c --- /dev/null +++ b/packages/grafana-ui/src/utils/valueProcessor.test.ts @@ -0,0 +1,107 @@ +import { getValueProcessor, getColorFromThreshold } from './valueProcessor'; +import { getTheme } from '../themes/index'; +import { GrafanaThemeType } from '../types/theme'; +import { MappingType, ValueMapping } from '../types/panel'; + +describe('Process values', () => { + const basicConversions = [ + { value: null, text: '' }, + { value: undefined, text: '' }, + { value: 1.23, text: '1.23' }, + { value: 1, text: '1' }, + { value: 'hello', text: 'hello' }, + { value: {}, text: '[object Object]' }, + { value: [], text: '' }, + { value: [1, 2, 3], text: '1,2,3' }, + { value: ['a', 'b', 'c'], text: 'a,b,c' }, + ]; + + it('should return return a string for any input value', () => { + const processor = getValueProcessor(); + basicConversions.forEach(item => { + expect(processor(item.value).text).toBe(item.text); + }); + }); + + it('should add a suffix to any value', () => { + const processor = getValueProcessor({ + prefix: 'xxx', + theme: getTheme(GrafanaThemeType.Dark), + }); + basicConversions.forEach(item => { + expect(processor(item.value).text).toBe('xxx' + item.text); + }); + }); +}); + +describe('Get color from threshold', () => { + it('should get first threshold color when only one threshold', () => { + const thresholds = [{ index: 0, value: -Infinity, color: '#7EB26D' }]; + expect(getColorFromThreshold(49, thresholds)).toEqual('#7EB26D'); + }); + + it('should get the threshold color if value is same as a threshold', () => { + const thresholds = [ + { index: 2, value: 75, color: '#6ED0E0' }, + { index: 1, value: 50, color: '#EAB839' }, + { index: 0, value: -Infinity, color: '#7EB26D' }, + ]; + expect(getColorFromThreshold(50, thresholds)).toEqual('#EAB839'); + }); + + it('should get the nearest threshold color between thresholds', () => { + const thresholds = [ + { index: 2, value: 75, color: '#6ED0E0' }, + { index: 1, value: 50, color: '#EAB839' }, + { index: 0, value: -Infinity, color: '#7EB26D' }, + ]; + expect(getColorFromThreshold(55, thresholds)).toEqual('#EAB839'); + }); +}); + +describe('Format value', () => { + it('should return if value isNaN', () => { + const valueMappings: ValueMapping[] = []; + const value = 'N/A'; + const instance = getValueProcessor({ mappings: valueMappings }); + + const result = instance(value); + + expect(result.text).toEqual('N/A'); + }); + + it('should return formatted value if there are no value mappings', () => { + const valueMappings: ValueMapping[] = []; + const value = '6'; + + const instance = getValueProcessor({ mappings: valueMappings, decimals: 1 }); + + const result = instance(value); + + expect(result.text).toEqual('6.0'); + }); + + it('should return formatted value if there are no matching value mappings', () => { + const valueMappings: ValueMapping[] = [ + { id: 0, operator: '', text: 'elva', type: MappingType.ValueToText, value: '11' }, + { id: 1, operator: '', text: '1-9', type: MappingType.RangeToText, from: '1', to: '9' }, + ]; + const value = '10'; + const instance = getValueProcessor({ mappings: valueMappings, decimals: 1 }); + + const result = instance(value); + + expect(result.text).toEqual('10.0'); + }); + + it('should return mapped value if there are matching value mappings', () => { + const valueMappings: ValueMapping[] = [ + { id: 0, operator: '', text: '1-20', type: MappingType.RangeToText, from: '1', to: '20' }, + { id: 1, operator: '', text: 'elva', type: MappingType.ValueToText, value: '11' }, + ]; + const value = '11'; + const instance = getValueProcessor({ mappings: valueMappings, decimals: 1 }); + + expect(instance(value).text).toEqual('1-20'); + }); +}); diff --git a/packages/grafana-ui/src/utils/valueProcessor.ts b/packages/grafana-ui/src/utils/valueProcessor.ts new file mode 100644 index 00000000000..243904c269c --- /dev/null +++ b/packages/grafana-ui/src/utils/valueProcessor.ts @@ -0,0 +1,97 @@ +import { ValueMapping, Threshold } from '../types/panel'; +import _ from 'lodash'; +import { getValueFormat, DecimalCount } from './valueFormats/valueFormats'; +import { getMappedValue } from './valueMappings'; +import { GrafanaTheme, GrafanaThemeType } from '../types/theme'; +import { getColorFromHexRgbOrName } from './namedColorsPalette'; + +export interface DisplayValue { + text: string; // How the value should be displayed + numeric?: number; // the value as a number + color?: string; // suggested color +} + +export interface DisplayValueOptions { + unit?: string; + decimals?: DecimalCount; + scaledDecimals?: DecimalCount; + isUtc?: boolean; + + color?: string; + mappings?: ValueMapping[]; + thresholds?: Threshold[]; + prefix?: string; + suffix?: string; + + noValue?: string; + theme?: GrafanaTheme; // Will pick 'dark' if not defined +} + +export type ValueProcessor = (value: any) => DisplayValue; + +export function getValueProcessor(options?: DisplayValueOptions): ValueProcessor { + if (options && !_.isEmpty(options)) { + const formatFunc = getValueFormat(options.unit || 'none'); + return (value: any) => { + const { prefix, suffix, mappings, thresholds, theme } = options; + let color = options.color; + + let text = _.toString(value); + const numeric = _.toNumber(value); + + if (mappings && mappings.length > 0) { + const mappedValue = getMappedValue(mappings, value); + if (mappedValue) { + text = mappedValue.text; + // TODO? convert the mapped value back to a number? + } + } + + if (_.isNumber(numeric)) { + text = formatFunc(numeric, options.decimals, options.scaledDecimals, options.isUtc); + if (thresholds && thresholds.length > 0) { + color = getColorFromThreshold(numeric, thresholds, theme); + } + } + + if (!text) { + text = options.noValue ? options.noValue : ''; + } + if (prefix) { + text = prefix + text; + } + if (suffix) { + text = text + suffix; + } + return { text, numeric, color }; + }; + } + return toStringProcessor; +} + +function toStringProcessor(value: any): DisplayValue { + return { text: _.toString(value), numeric: _.toNumber(value) }; +} + +export function getColorFromThreshold(value: number, thresholds: Threshold[], theme?: GrafanaTheme): string { + const themeType = theme ? theme.type : GrafanaThemeType.Dark; + + if (thresholds.length === 1) { + return getColorFromHexRgbOrName(thresholds[0].color, themeType); + } + + const atThreshold = thresholds.filter(threshold => value === threshold.value)[0]; + if (atThreshold) { + return getColorFromHexRgbOrName(atThreshold.color, themeType); + } + + const belowThreshold = thresholds.filter(threshold => value > threshold.value); + + if (belowThreshold.length > 0) { + const nearestThreshold = belowThreshold.sort((t1, t2) => t2.value - t1.value)[0]; + return getColorFromHexRgbOrName(nearestThreshold.color, themeType); + } + + // Use the first threshold as the default color + return getColorFromHexRgbOrName(thresholds[0].color, themeType); +} diff --git a/public/app/plugins/panel/gauge/DisplayValueEditor.tsx b/public/app/plugins/panel/gauge/DisplayValueEditor.tsx new file mode 100644 index 00000000000..51c956a9529 --- /dev/null +++ b/public/app/plugins/panel/gauge/DisplayValueEditor.tsx @@ -0,0 +1,64 @@ +// Libraries +import React, { PureComponent } from 'react'; + +// Components +import { FormField, FormLabel, PanelOptionsGroup, UnitPicker } from '@grafana/ui'; + +// Types +import { DisplayValueOptions } from '@grafana/ui/src/utils/valueProcessor'; + +const labelWidth = 6; + +export interface Props { + options: DisplayValueOptions; + onChange: (options: DisplayValueOptions) => void; +} + +export class DisplayValueEditor extends PureComponent { + onUnitChange = unit => this.props.onChange({ ...this.props.options, unit: unit.value }); + + onDecimalChange = event => { + if (!isNaN(event.target.value)) { + this.props.onChange({ + ...this.props.options, + decimals: parseInt(event.target.value, 10), + }); + } else { + this.props.onChange({ + ...this.props.options, + decimals: null, + }); + } + }; + + onPrefixChange = event => this.props.onChange({ ...this.props.options, prefix: event.target.value }); + onSuffixChange = event => this.props.onChange({ ...this.props.options, suffix: event.target.value }); + + render() { + const { unit, decimals, prefix, suffix } = this.props.options; + + let decimalsString = ''; + if (Number.isFinite(decimals)) { + decimalsString = decimals.toString(); + } + + return ( + +
+ Unit + +
+ + + + + ); + } +} diff --git a/public/app/plugins/panel/gauge/GaugePanel.tsx b/public/app/plugins/panel/gauge/GaugePanel.tsx index b75d4a1c7f3..425ccb00356 100644 --- a/public/app/plugins/panel/gauge/GaugePanel.tsx +++ b/public/app/plugins/panel/gauge/GaugePanel.tsx @@ -9,30 +9,50 @@ import { Gauge } from '@grafana/ui'; // Types import { GaugeOptions } from './types'; -import { PanelProps, NullValueMode, TimeSeriesValue } from '@grafana/ui/src/types'; +import { PanelProps, NullValueMode, BasicGaugeColor } from '@grafana/ui/src/types'; +import { DisplayValue, getValueProcessor } from '@grafana/ui/src/utils/valueProcessor'; interface Props extends PanelProps {} interface State { - value: TimeSeriesValue; + value: DisplayValue; } export class GaugePanel extends Component { constructor(props: Props) { super(props); + + if (props.options.valueOptions) { + console.warn('TODO!! how do we best migration options?'); + } + this.state = { - value: this.findValue(props), + value: this.findDisplayValue(props), }; } componentDidUpdate(prevProps: Props) { if (this.props.panelData !== prevProps.panelData) { - this.setState({ value: this.findValue(this.props) }); + this.setState({ value: this.findDisplayValue(this.props) }); } } + findDisplayValue(props: Props): DisplayValue { + const { replaceVariables, options } = this.props; + const { displayOptions } = options; + + const prefix = replaceVariables(displayOptions.prefix); + const suffix = replaceVariables(displayOptions.suffix); + return getValueProcessor({ + color: BasicGaugeColor.Red, // The default color + ...displayOptions, + prefix, + suffix, + // ??? theme:getTheme(GrafanaThemeType.Dark), !! how do I get it here??? + })(this.findValue(props)); + } + findValue(props: Props): number | null { const { panelData, options } = props; - const { valueOptions } = options; if (panelData.timeSeries) { const vmSeries = processTimeSeries({ @@ -41,7 +61,7 @@ export class GaugePanel extends Component { }); if (vmSeries[0]) { - return vmSeries[0].stats[valueOptions.stat]; + return vmSeries[0].stats[options.stat]; } } else if (panelData.tableData) { return panelData.tableData.rows[0].find(prop => prop > 0); @@ -50,12 +70,9 @@ export class GaugePanel extends Component { } render() { - const { width, height, replaceVariables, options } = this.props; - const { valueOptions } = options; + const { width, height, options } = this.props; const { value } = this.state; - const prefix = replaceVariables(valueOptions.prefix); - const suffix = replaceVariables(valueOptions.suffix); return ( {theme => ( @@ -63,12 +80,7 @@ export class GaugePanel extends Component { value={value} width={width} height={height} - prefix={prefix} - suffix={suffix} - unit={valueOptions.unit} - decimals={valueOptions.decimals} thresholds={options.thresholds} - valueMappings={options.valueMappings} showThresholdLabels={options.showThresholdLabels} showThresholdMarkers={options.showThresholdMarkers} minValue={options.minValue} diff --git a/public/app/plugins/panel/gauge/GaugePanelEditor.tsx b/public/app/plugins/panel/gauge/GaugePanelEditor.tsx index f226be7328c..55a0377848d 100644 --- a/public/app/plugins/panel/gauge/GaugePanelEditor.tsx +++ b/public/app/plugins/panel/gauge/GaugePanelEditor.tsx @@ -11,6 +11,8 @@ import { import { SingleStatValueEditor } from 'app/plugins/panel/gauge/SingleStatValueEditor'; import { GaugeOptionsBox } from './GaugeOptionsBox'; import { GaugeOptions, SingleStatValueOptions } from './types'; +import { DisplayValueEditor } from './DisplayValueEditor'; +import { DisplayValueOptions } from '@grafana/ui/src/utils/valueProcessor'; export class GaugePanelEditor extends PureComponent> { onThresholdsChanged = (thresholds: Threshold[]) => @@ -31,13 +33,22 @@ export class GaugePanelEditor extends PureComponent + this.props.onOptionsChange({ + ...this.props.options, + displayOptions, + }); + render() { const { onOptionsChange, options } = this.props; return ( <> - + {/* This just sets the 'stats', that should be moved to somethign more general */} + + + diff --git a/public/app/plugins/panel/gauge/SingleStatValueEditor.tsx b/public/app/plugins/panel/gauge/SingleStatValueEditor.tsx index e711df6a2d3..414a606b108 100644 --- a/public/app/plugins/panel/gauge/SingleStatValueEditor.tsx +++ b/public/app/plugins/panel/gauge/SingleStatValueEditor.tsx @@ -2,10 +2,10 @@ import React, { PureComponent } from 'react'; // Components -import { FormField, FormLabel, PanelOptionsGroup, Select, UnitPicker } from '@grafana/ui'; +import { FormLabel, PanelOptionsGroup, Select } from '@grafana/ui'; // Types -import { SingleStatValueOptions } from './types'; +import { GaugeOptions } from './types'; const statOptions = [ { value: 'min', label: 'Min' }, @@ -24,41 +24,18 @@ const statOptions = [ const labelWidth = 6; export interface Props { - options: SingleStatValueOptions; - onChange: (valueOptions: SingleStatValueOptions) => void; + options: GaugeOptions; + onChange: (options: GaugeOptions) => void; } export class SingleStatValueEditor extends PureComponent { - onUnitChange = unit => this.props.onChange({ ...this.props.options, unit: unit.value }); onStatChange = stat => this.props.onChange({ ...this.props.options, stat: stat.value }); - onDecimalChange = event => { - if (!isNaN(event.target.value)) { - this.props.onChange({ - ...this.props.options, - decimals: parseInt(event.target.value, 10), - }); - } else { - this.props.onChange({ - ...this.props.options, - decimals: null, - }); - } - }; - - onPrefixChange = event => this.props.onChange({ ...this.props.options, prefix: event.target.value }); - onSuffixChange = event => this.props.onChange({ ...this.props.options, suffix: event.target.value }); - render() { - const { stat, unit, decimals, prefix, suffix } = this.props.options; - - let decimalsString = ''; - if (Number.isFinite(decimals)) { - decimalsString = decimals.toString(); - } + const { stat } = this.props.options; return ( - +
Stat option.value === stat)} - /> -
- - ); - } -} diff --git a/public/app/plugins/panel/gauge/__snapshots__/module.test.ts.snap b/public/app/plugins/panel/gauge/__snapshots__/module.test.ts.snap deleted file mode 100644 index 502c8d7c853..00000000000 --- a/public/app/plugins/panel/gauge/__snapshots__/module.test.ts.snap +++ /dev/null @@ -1,31 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`Gauge Module migrations should migrate from 6.0 settings to 6.1 1`] = ` -Object { - "display": Object { - "decimals": 4, - "mappings": Array [], - "prefix": "a", - "stat": "avg", - "suffix": "z", - "thresholds": Array [ - Object { - "color": "green", - "index": 0, - "value": -Infinity, - }, - Object { - "color": "red", - "index": 1, - "value": 80, - }, - ], - "unit": "ms", - }, - "maxValue": 60, - "minValue": 50, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "stat": "avg", -} -`; diff --git a/public/app/plugins/panel/gauge/module.test.ts b/public/app/plugins/panel/gauge/module.test.ts deleted file mode 100644 index 8dc29d42643..00000000000 --- a/public/app/plugins/panel/gauge/module.test.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { gaugePanelTypeChangedHook } from './module'; - -describe('Gauge Module', () => { - describe('migrations', () => { - it('should migrate from 6.0 settings to 6.1', () => { - const v60 = { - minValue: 50, - maxValue: 60, - showThresholdMarkers: true, - showThresholdLabels: false, - valueOptions: { - prefix: 'a', - suffix: 'z', - decimals: 4, - stat: 'avg', - unit: 'ms', - }, - valueMappings: [], - thresholds: [{ index: 0, value: -Infinity, color: 'green' }, { index: 1, value: 80, color: 'red' }], - }; - - const after = gaugePanelTypeChangedHook(v60); - expect((after.stat = 'avg')); - expect(after).toMatchSnapshot(); - }); - }); -}); diff --git a/public/app/plugins/panel/gauge/module.tsx b/public/app/plugins/panel/gauge/module.tsx index e33d8f0a878..7d56ac5641b 100644 --- a/public/app/plugins/panel/gauge/module.tsx +++ b/public/app/plugins/panel/gauge/module.tsx @@ -1,58 +1,12 @@ -import { ReactPanelPlugin, DisplayValueOptions } from '@grafana/ui'; -import cloneDeep from 'lodash/cloneDeep'; +import { ReactPanelPlugin } from '@grafana/ui'; import { GaugePanelEditor } from './GaugePanelEditor'; import { GaugePanel } from './GaugePanel'; import { GaugeOptions, defaults } from './types'; +import { singleStatOptionsCheck } from '../singlestat2/module'; export const reactPanel = new ReactPanelPlugin(GaugePanel); -// Bar Gauge uses the same handler - -const optionsToCheck = ['display', 'stat', 'maxValue', 'maxValue']; - -export const gaugePanelTypeChangedHook = (options: Partial, prevPluginId?: string, prevOptions?: any) => { - // TODO! migrate to new settings format - // - // thresholds?: Threshold[]; - // valueMappings?: ValueMapping[]; - // valueOptions?: SingleStatValueOptions; - // - // if (props.options.valueOptions) { - // console.warn('TODO!! how do we best migration options?'); - // } - - // 6.0 -> 6.1, settings were stored on the root, now moved to display - if (!options.display && !prevOptions && options.hasOwnProperty('thresholds')) { - console.log('Migrating old gauge settings format', options); - const migrate = options as any; - const display = (migrate.valueOptions || {}) as DisplayValueOptions; - - display.thresholds = migrate.thresholds; - display.mappings = migrate.valueMappings; - if (migrate.valueMappings) { - options.stat = migrate.valueMappings.stat; - delete migrate.valueMappings.stat; - } - - delete migrate.valueOptions; - delete migrate.thresholds; - delete migrate.valueMappings; - - options.display = display; - } - - if (prevOptions) { - optionsToCheck.forEach(v => { - if (prevOptions.hasOwnProperty(v)) { - options[v] = cloneDeep(prevOptions.display); - } - }); - } - - return options; -}; - reactPanel.setEditor(GaugePanelEditor); reactPanel.setDefaults(defaults); -reactPanel.setPanelTypeChangedHook(gaugePanelTypeChangedHook); +reactPanel.setPanelTypeChangedHook(singleStatOptionsCheck); diff --git a/public/app/plugins/panel/gauge/types.ts b/public/app/plugins/panel/gauge/types.ts index 14c1a739dff..bab29d5d2ad 100644 --- a/public/app/plugins/panel/gauge/types.ts +++ b/public/app/plugins/panel/gauge/types.ts @@ -1,6 +1,7 @@ -import { SingleStatOptions } from '@grafana/ui'; +import { SingleStatBaseOptions } from '../singlestat2/types'; +import { VizOrientation } from '@grafana/ui'; -export interface GaugeOptions extends SingleStatOptions { +export interface GaugeOptions extends SingleStatBaseOptions { maxValue: number; minValue: number; showThresholdLabels: boolean; @@ -12,14 +13,14 @@ export const defaults: GaugeOptions = { maxValue: 100, showThresholdMarkers: true, showThresholdLabels: false, - - stat: 'avg', - display: { + valueOptions: { prefix: '', suffix: '', decimals: null, + stat: 'avg', unit: 'none', - mappings: [], - thresholds: [{ index: 0, value: -Infinity, color: 'green' }, { index: 1, value: 80, color: 'red' }], }, + valueMappings: [], + thresholds: [{ index: 0, value: -Infinity, color: 'green' }, { index: 1, value: 80, color: 'red' }], + orientation: VizOrientation.Auto, }; diff --git a/public/app/plugins/panel/singlestat2/README.md b/public/app/plugins/panel/singlestat2/README.md new file mode 100644 index 00000000000..42d72825c27 --- /dev/null +++ b/public/app/plugins/panel/singlestat2/README.md @@ -0,0 +1,9 @@ +# Singlestat Panel - Native Plugin + +The Singlestat Panel is **included** with Grafana. + +The Singlestat Panel allows you to show the one main summary stat of a SINGLE series. It reduces the series into a single number (by looking at the max, min, average, or sum of values in the series). Singlestat also provides thresholds to color the stat or the Panel background. It can also translate the single number into a text value, and show a sparkline summary of the series. + +Read more about it here: + +[http://docs.grafana.org/reference/singlestat/](http://docs.grafana.org/reference/singlestat/) \ No newline at end of file diff --git a/public/app/plugins/panel/gauge/SingleStatPanel.tsx b/public/app/plugins/panel/singlestat2/SingleStatBase.tsx similarity index 64% rename from public/app/plugins/panel/gauge/SingleStatPanel.tsx rename to public/app/plugins/panel/singlestat2/SingleStatBase.tsx index c891e05988a..fb5e54b68ed 100644 --- a/public/app/plugins/panel/gauge/SingleStatPanel.tsx +++ b/public/app/plugins/panel/singlestat2/SingleStatBase.tsx @@ -1,21 +1,16 @@ -// Libraries import React, { PureComponent } from 'react'; - -// Services & Utils -import { processSingleStatPanelData, SingleStatOptions, DisplayValue, PanelProps, VizOrientation } from '@grafana/ui'; +import { processSingleStatPanelData, DisplayValue, PanelProps } from '@grafana/ui'; import { config } from 'app/core/config'; - -// Components import { VizRepeater, getDisplayProcessor } from '@grafana/ui'; +import { SingleStatBaseOptions } from './types'; -interface State { +export interface State { values: DisplayValue[]; } -export class SingleStatPanel extends PureComponent, State> { +export class SingleStatBase extends PureComponent, State> { constructor(props: PanelProps) { super(props); - this.state = { values: this.findDisplayValues(props), }; @@ -29,18 +24,20 @@ export class SingleStatPanel extends PureComponent< findDisplayValues(props: PanelProps): DisplayValue[] { const { panelData, replaceVariables, options } = this.props; - const { display } = options; - + const { valueOptions, valueMappings } = options; const processor = getDisplayProcessor({ - ...display, - prefix: replaceVariables(display.prefix), - suffix: replaceVariables(display.suffix), + unit: valueOptions.unit, + decimals: valueOptions.decimals, + mappings: valueMappings, + thresholds: options.thresholds, + + prefix: replaceVariables(valueOptions.prefix), + suffix: replaceVariables(valueOptions.suffix), theme: config.theme, }); - return processSingleStatPanelData({ panelData: panelData, - stat: options.stat, + stat: valueOptions.stat, }).map(stat => processor(stat.value)); } @@ -51,17 +48,12 @@ export class SingleStatPanel extends PureComponent< return
{value.text}
; } - // Or we could add this to single stat props? - getOrientation(): VizOrientation { - return VizOrientation.Auto; - } - render() { - const { height, width } = this.props; + const { height, width, options } = this.props; + const { orientation } = options; const { values } = this.state; - return ( - + {({ vizHeight, vizWidth, value }) => this.renderStat(value, vizWidth, vizHeight)} ); diff --git a/public/app/plugins/panel/singlestat2/SingleStatEditor.tsx b/public/app/plugins/panel/singlestat2/SingleStatEditor.tsx new file mode 100644 index 00000000000..61b8588adce --- /dev/null +++ b/public/app/plugins/panel/singlestat2/SingleStatEditor.tsx @@ -0,0 +1,48 @@ +// Libraries +import React, { PureComponent } from 'react'; +import { + PanelEditorProps, + ThresholdsEditor, + Threshold, + PanelOptionsGrid, + ValueMappingsEditor, + ValueMapping, +} from '@grafana/ui'; + +import { SingleStatOptions, SingleStatValueOptions } from './types'; +import { SingleStatValueEditor } from './SingleStatValueEditor'; + +export class SingleStatEditor extends PureComponent> { + onThresholdsChanged = (thresholds: Threshold[]) => + this.props.onOptionsChange({ + ...this.props.options, + thresholds, + }); + + onValueMappingsChanged = (valueMappings: ValueMapping[]) => + this.props.onOptionsChange({ + ...this.props.options, + valueMappings, + }); + + onValueOptionsChanged = (valueOptions: SingleStatValueOptions) => + this.props.onOptionsChange({ + ...this.props.options, + valueOptions, + }); + + render() { + const { options } = this.props; + + return ( + <> + + + + + + + + ); + } +} diff --git a/public/app/plugins/panel/singlestat2/SingleStatPanel.tsx b/public/app/plugins/panel/singlestat2/SingleStatPanel.tsx new file mode 100644 index 00000000000..f9f20487c95 --- /dev/null +++ b/public/app/plugins/panel/singlestat2/SingleStatPanel.tsx @@ -0,0 +1,17 @@ +// Libraries +import React from 'react'; + +// Types +import { SingleStatOptions } from './types'; +import { DisplayValue } from '@grafana/ui/src/utils/displayValue'; +import { SingleStatBase } from './SingleStatBase'; + +export class SingleStatPanel extends SingleStatBase { + renderStat(value: DisplayValue, width: number, height: number) { + return ( +
+ {value.text} +
+ ); + } +} diff --git a/public/app/plugins/panel/gauge/DisplayValueEditor.tsx b/public/app/plugins/panel/singlestat2/SingleStatValueEditor.tsx similarity index 55% rename from public/app/plugins/panel/gauge/DisplayValueEditor.tsx rename to public/app/plugins/panel/singlestat2/SingleStatValueEditor.tsx index 94660de85af..e711df6a2d3 100644 --- a/public/app/plugins/panel/gauge/DisplayValueEditor.tsx +++ b/public/app/plugins/panel/singlestat2/SingleStatValueEditor.tsx @@ -2,20 +2,35 @@ import React, { PureComponent } from 'react'; // Components -import { FormField, FormLabel, PanelOptionsGroup, UnitPicker } from '@grafana/ui'; +import { FormField, FormLabel, PanelOptionsGroup, Select, UnitPicker } from '@grafana/ui'; // Types -import { DisplayValueOptions } from '@grafana/ui'; +import { SingleStatValueOptions } from './types'; + +const statOptions = [ + { value: 'min', label: 'Min' }, + { value: 'max', label: 'Max' }, + { value: 'avg', label: 'Average' }, + { value: 'current', label: 'Current' }, + { value: 'total', label: 'Total' }, + { value: 'name', label: 'Name' }, + { value: 'first', label: 'First' }, + { value: 'delta', label: 'Delta' }, + { value: 'diff', label: 'Difference' }, + { value: 'range', label: 'Range' }, + { value: 'last_time', label: 'Time of last point' }, +]; const labelWidth = 6; export interface Props { - options: DisplayValueOptions; - onChange: (options: DisplayValueOptions) => void; + options: SingleStatValueOptions; + onChange: (valueOptions: SingleStatValueOptions) => void; } -export class DisplayValueEditor extends PureComponent { +export class SingleStatValueEditor extends PureComponent { onUnitChange = unit => this.props.onChange({ ...this.props.options, unit: unit.value }); + onStatChange = stat => this.props.onChange({ ...this.props.options, stat: stat.value }); onDecimalChange = event => { if (!isNaN(event.target.value)) { @@ -35,7 +50,7 @@ export class DisplayValueEditor extends PureComponent { onSuffixChange = event => this.props.onChange({ ...this.props.options, suffix: event.target.value }); render() { - const { unit, decimals, prefix, suffix } = this.props.options; + const { stat, unit, decimals, prefix, suffix } = this.props.options; let decimalsString = ''; if (Number.isFinite(decimals)) { @@ -43,7 +58,16 @@ export class DisplayValueEditor extends PureComponent { } return ( - + +
+ Stat +