From d99af239462cf015db935d2e34a6fd885f350dc0 Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 15 Feb 2019 14:31:52 +0100 Subject: [PATCH] add garbage collector for database cache --- pkg/infra/distcache/database_storage.go | 36 +++++++++++-- pkg/infra/distcache/database_storage_test.go | 50 +++++++++++++++++++ .../sqlstore/migrations/cache_data_mig.go | 23 +++++---- 3 files changed, 97 insertions(+), 12 deletions(-) create mode 100644 pkg/infra/distcache/database_storage_test.go diff --git a/pkg/infra/distcache/database_storage.go b/pkg/infra/distcache/database_storage.go index 8286f65fea6..ed55208e18d 100644 --- a/pkg/infra/distcache/database_storage.go +++ b/pkg/infra/distcache/database_storage.go @@ -3,18 +3,48 @@ package distcache import ( "time" + "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/services/sqlstore" ) type databaseCache struct { SQLStore *sqlstore.SqlStore + log log.Logger +} + +func newDatabaseCache(sqlstore *sqlstore.SqlStore) *databaseCache { + dc := &databaseCache{ + SQLStore: sqlstore, + log: log.New("distcache.database"), + } + + go dc.StartGC() + return dc } var getTime = time.Now -func (dc *databaseCache) Get(key string) (interface{}, error) { - //now := getTime().Unix() +func (dc *databaseCache) internalRunGC() { + now := getTime().Unix() + sql := `DELETE FROM cache_data WHERE (? - created) >= expire` + //EXTRACT(EPOCH FROM NOW()) - created >= expire + //UNIX_TIMESTAMP(NOW()) - created >= expire + _, err := dc.SQLStore.NewSession().Exec(sql, now) + if err != nil { + dc.log.Error("failed to run garbage collect", "error", err) + } +} + +func (dc *databaseCache) StartGC() { + dc.internalRunGC() + + time.AfterFunc(time.Second*10, func() { + dc.StartGC() + }) +} + +func (dc *databaseCache) Get(key string) (interface{}, error) { cacheHits := []CacheData{} err := dc.SQLStore.NewSession().Where(`key = ?`, key).Find(&cacheHits) if err != nil { @@ -65,7 +95,7 @@ func (dc *databaseCache) Put(key string, value interface{}, expire int64) error } if len(cacheHits) > 0 { - _, err = dc.SQLStore.NewSession().Exec("UPDATE cached_data SET data=?, created=?, expire=? WHERE key=?", data, now, expire, key) + _, err = dc.SQLStore.NewSession().Exec("UPDATE cache_data SET data=?, created=?, expire=? WHERE key=?", data, now, expire, key) } else { _, err = dc.SQLStore.NewSession().Exec("INSERT INTO cache_data(key,data,created_at,expires) VALUES(?,?,?,?)", key, data, now, expire) } diff --git a/pkg/infra/distcache/database_storage_test.go b/pkg/infra/distcache/database_storage_test.go new file mode 100644 index 00000000000..2e6339c7c32 --- /dev/null +++ b/pkg/infra/distcache/database_storage_test.go @@ -0,0 +1,50 @@ +package distcache + +import ( + "testing" + "time" + + "github.com/bmizerany/assert" + + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/services/sqlstore" +) + +func TestDatabaseStorageGarbageCollection(t *testing.T) { + sqlstore := sqlstore.InitTestDB(t) + + db := &databaseCache{ + SQLStore: sqlstore, + log: log.New("distcache.database"), + } + + obj := &CacheableStruct{String: "foolbar"} + + //set time.now to 2 weeks ago + getTime = func() time.Time { return time.Now().AddDate(0, 0, -2) } + db.Put("key1", obj, 1000) + db.Put("key2", obj, 1000) + db.Put("key3", obj, 1000) + + // insert object that should never expire + db.Put("key4", obj, 0) + + getTime = time.Now + db.Put("key5", obj, 1000) + + //run GC + db.internalRunGC() + + //try to read values + _, err := db.Get("key1") + assert.Equal(t, err, ErrCacheItemNotFound) + _, err = db.Get("key2") + assert.Equal(t, err, ErrCacheItemNotFound) + _, err = db.Get("key3") + assert.Equal(t, err, ErrCacheItemNotFound) + + _, err = db.Get("key4") + assert.Equal(t, err, nil) + _, err = db.Get("key5") + assert.Equal(t, err, nil) +} diff --git a/pkg/services/sqlstore/migrations/cache_data_mig.go b/pkg/services/sqlstore/migrations/cache_data_mig.go index 1201b38e337..f12f7f797c8 100644 --- a/pkg/services/sqlstore/migrations/cache_data_mig.go +++ b/pkg/services/sqlstore/migrations/cache_data_mig.go @@ -1,17 +1,22 @@ package migrations -import . "github.com/grafana/grafana/pkg/services/sqlstore/migrator" +import "github.com/grafana/grafana/pkg/services/sqlstore/migrator" -func addCacheMigration(mg *Migrator) { - var cacheDataV1 = Table{ +func addCacheMigration(mg *migrator.Migrator) { + var cacheDataV1 = migrator.Table{ Name: "cache_data", - Columns: []*Column{ - {Name: "key", Type: DB_Char, IsPrimaryKey: true, Length: 16}, - {Name: "data", Type: DB_Blob}, - {Name: "expires", Type: DB_Integer, Length: 255, Nullable: false}, - {Name: "created_at", Type: DB_Integer, Length: 255, Nullable: false}, + Columns: []*migrator.Column{ + {Name: "key", Type: migrator.DB_NVarchar, IsPrimaryKey: true, Length: 168}, + {Name: "data", Type: migrator.DB_Blob}, + {Name: "expires", Type: migrator.DB_Integer, Length: 255, Nullable: false}, + {Name: "created_at", Type: migrator.DB_Integer, Length: 255, Nullable: false}, + }, + Indices: []*migrator.Index{ + {Cols: []string{"key"}, Type: migrator.UniqueIndex}, }, } - mg.AddMigration("create cache_data table", NewAddTableMigration(cacheDataV1)) + mg.AddMigration("create cache_data table", migrator.NewAddTableMigration(cacheDataV1)) + + mg.AddMigration("add unique index cache_data.key", migrator.NewAddIndexMigration(cacheDataV1, cacheDataV1.Indices[0])) }