Plugins: Angular detector: Add database cache store for remote patterns (#70693)

* Plugins: Angular detector: Remote patterns fetching

* Renamed PatternType to GCOMPatternType

* Renamed files

* Renamed more files

* Moved files again

* Add type checks, unexport GCOM structs

* Cache failures, update log messages, fix GCOM URL

* Fail silently for unknown pattern types, update docstrings

* Fix tests

* Rename gcomPattern.Value to gcomPattern.Pattern

* Refactoring

* Add FlagPluginsRemoteAngularDetectionPatterns feature flag

* Fix tests

* Re-generate feature flags

* Add TestProvideInspector, renamed TestDefaultStaticDetectorsInspector

* Add TestProvideInspector

* Add TestContainsBytesDetector and TestRegexDetector

* Renamed getter to provider

* More tests

* TestStaticDetectorsProvider, TestSequenceDetectorsProvider

* GCOM tests

* Lint

* Made detector.detect unexported, updated docstrings

* Allow changing grafana.com URL

* Fix API path, add more logs

* Update tryUpdateRemoteDetectors docstring

* Use angulardetector http client

* Return false, nil if module.js does not exist

* Chore: Split angualrdetector into angularinspector and angulardetector packages

Moved files around, changed references and fixed tests:
- Split the old angulardetector package into angular/angulardetector and angular/angularinspector
- angulardetector provides the detection structs/interfaces (Detector, DetectorsProvider...)
- angularinspector provides the actual angular detection service used directly in pluginsintegration
- Exported most of the stuff that was private and now put into angulardetector, as it is not required by angularinspector

* Renamed detector.go -> angulardetector.go and inspector.go -> angularinspector.go

Forgot to rename those two files to match the package's names

* Renamed angularinspector.ProvideInspector to angularinspector.ProvideService

* Renamed "harcoded" to "static" and "remote" to "dynamic"

from PR review, matches the same naming schema used for signing keys fetching

* WIP: Angular: cache patterns in db, moved gcom into pluginsintegration

More similar to signing keys fetching

* Rename package, refactoring

* try to solve circular import

* Fix merge conflict on updated angular patterns

* Fix circular imports

* Fix wire gen

* Add docstrings, refactoring

* Removed angualrdetectorsprovider dependency into angularpatternsstore

* Moved GCOM test files

* Removed GCOM cache

* Renamed Detect to DetectAngular and Detector to AngularDetector

* Fix call to NewGCOMDetectorsProvider in newDynamicInspector

* Removed unused test function newError500GCOMScenario

* Added angularinspector service definition in pluginsintegration

* refactoring

* lint

* Fix angularinspector TestProvideService

* cleanup

* Await initial restore

* Register dynamicAngularDetector background service

* Removed static detectors provider from pluginsintegration

* Add tests for kvstore

* Add more tests

* order imports in dynamic_test.go

* Fix potential panic in dynamic_test

* Add "runs the job periodically" test

* lint

* add timeout to test

* refactoring

* Removed context.Context from DetectorsProvider

* Refactoring, ensure angular dynamic background service is not started if feature flag is off

* Fix deadlock on startup

* Fix angulardetectorsprovider tests

* Revert "Removed context.Context from DetectorsProvider"

This reverts commit 4e8c6dded7.

* Fix wrong argument number in dynamic_teset

* Standardize gcom http client

* Reduce context timeout for angular inspector in plugins loader

* Simplify initial restore logic

* Fix dynamic detectors provider tests

* Chore: removed angulardetector/provider.go

* Add more tests

* Removed backgroundJob interface, PR review feedback

* Update tests

* PR review feedback: remove ErrNoCachedValue from kv store Get

* Update tests

* PR review feedback: add IsDisabled and remove nop background srevice

* Update tests

* Remove initialRestore channel, use mux instead

* Removed backgroundJobInterval, use package-level variable instead

* Add TestDynamicAngularDetectorsProviderBackgroundService

* Removed timeouts

* pr review feedback: restore from store before returning the service

* Update tests

* Log duration on startup restore and cron run

* Switch cron job start log to debug level

* Do not attempt to restore if disabled

pkg/services/pluginsintegration/angularpatternsstore/store.go

@@ -0,0 +1,75 @@
+package angularpatternsstore
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/grafana/grafana/pkg/infra/kvstore"
+)
+
+type Service interface {
+	Get(ctx context.Context) (string, bool, error)
+	Set(ctx context.Context, patterns any) error
+	GetLastUpdated(ctx context.Context) (time.Time, error)
+}
+
+const (
+	kvNamespace = "plugin.angularpatterns"
+
+	keyPatterns    = "angular_patterns"
+	keyLastUpdated = "last_updated"
+)
+
+// KVStoreService allows to cache GCOM angular patterns into the database, as a cache.
+type KVStoreService struct {
+	kv *kvstore.NamespacedKVStore
+}
+
+func ProvideService(kv kvstore.KVStore) Service {
+	return &KVStoreService{
+		kv: kvstore.WithNamespace(kv, 0, kvNamespace),
+	}
+}
+
+// Get returns the raw cached angular detection patterns. The returned value is a JSON-encoded string.
+// If no value is present, the second argument is false and the returned error is nil.
+func (s *KVStoreService) Get(ctx context.Context) (string, bool, error) {
+	return s.kv.Get(ctx, keyPatterns)
+}
+
+// Set sets the cached angular detection patterns and the latest update time to time.Now().
+// patterns must implement json.Marshaler.
+func (s *KVStoreService) Set(ctx context.Context, patterns any) error {
+	b, err := json.Marshal(patterns)
+	if err != nil {
+		return fmt.Errorf("json marshal: %w", err)
+	}
+	if err := s.kv.Set(ctx, keyPatterns, string(b)); err != nil {
+		return fmt.Errorf("kv set: %w", err)
+	}
+	if err := s.kv.Set(ctx, keyLastUpdated, time.Now().Format(time.RFC3339)); err != nil {
+		return fmt.Errorf("kv last updated set: %w", err)
+	}
+	return nil
+}
+
+// GetLastUpdated returns the time when Set was last called. If the value cannot be unmarshalled correctly,
+// it returns a zero-value time.Time.
+func (s *KVStoreService) GetLastUpdated(ctx context.Context) (time.Time, error) {
+	v, ok, err := s.kv.Get(ctx, keyLastUpdated)
+	if err != nil {
+		return time.Time{}, fmt.Errorf("kv get: %w", err)
+	}
+	if !ok {
+		return time.Time{}, nil
+	}
+	t, err := time.Parse(time.RFC3339, v)
+	if err != nil {
+		// Ignore decode errors, so we can change the format in future versions
+		// and keep backwards/forwards compatibility
+		return time.Time{}, nil
+	}
+	return t, nil
+}

pkg/services/pluginsintegration/angularpatternsstore/store_test.go

@@ -0,0 +1,70 @@
+package angularpatternsstore
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/grafana/grafana/pkg/infra/kvstore"
+)
+
+func TestAngularPatternsStore(t *testing.T) {
+	mockPatterns := []map[string]interface{}{
+		{"name": "PanelCtrl", "type": "contains", "pattern": "PanelCtrl"},
+		{"name": "ConfigCtrl", "type": "contains", "pattern": "ConfigCtrl"},
+	}
+
+	t.Run("get set", func(t *testing.T) {
+		svc := ProvideService(kvstore.NewFakeKVStore())
+
+		t.Run("get empty", func(t *testing.T) {
+			_, ok, err := svc.Get(context.Background())
+			require.NoError(t, err)
+			require.False(t, ok)
+		})
+
+		t.Run("set and get", func(t *testing.T) {
+			err := svc.Set(context.Background(), mockPatterns)
+			require.NoError(t, err)
+
+			expV, err := json.Marshal(mockPatterns)
+			require.NoError(t, err)
+
+			dbV, ok, err := svc.Get(context.Background())
+			require.NoError(t, err)
+			require.True(t, ok)
+			require.Equal(t, string(expV), dbV)
+		})
+	})
+
+	t.Run("latest update", func(t *testing.T) {
+		svc := ProvideService(kvstore.NewFakeKVStore())
+
+		t.Run("empty", func(t *testing.T) {
+			lastUpdated, err := svc.GetLastUpdated(context.Background())
+			require.NoError(t, err)
+			require.Zero(t, lastUpdated)
+		})
+
+		t.Run("not empty", func(t *testing.T) {
+			err := svc.Set(context.Background(), mockPatterns)
+			require.NoError(t, err)
+
+			lastUpdated, err := svc.GetLastUpdated(context.Background())
+			require.NoError(t, err)
+			require.WithinDuration(t, time.Now(), lastUpdated, time.Second*10)
+		})
+
+		t.Run("invalid timestamp stored", func(t *testing.T) {
+			err := svc.(*KVStoreService).kv.Set(context.Background(), keyLastUpdated, "abcd")
+			require.NoError(t, err)
+
+			lastUpdated, err := svc.GetLastUpdated(context.Background())
+			require.NoError(t, err)
+			require.Zero(t, lastUpdated)
+		})
+	})
+}