Alerting/Annotations: Prevent panics from composite store jobs from crashing Grafana (#83459)

* Don't directly use pointer to json

* Don't crash entire process if a store job panics

* Add debug logs when failing to parse/handle Loki entries

pkg/services/annotations/annotationsimpl/loki/historian_store.go

@@ -69,6 +69,10 @@ func NewLokiHistorianStore(cfg setting.UnifiedAlertingStateHistorySettings, ft f
 	}
 }
 
+func (r *LokiHistorianStore) Type() string {
+	return "loki"
+}
+
 func (r *LokiHistorianStore) Get(ctx context.Context, query *annotations.ItemQuery, accessResources *accesscontrol.AccessResources) ([]*annotations.ItemDTO, error) {
 	if query.Type == "annotation" {
 		return make([]*annotations.ItemDTO, 0), nil
@@ -124,6 +128,7 @@ func (r *LokiHistorianStore) annotationsFromStream(stream historian.Stream, ac a
 		err := json.Unmarshal([]byte(sample.V), &entry)
 		if err != nil {
 			// bad data, skip
+			r.log.Debug("failed to unmarshal loki entry", "error", err, "entry", sample.V)
 			continue
 		}
 
@@ -135,6 +140,7 @@ func (r *LokiHistorianStore) annotationsFromStream(stream historian.Stream, ac a
 		transition, err := buildTransition(entry)
 		if err != nil {
 			// bad data, skip
+			r.log.Debug("failed to build transition", "error", err, "entry", entry)
 			continue
 		}
 
@@ -207,6 +213,10 @@ type number interface {
 
 // numericMap converts a simplejson map[string]any to a map[string]N, where N is numeric (int or float).
 func numericMap[N number](j *simplejson.Json) (map[string]N, error) {
+	if j == nil {
+		return nil, fmt.Errorf("unexpected nil value")
+	}
+
 	m, err := j.Map()
 	if err != nil {
 		return nil, err