public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Init dualwriter metrics (#89003)

Browse Source 
* Pass prometheus registerer to the dual writer

* Fix tests

* Remove unused var

* Fix tests

* Uncomment test

* Remove leading line

* Fix tests. Reuse registerer if there's already one

* Lint

* Improve double registering logic

* Rebase main
This commit is contained in:
Leonor Oliveira
2024-06-14 11:01:49 +02:00
committed by GitHub
parent e3da5ed35d
commit fd44f2ee4f
28 changed files with 114 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/apiserver/builder/common.go
+2
View File
@@ -4,6 +4,7 @@ import (
"net/http"
grafanarest "github.com/grafana/grafana/pkg/apiserver/rest"
"github.com/prometheus/client_golang/prometheus"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
@@ -29,6 +30,7 @@ type APIGroupBuilder interface {
codecs serializer.CodecFactory,
optsGetter generic.RESTOptionsGetter,
desiredMode grafanarest.DualWriterMode,
reg prometheus.Registerer,
) (*genericapiserver.APIGroupInfo, error)
// Get OpenAPI definitions
pkg/apiserver/builder/helper.go
+3 -1
View File
@@ -25,6 +25,7 @@ import (
"github.com/grafana/grafana/pkg/apiserver/endpoints/filters"
"github.com/grafana/grafana/pkg/services/apiserver/options"
"github.com/prometheus/client_golang/prometheus"
)
// TODO: this is a temporary hack to make rest.Connecter work with resource level routes
@@ -128,6 +129,7 @@ func InstallAPIs(
optsGetter generic.RESTOptionsGetter,
builders []APIGroupBuilder,
storageOpts *options.StorageOptions,
reg prometheus.Registerer,
) error {
// dual writing is only enabled when the storage type is not legacy.
// this is needed to support setting a default RESTOptionsGetter for new APIs that don't
@@ -136,7 +138,7 @@ func InstallAPIs(
for _, b := range builders {
mode := b.GetDesiredDualWriterMode(dualWriteEnabled, storageOpts.DualWriterDesiredModes)
g, err := b.GetAPIGroupInfo(scheme, codecs, optsGetter, mode)
g, err := b.GetAPIGroupInfo(scheme, codecs, optsGetter, mode, reg)
if err != nil {
return err
}
pkg/apiserver/rest/dualwriter.go
+12 -7
View File
@@ -6,6 +6,7 @@ import (
"fmt"
"github.com/grafana/grafana/pkg/infra/kvstore"
"github.com/prometheus/client_golang/prometheus"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apiserver/pkg/registry/rest"
@@ -95,25 +96,28 @@ const (
Mode4
)
// TODO: make this function private as there should only be one public way of setting the dual writing mode
// NewDualWriter returns a new DualWriter.
func NewDualWriter(mode DualWriterMode, legacy LegacyStorage, storage Storage) DualWriter {
func NewDualWriter(mode DualWriterMode, legacy LegacyStorage, storage Storage, reg prometheus.Registerer) DualWriter {
metrics := &dualWriterMetrics{}
metrics.init(reg)
switch mode {
// It is not possible to initialize a mode 0 dual writer. Mode 0 represents
// writing to legacy storage without `unifiedStorage` enabled.
case Mode1:
// read and write only from legacy storage
return newDualWriterMode1(legacy, storage)
return newDualWriterMode1(legacy, storage, metrics)
case Mode2:
// write to both, read from storage but use legacy as backup
return newDualWriterMode2(legacy, storage)
return newDualWriterMode2(legacy, storage, metrics)
case Mode3:
// write to both, read from storage only
return newDualWriterMode3(legacy, storage)
return newDualWriterMode3(legacy, storage, metrics)
case Mode4:
// read and write only from storage
return newDualWriterMode4(legacy, storage)
return newDualWriterMode4(legacy, storage, metrics)
default:
return newDualWriterMode1(legacy, storage)
return newDualWriterMode1(legacy, storage, metrics)
}
}
@@ -142,6 +146,7 @@ func SetDualWritingMode(
storage Storage,
entity string,
desiredMode DualWriterMode,
reg prometheus.Registerer,
) (DualWriter, error) {
toMode := map[string]DualWriterMode{
// It is not possible to initialize a mode 0 dual writer. Mode 0 represents
@@ -200,5 +205,5 @@ func SetDualWritingMode(
// #TODO add support for other combinations of desired and current modes
return NewDualWriter(currentMode, legacy, storage), nil
return NewDualWriter(currentMode, legacy, storage, reg), nil
}
pkg/apiserver/rest/dualwriter_mode1.go
+2 -4
View File
@@ -24,10 +24,8 @@ const mode1Str = "1"
// NewDualWriterMode1 returns a new DualWriter in mode 1.
// Mode 1 represents writing to and reading from LegacyStorage.
func newDualWriterMode1(legacy LegacyStorage, storage Storage) *DualWriterMode1 {
metrics := &dualWriterMetrics{}
metrics.init()
return &DualWriterMode1{Legacy: legacy, Storage: storage, Log: klog.NewKlogr().WithName("DualWriterMode1"), dualWriterMetrics: metrics}
func newDualWriterMode1(legacy LegacyStorage, storage Storage, dwm *dualWriterMetrics) *DualWriterMode1 {
return &DualWriterMode1{Legacy: legacy, Storage: storage, Log: klog.NewKlogr().WithName("DualWriterMode1"), dualWriterMetrics: dwm}
}
// Mode returns the mode of the dual writer.
pkg/apiserver/rest/dualwriter_mode1_test.go
+10 -6
View File
@@ -5,6 +5,7 @@ import (
"errors"
"testing"
"github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"k8s.io/apimachinery/pkg/api/meta"
@@ -22,6 +23,8 @@ var failingObj = &example.Pod{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ObjectMeta
var exampleList = &example.PodList{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ListMeta: metav1.ListMeta{}, Items: []example.Pod{*exampleObj}}
var anotherList = &example.PodList{Items: []example.Pod{*anotherObj}}
var p = prometheus.NewRegistry()
func TestMode1_Create(t *testing.T) {
type testCase struct {
input runtime.Object
@@ -68,7 +71,7 @@ func TestMode1_Create(t *testing.T) {
tt.setupStorageFn(m)
}
dw := NewDualWriter(Mode1, ls, us)
dw := NewDualWriter(Mode1, ls, us, p)
obj, err := dw.Create(context.Background(), tt.input, func(context.Context, runtime.Object) error { return nil }, &metav1.CreateOptions{})
@@ -131,7 +134,8 @@ func TestMode1_Get(t *testing.T) {
tt.setupStorageFn(m, tt.input)
}
dw := NewDualWriter(Mode1, ls, us)
p := prometheus.NewRegistry()
dw := NewDualWriter(Mode1, ls, us, p)
obj, err := dw.Get(context.Background(), tt.input, &metav1.GetOptions{})
@@ -182,7 +186,7 @@ func TestMode1_List(t *testing.T) {
tt.setupStorageFn(m)
}
dw := NewDualWriter(Mode1, ls, us)
dw := NewDualWriter(Mode1, ls, us, p)
_, err := dw.List(context.Background(), &metainternalversion.ListOptions{})
@@ -237,7 +241,7 @@ func TestMode1_Delete(t *testing.T) {
tt.setupStorageFn(m, tt.input)
}
dw := NewDualWriter(Mode1, ls, us)
dw := NewDualWriter(Mode1, ls, us, p)
obj, _, err := dw.Delete(context.Background(), tt.input, func(ctx context.Context, obj runtime.Object) error { return nil }, &metav1.DeleteOptions{})
@@ -296,7 +300,7 @@ func TestMode1_DeleteCollection(t *testing.T) {
tt.setupStorageFn(m, tt.input)
}
dw := NewDualWriter(Mode1, ls, us)
dw := NewDualWriter(Mode1, ls, us, p)
obj, err := dw.DeleteCollection(context.Background(), func(ctx context.Context, obj runtime.Object) error { return nil }, tt.input, &metainternalversion.ListOptions{})
@@ -372,7 +376,7 @@ func TestMode1_Update(t *testing.T) {
tt.setupGetFn(m, tt.input)
}
dw := NewDualWriter(Mode1, ls, us)
dw := NewDualWriter(Mode1, ls, us, p)
obj, _, err := dw.Update(context.Background(), tt.input, updatedObjInfoObj{}, func(ctx context.Context, obj runtime.Object) error { return nil }, func(ctx context.Context, obj, old runtime.Object) error { return nil }, false, &metav1.UpdateOptions{})
pkg/apiserver/rest/dualwriter_mode2.go
+3 -6
View File
@@ -4,6 +4,7 @@ import (
"context"
"time"
"github.com/grafana/grafana/pkg/apimachinery/utils"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
@@ -13,8 +14,6 @@ import (
"k8s.io/apimachinery/pkg/selection"
"k8s.io/apiserver/pkg/registry/rest"
"k8s.io/klog/v2"
"github.com/grafana/grafana/pkg/apimachinery/utils"
)
type DualWriterMode2 struct {
@@ -28,10 +27,8 @@ const mode2Str = "2"
// NewDualWriterMode2 returns a new DualWriter in mode 2.
// Mode 2 represents writing to LegacyStorage and Storage and reading from LegacyStorage.
func newDualWriterMode2(legacy LegacyStorage, storage Storage) *DualWriterMode2 {
metrics := &dualWriterMetrics{}
metrics.init()
return &DualWriterMode2{Legacy: legacy, Storage: storage, Log: klog.NewKlogr().WithName("DualWriterMode2"), dualWriterMetrics: metrics}
func newDualWriterMode2(legacy LegacyStorage, storage Storage, dwm *dualWriterMetrics) *DualWriterMode2 {
return &DualWriterMode2{Legacy: legacy, Storage: storage, Log: klog.NewKlogr().WithName("DualWriterMode2"), dualWriterMetrics: dwm}
}
// Mode returns the mode of the dual writer.
pkg/apiserver/rest/dualwriter_mode2_test.go
+6 -6
View File
@@ -67,7 +67,7 @@ func TestMode2_Create(t *testing.T) {
tt.setupStorageFn(m, tt.input)
}
dw := NewDualWriter(Mode2, ls, us)
dw := NewDualWriter(Mode2, ls, us, p)
obj, err := dw.Create(context.Background(), tt.input, createFn, &metav1.CreateOptions{})
@@ -143,7 +143,7 @@ func TestMode2_Get(t *testing.T) {
tt.setupStorageFn(m, tt.input)
}
dw := NewDualWriter(Mode2, ls, us)
dw := NewDualWriter(Mode2, ls, us, p)
obj, err := dw.Get(context.Background(), tt.input, &metav1.GetOptions{})
@@ -196,7 +196,7 @@ func TestMode2_List(t *testing.T) {
tt.setupStorageFn(m)
}
dw := NewDualWriter(Mode2, ls, us)
dw := NewDualWriter(Mode2, ls, us, p)
obj, err := dw.List(context.Background(), &metainternalversion.ListOptions{})
@@ -289,7 +289,7 @@ func TestMode2_Delete(t *testing.T) {
tt.setupStorageFn(m, tt.input)
}
dw := NewDualWriter(Mode2, ls, us)
dw := NewDualWriter(Mode2, ls, us, p)
obj, _, err := dw.Delete(context.Background(), tt.input, func(context.Context, runtime.Object) error { return nil }, &metav1.DeleteOptions{})
@@ -361,7 +361,7 @@ func TestMode2_DeleteCollection(t *testing.T) {
tt.setupStorageFn(m)
}
dw := NewDualWriter(Mode2, ls, us)
dw := NewDualWriter(Mode2, ls, us, p)
obj, err := dw.DeleteCollection(context.Background(), func(ctx context.Context, obj runtime.Object) error { return nil }, &metav1.DeleteOptions{TypeMeta: metav1.TypeMeta{Kind: tt.input}}, &metainternalversion.ListOptions{})
@@ -469,7 +469,7 @@ func TestMode2_Update(t *testing.T) {
tt.setupStorageFn(m, tt.input)
}
dw := NewDualWriter(Mode2, ls, us)
dw := NewDualWriter(Mode2, ls, us, p)
obj, _, err := dw.Update(context.Background(), tt.input, updatedObjInfoObj{}, func(ctx context.Context, obj runtime.Object) error { return nil }, func(ctx context.Context, obj, old runtime.Object) error { return nil }, false, &metav1.UpdateOptions{})
pkg/apiserver/rest/dualwriter_mode3.go
+2 -4
View File
@@ -20,10 +20,8 @@ type DualWriterMode3 struct {
// newDualWriterMode3 returns a new DualWriter in mode 3.
// Mode 3 represents writing to LegacyStorage and Storage and reading from Storage.
func newDualWriterMode3(legacy LegacyStorage, storage Storage) *DualWriterMode3 {
metrics := &dualWriterMetrics{}
metrics.init()
return &DualWriterMode3{Legacy: legacy, Storage: storage, Log: klog.NewKlogr().WithName("DualWriterMode3"), dualWriterMetrics: metrics}
func newDualWriterMode3(legacy LegacyStorage, storage Storage, dwm *dualWriterMetrics) *DualWriterMode3 {
return &DualWriterMode3{Legacy: legacy, Storage: storage, Log: klog.NewKlogr().WithName("DualWriterMode3"), dualWriterMetrics: dwm}
}
// Mode returns the mode of the dual writer.
pkg/apiserver/rest/dualwriter_mode4.go
+2 -4
View File
@@ -19,10 +19,8 @@ type DualWriterMode4 struct {
// newDualWriterMode4 returns a new DualWriter in mode 4.
// Mode 4 represents writing and reading from Storage.
func newDualWriterMode4(legacy LegacyStorage, storage Storage) *DualWriterMode4 {
metrics := &dualWriterMetrics{}
metrics.init()
return &DualWriterMode4{Legacy: legacy, Storage: storage, Log: klog.NewKlogr().WithName("DualWriterMode4"), dualWriterMetrics: metrics}
func newDualWriterMode4(legacy LegacyStorage, storage Storage, dwm *dualWriterMetrics) *DualWriterMode4 {
return &DualWriterMode4{Legacy: legacy, Storage: storage, Log: klog.NewKlogr().WithName("DualWriterMode4"), dualWriterMetrics: dwm}
}
// Mode returns the mode of the dual writer.
pkg/apiserver/rest/dualwriter_test.go
+3 -1
View File
@@ -7,6 +7,7 @@ import (
playlist "github.com/grafana/grafana/pkg/apis/playlist/v0alpha1"
"github.com/grafana/grafana/pkg/infra/kvstore"
"github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
)
@@ -45,7 +46,8 @@ func TestSetDualWritingMode(t *testing.T) {
kvStore := kvstore.WithNamespace(kvstore.NewFakeKVStore(), 0, "storage.dualwriting."+tt.stackID)
dw, err := SetDualWritingMode(context.Background(), kvStore, ls, us, playlist.GROUPRESOURCE, tt.desiredMode)
p := prometheus.NewRegistry()
dw, err := SetDualWritingMode(context.Background(), kvStore, ls, us, playlist.GROUPRESOURCE, tt.desiredMode, p)
assert.NoError(t, err)
assert.Equal(t, tt.expectedMode, dw.Mode())
pkg/apiserver/rest/metrics.go
+9 -1
View File
@@ -5,6 +5,7 @@ import (
"time"
"github.com/prometheus/client_golang/prometheus"
"k8s.io/klog/v2"
)
type dualWriterMetrics struct {
@@ -37,10 +38,17 @@ var DualWriterOutcome = prometheus.NewHistogramVec(prometheus.HistogramOpts{
NativeHistogramBucketFactor: 1.1,
}, []string{"mode", "name", "method"})
func (m *dualWriterMetrics) init() {
func (m *dualWriterMetrics) init(reg prometheus.Registerer) {
log := klog.NewKlogr()
m.legacy = DualWriterLegacyDuration
m.storage = DualWriterStorageDuration
m.outcome = DualWriterOutcome
errLegacy := reg.Register(m.legacy)
errStorage := reg.Register(m.storage)
errOutcome := reg.Register(m.outcome)
if errLegacy != nil || errStorage != nil || errOutcome != nil {
log.Info("cloud migration metrics already registered")
}
}
func (m *dualWriterMetrics) recordLegacyDuration(isError bool, mode string, name string, method string, startFrom time.Time) {