From 3457f219be1c8bce99f713d7a907ee339ef38229 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Thu, 17 Oct 2024 13:18:29 +0300 Subject: [PATCH] Storage: Add blob storage interfaces (#90932) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jean-Philippe Quémémer --- .../feature-toggles/index.md | 1 + .../src/types/featureToggles.gen.ts | 1 + pkg/registry/apis/dashboard/legacy_storage.go | 17 +- pkg/registry/apis/dashboard/register.go | 7 +- pkg/server/module_server.go | 2 +- .../apiserver/aggregator/aggregator_test.go | 2 +- pkg/services/apiserver/builder/helper.go | 2 + pkg/services/apiserver/config.go | 1 + .../apiserver/options/grafana-aggregator.go | 2 +- .../apiserver/options/kube-aggregator.go | 2 +- pkg/services/apiserver/options/storage.go | 12 + pkg/services/apiserver/service.go | 10 +- pkg/services/featuremgmt/registry.go | 6 + pkg/services/featuremgmt/toggles_gen.csv | 1 + pkg/services/featuremgmt/toggles_gen.go | 4 + pkg/services/featuremgmt/toggles_gen.json | 25 + pkg/storage/unified/apistore/prepare.go | 35 +- pkg/storage/unified/apistore/restoptions.go | 22 +- pkg/storage/unified/apistore/store.go | 13 +- pkg/storage/unified/apistore/watcher_test.go | 1 + pkg/storage/unified/client.go | 14 +- pkg/storage/unified/resource/cdk_backend.go | 8 +- pkg/storage/unified/resource/cdk_blob.go | 185 +++++ pkg/storage/unified/resource/cdk_blob_test.go | 66 ++ pkg/storage/unified/resource/cdk_bucket.go | 183 +++++ .../unified/resource/cdk_bucket_test.go | 188 +++++ pkg/storage/unified/resource/client.go | 5 + pkg/storage/unified/resource/go.mod | 49 ++ pkg/storage/unified/resource/go.sum | 53 +- pkg/storage/unified/resource/resource.pb.go | 762 ++++++++++++++---- pkg/storage/unified/resource/resource.proto | 89 +- .../unified/resource/resource_grpc.pb.go | 133 +++ pkg/storage/unified/resource/server.go | 134 ++- pkg/storage/unified/sql/server.go | 20 +- pkg/storage/unified/sql/service.go | 6 +- .../unified/sql/test/integration_test.go | 3 +- pkg/tests/testinfra/testinfra.go | 4 +- 37 files changed, 1891 insertions(+), 177 deletions(-) create mode 100644 pkg/storage/unified/resource/cdk_blob.go create mode 100644 pkg/storage/unified/resource/cdk_blob_test.go create mode 100644 pkg/storage/unified/resource/cdk_bucket.go create mode 100644 pkg/storage/unified/resource/cdk_bucket_test.go diff --git a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md index a2d48a065ef..b2e28ff7d7a 100644 --- a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md +++ b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md @@ -209,6 +209,7 @@ Experimental features might be changed or removed without prior notice. | `alertingQueryAndExpressionsStepMode` | Enables step mode for alerting queries and expressions | | `rolePickerDrawer` | Enables the new role picker drawer design | | `pluginsSriChecks` | Enables SRI checks for plugin assets | +| `unifiedStorageBigObjectsSupport` | Enables to save big objects in blob storage | ## Development feature toggles diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index 0571e693109..fa3f8ae2268 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -220,4 +220,5 @@ export interface FeatureToggles { rolePickerDrawer?: boolean; unifiedStorageSearch?: boolean; pluginsSriChecks?: boolean; + unifiedStorageBigObjectsSupport?: boolean; } diff --git a/pkg/registry/apis/dashboard/legacy_storage.go b/pkg/registry/apis/dashboard/legacy_storage.go index f8607e2a75c..42a88385a23 100644 --- a/pkg/registry/apis/dashboard/legacy_storage.go +++ b/pkg/registry/apis/dashboard/legacy_storage.go @@ -1,6 +1,9 @@ package dashboard import ( + "context" + + "github.com/prometheus/client_golang/prometheus" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/registry/rest" @@ -9,6 +12,7 @@ import ( grafanaregistry "github.com/grafana/grafana/pkg/apiserver/registry/generic" grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" "github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/storage/unified/apistore" "github.com/grafana/grafana/pkg/storage/unified/resource" ) @@ -18,13 +22,15 @@ type dashboardStorage struct { access legacy.DashboardAccess tableConverter rest.TableConvertor - server resource.ResourceServer + server resource.ResourceServer + features featuremgmt.FeatureToggles } -func (s *dashboardStorage) newStore(scheme *runtime.Scheme, defaultOptsGetter generic.RESTOptionsGetter) (grafanarest.LegacyStorage, error) { +func (s *dashboardStorage) newStore(scheme *runtime.Scheme, defaultOptsGetter generic.RESTOptionsGetter, reg prometheus.Registerer) (grafanarest.LegacyStorage, error) { server, err := resource.NewResourceServer(resource.ResourceServerOptions{ Backend: s.access, Index: s.access, + Reg: reg, // WriteAccess: resource.WriteAccessHooks{ // Folder: func(ctx context.Context, user identity.Requester, uid string) bool { // // ??? @@ -42,8 +48,15 @@ func (s *dashboardStorage) newStore(scheme *runtime.Scheme, defaultOptsGetter ge return nil, err } client := resource.NewLocalResourceClient(server) + // This is needed as the apistore doesn't allow any core grafana dependencies. We extract the needed features + // to a map, to check them in the apistore itself. + features := make(map[string]any) + if s.features.IsEnabled(context.Background(), featuremgmt.FlagUnifiedStorageBigObjectsSupport) { + features[featuremgmt.FlagUnifiedStorageBigObjectsSupport] = struct{}{} + } optsGetter := apistore.NewRESTOptionsGetterForClient(client, defaultOpts.StorageConfig.Config, + features, ) return grafanaregistry.NewRegistryStore(scheme, resourceInfo, optsGetter) diff --git a/pkg/registry/apis/dashboard/register.go b/pkg/registry/apis/dashboard/register.go index d5ff691180a..40892d7556b 100644 --- a/pkg/registry/apis/dashboard/register.go +++ b/pkg/registry/apis/dashboard/register.go @@ -43,6 +43,7 @@ type DashboardsAPIBuilder struct { unified resource.ResourceClient log log.Logger + reg prometheus.Registerer } func RegisterAPIService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, @@ -74,7 +75,9 @@ func RegisterAPIService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, resource: dashboard.DashboardResourceInfo, access: legacy.NewDashboardAccess(dbp, namespacer, dashStore, provisioning, softDelete), tableConverter: dashboard.DashboardResourceInfo.TableConverter(), + features: features, }, + reg: reg, } apiregistration.RegisterAPI(builder) return builder @@ -125,11 +128,11 @@ func (b *DashboardsAPIBuilder) InstallSchema(scheme *runtime.Scheme) error { func (b *DashboardsAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupInfo, opts builder.APIGroupOptions) error { scheme := opts.Scheme + optsGetter := opts.OptsGetter dualWriteBuilder := opts.DualWriteBuilder - dash := b.legacy.resource - legacyStore, err := b.legacy.newStore(scheme, optsGetter) + legacyStore, err := b.legacy.newStore(scheme, optsGetter, b.reg) if err != nil { return err } diff --git a/pkg/server/module_server.go b/pkg/server/module_server.go index 053c62c837e..366f2413cdd 100644 --- a/pkg/server/module_server.go +++ b/pkg/server/module_server.go @@ -131,7 +131,7 @@ func (s *ModuleServer) Run() error { //} m.RegisterModule(modules.StorageServer, func() (services.Service, error) { - return sql.ProvideUnifiedStorageGrpcService(s.cfg, s.features, nil, s.log) + return sql.ProvideUnifiedStorageGrpcService(s.cfg, s.features, nil, s.log, nil) }) m.RegisterModule(modules.ZanzanaServer, func() (services.Service, error) { diff --git a/pkg/services/apiserver/aggregator/aggregator_test.go b/pkg/services/apiserver/aggregator/aggregator_test.go index f1a97039d61..30af4a352da 100644 --- a/pkg/services/apiserver/aggregator/aggregator_test.go +++ b/pkg/services/apiserver/aggregator/aggregator_test.go @@ -41,7 +41,7 @@ func TestAggregatorPostStartHooks(t *testing.T) { cfg.GenericConfig.SharedInformerFactory = informers.NewSharedInformerFactory(fake.NewSimpleClientset(), 10*time.Minute) // override the RESTOptionsGetter to use the in memory storage options - restOptionsGetter, err := apistore.NewRESTOptionsGetterMemory(*storagebackend.NewDefaultConfig("memory", nil)) + restOptionsGetter, err := apistore.NewRESTOptionsGetterMemory(*storagebackend.NewDefaultConfig("memory", nil), make(map[string]any)) require.NoError(t, err) cfg.GenericConfig.RESTOptionsGetter = restOptionsGetter diff --git a/pkg/services/apiserver/builder/helper.go b/pkg/services/apiserver/builder/helper.go index ae5b214ecbf..d22a7cc9b2a 100644 --- a/pkg/services/apiserver/builder/helper.go +++ b/pkg/services/apiserver/builder/helper.go @@ -7,6 +7,7 @@ import ( "regexp" "time" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/prometheus/client_golang/prometheus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -165,6 +166,7 @@ func InstallAPIs( namespaceMapper request.NamespaceMapper, kvStore grafanarest.NamespacedKVStore, serverLock ServerLockService, + features featuremgmt.FeatureToggles, ) error { // dual writing is only enabled when the storage type is not legacy. // this is needed to support setting a default RESTOptionsGetter for new APIs that don't diff --git a/pkg/services/apiserver/config.go b/pkg/services/apiserver/config.go index adde28a322a..2d126537b52 100644 --- a/pkg/services/apiserver/config.go +++ b/pkg/services/apiserver/config.go @@ -53,6 +53,7 @@ func applyGrafanaConfig(cfg *setting.Cfg, features featuremgmt.FeatureToggles, o o.StorageOptions.StorageType = options.StorageType(apiserverCfg.Key("storage_type").MustString(string(options.StorageTypeLegacy))) o.StorageOptions.DataPath = apiserverCfg.Key("storage_path").MustString(filepath.Join(cfg.DataPath, "grafana-apiserver")) o.StorageOptions.Address = apiserverCfg.Key("address").MustString(o.StorageOptions.Address) + o.StorageOptions.BlobStoreURL = apiserverCfg.Key("blob_url").MustString(o.StorageOptions.BlobStoreURL) // unified storage configs look like // [unified_storage..] diff --git a/pkg/services/apiserver/options/grafana-aggregator.go b/pkg/services/apiserver/options/grafana-aggregator.go index 693f27ddd34..33a3961605c 100644 --- a/pkg/services/apiserver/options/grafana-aggregator.go +++ b/pkg/services/apiserver/options/grafana-aggregator.go @@ -74,7 +74,7 @@ func (o *GrafanaAggregatorOptions) ApplyTo(aggregatorConfig *aggregatorapiserver return err } // override the RESTOptionsGetter to use the in memory storage options - restOptionsGetter, err := apistore.NewRESTOptionsGetterMemory(etcdOptions.StorageConfig) + restOptionsGetter, err := apistore.NewRESTOptionsGetterMemory(etcdOptions.StorageConfig, make(map[string]any)) if err != nil { return err } diff --git a/pkg/services/apiserver/options/kube-aggregator.go b/pkg/services/apiserver/options/kube-aggregator.go index 1186747f379..cf6937f5099 100644 --- a/pkg/services/apiserver/options/kube-aggregator.go +++ b/pkg/services/apiserver/options/kube-aggregator.go @@ -81,7 +81,7 @@ func (o *KubeAggregatorOptions) ApplyTo(aggregatorConfig *aggregatorapiserver.Co return err } // override the RESTOptionsGetter to use the in memory storage options - restOptionsGetter, err := apistore.NewRESTOptionsGetterMemory(etcdOptions.StorageConfig) + restOptionsGetter, err := apistore.NewRESTOptionsGetterMemory(etcdOptions.StorageConfig, make(map[string]any)) if err != nil { return err } diff --git a/pkg/services/apiserver/options/storage.go b/pkg/services/apiserver/options/storage.go index 512619baa9e..1cd9e19aabd 100644 --- a/pkg/services/apiserver/options/storage.go +++ b/pkg/services/apiserver/options/storage.go @@ -30,6 +30,13 @@ type StorageOptions struct { // The desired storage type // For file storage, this is the requested path DataPath string + // Optional blob storage connection string + // file:///path/to/dir + // gs://my-bucket (using default credentials) + // s3://my-bucket?region=us-west-1 (using default credentials) + // azblob://my-container + BlobStoreURL string + // {resource}.{group} = 1|2|3|4 UnifiedStorageConfig map[string]setting.UnifiedStorageConfig } @@ -59,6 +66,11 @@ func (o *StorageOptions) Validate() []error { if _, _, err := net.SplitHostPort(o.Address); err != nil { errs = append(errs, fmt.Errorf("--grafana-apiserver-storage-address must be a valid network address: %v", err)) } + + // Only works for single tenant grafana right now + if o.BlobStoreURL != "" && o.StorageType != StorageTypeUnified { + errs = append(errs, fmt.Errorf("blob storage is only valid with unified storage")) + } return errs } diff --git a/pkg/services/apiserver/service.go b/pkg/services/apiserver/service.go index 243b7e13af4..b1ca2d7e7ac 100644 --- a/pkg/services/apiserver/service.go +++ b/pkg/services/apiserver/service.go @@ -160,7 +160,6 @@ func ProvideService( serverLockService: serverLockService, unified: unified, } - // This will be used when running as a dskit service s.BasicService = services.NewBasicService(s.start, s.running, nil).WithName(modules.GrafanaAPIServer) @@ -290,9 +289,14 @@ func (s *service) start(ctx context.Context) error { return err } } else { + // This is needed as the apistore doesn't allow any core grafana dependencies. + features := make(map[string]any) + if s.features.IsEnabled(context.Background(), featuremgmt.FlagUnifiedStorageBigObjectsSupport) { + features[featuremgmt.FlagUnifiedStorageBigObjectsSupport] = struct{}{} + } // Use unified storage client serverConfig.Config.RESTOptionsGetter = apistore.NewRESTOptionsGetterForClient( - s.unified, o.RecommendedOptions.Etcd.StorageConfig) + s.unified, o.RecommendedOptions.Etcd.StorageConfig, features) } // Add OpenAPI specs for each group+version @@ -319,7 +323,7 @@ func (s *service) start(ctx context.Context) error { // Install the API group+version err = builder.InstallAPIs(Scheme, Codecs, server, serverConfig.RESTOptionsGetter, builders, o.StorageOptions, // Required for the dual writer initialization - s.metrics, request.GetNamespaceMapper(s.cfg), kvstore.WithNamespace(s.kvStore, 0, "storage.dualwriting"), s.serverLockService, + s.metrics, request.GetNamespaceMapper(s.cfg), kvstore.WithNamespace(s.kvStore, 0, "storage.dualwriting"), s.serverLockService, s.features, ) if err != nil { return err diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index 82a06e7885a..6948a3df00d 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -1516,6 +1516,12 @@ var ( Stage: FeatureStageExperimental, Owner: grafanaPluginsPlatformSquad, }, + { + Name: "unifiedStorageBigObjectsSupport", + Description: "Enables to save big objects in blob storage", + Stage: FeatureStageExperimental, + Owner: grafanaSearchAndStorageSquad, + }, } ) diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index 4f88acddc57..9fddabc0d8d 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -201,3 +201,4 @@ useSessionStorageForRedirection,preview,@grafana/identity-access-team,false,fals rolePickerDrawer,experimental,@grafana/identity-access-team,false,false,false unifiedStorageSearch,experimental,@grafana/search-and-storage,false,false,false pluginsSriChecks,experimental,@grafana/plugins-platform-backend,false,false,false +unifiedStorageBigObjectsSupport,experimental,@grafana/search-and-storage,false,false,false diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index 655e1992622..9c2d6d83de5 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -814,4 +814,8 @@ const ( // FlagPluginsSriChecks // Enables SRI checks for plugin assets FlagPluginsSriChecks = "pluginsSriChecks" + + // FlagUnifiedStorageBigObjectsSupport + // Enables to save big objects in blob storage + FlagUnifiedStorageBigObjectsSupport = "unifiedStorageBigObjectsSupport" ) diff --git a/pkg/services/featuremgmt/toggles_gen.json b/pkg/services/featuremgmt/toggles_gen.json index 279d08e224b..a4cf94cabdb 100644 --- a/pkg/services/featuremgmt/toggles_gen.json +++ b/pkg/services/featuremgmt/toggles_gen.json @@ -3080,6 +3080,31 @@ "requiresRestart": true } }, + { + "metadata": { + "name": "unifiedStorageBigObjectSupport", + "resourceVersion": "1728561321640", + "creationTimestamp": "2024-10-10T11:55:21Z", + "deletionTimestamp": "2024-10-15T12:09:18Z" + }, + "spec": { + "description": "Enables to save big objects in blob storage", + "stage": "experimental", + "codeowner": "@grafana/search-and-storage" + } + }, + { + "metadata": { + "name": "unifiedStorageBigObjectsSupport", + "resourceVersion": "1728994158474", + "creationTimestamp": "2024-10-15T12:09:18Z" + }, + "spec": { + "description": "Enables to save big objects in blob storage", + "stage": "experimental", + "codeowner": "@grafana/search-and-storage" + } + }, { "metadata": { "name": "unifiedStorageSearch", diff --git a/pkg/storage/unified/apistore/prepare.go b/pkg/storage/unified/apistore/prepare.go index 6933b385673..ffffe23941c 100644 --- a/pkg/storage/unified/apistore/prepare.go +++ b/pkg/storage/unified/apistore/prepare.go @@ -11,6 +11,7 @@ import ( "github.com/grafana/grafana/pkg/apimachinery/identity" "github.com/grafana/grafana/pkg/apimachinery/utils" + "github.com/grafana/grafana/pkg/storage/unified/resource" ) // Called on create @@ -45,10 +46,13 @@ func (s *Storage) prepareObjectForStorage(ctx context.Context, newObject runtime obj.SetCreatedBy(user.GetUID()) var buf bytes.Buffer - err = s.codec.Encode(newObject, &buf) - if err != nil { + if err = s.codec.Encode(newObject, &buf); err != nil { return nil, err } + + if s.largeObjectSupport { + return s.handleLargeResources(ctx, obj, buf) + } return buf.Bytes(), nil } @@ -85,9 +89,32 @@ func (s *Storage) prepareObjectForUpdate(ctx context.Context, updateObject runti obj.SetUpdatedTimestampMillis(time.Now().UnixMilli()) var buf bytes.Buffer - err = s.codec.Encode(updateObject, &buf) - if err != nil { + if err = s.codec.Encode(updateObject, &buf); err != nil { return nil, err } + if s.largeObjectSupport { + return s.handleLargeResources(ctx, obj, buf) + } + return buf.Bytes(), nil +} + +func (s *Storage) handleLargeResources(ctx context.Context, obj utils.GrafanaMetaAccessor, buf bytes.Buffer) ([]byte, error) { + if buf.Len() > 1000 { + // !!! Currently just write the whole thing + // in reality we may only want to write the spec.... + _, err := s.store.PutBlob(ctx, &resource.PutBlobRequest{ + ContentType: "application/json", + Value: buf.Bytes(), + Resource: &resource.ResourceKey{ + Group: s.gr.Group, + Resource: s.gr.Resource, + Namespace: obj.GetNamespace(), + Name: obj.GetName(), + }, + }) + if err != nil { + return nil, err + } + } return buf.Bytes(), nil } diff --git a/pkg/storage/unified/apistore/restoptions.go b/pkg/storage/unified/apistore/restoptions.go index ba7d46471a4..8fba3747ccc 100644 --- a/pkg/storage/unified/apistore/restoptions.go +++ b/pkg/storage/unified/apistore/restoptions.go @@ -24,19 +24,25 @@ import ( var _ generic.RESTOptionsGetter = (*RESTOptionsGetter)(nil) +// This is a copy of the original flag, as we are not allowed to import grafana core. +const bigObjectSupportFlag = "unifiedStorageBigObjectsSupport" + type RESTOptionsGetter struct { client resource.ResourceClient original storagebackend.Config + // As we are not allowed to import the feature management directly, we pass a map of enabled features. + features map[string]any } -func NewRESTOptionsGetterForClient(client resource.ResourceClient, original storagebackend.Config) *RESTOptionsGetter { +func NewRESTOptionsGetterForClient(client resource.ResourceClient, original storagebackend.Config, features map[string]any) *RESTOptionsGetter { return &RESTOptionsGetter{ client: client, original: original, + features: features, } } -func NewRESTOptionsGetterMemory(originalStorageConfig storagebackend.Config) (*RESTOptionsGetter, error) { +func NewRESTOptionsGetterMemory(originalStorageConfig storagebackend.Config, features map[string]any) (*RESTOptionsGetter, error) { backend, err := resource.NewCDKBackend(context.Background(), resource.CDKBackendOptions{ Bucket: memblob.OpenBucket(&memblob.Options{}), }) @@ -52,6 +58,7 @@ func NewRESTOptionsGetterMemory(originalStorageConfig storagebackend.Config) (*R return NewRESTOptionsGetterForClient( resource.NewLocalResourceClient(server), originalStorageConfig, + features, ), nil } @@ -59,7 +66,8 @@ func NewRESTOptionsGetterMemory(originalStorageConfig storagebackend.Config) (*R // for resources that are required to be read/watched on startup and there // won't be any write operations that initially bootstrap their directories func NewRESTOptionsGetterForFile(path string, - originalStorageConfig storagebackend.Config) (*RESTOptionsGetter, error) { + originalStorageConfig storagebackend.Config, + features map[string]any) (*RESTOptionsGetter, error) { if path == "" { path = filepath.Join(os.TempDir(), "grafana-apiserver") } @@ -86,6 +94,7 @@ func NewRESTOptionsGetterForFile(path string, return NewRESTOptionsGetterForClient( resource.NewLocalResourceClient(server), originalStorageConfig, + features, ), nil } @@ -122,7 +131,12 @@ func (r *RESTOptionsGetter) GetRESTOptions(resource schema.GroupResource, _ runt trigger storage.IndexerFuncs, indexers *cache.Indexers, ) (storage.Interface, factory.DestroyFunc, error) { - return NewStorage(config, r.client, keyFunc, nil, newFunc, newListFunc, getAttrsFunc, trigger, indexers) + if _, enabled := r.features[bigObjectSupportFlag]; enabled { + return NewStorage(config, r.client, keyFunc, nil, newFunc, newListFunc, getAttrsFunc, + trigger, indexers, LargeObjectSupportEnabled) + } + return NewStorage(config, r.client, keyFunc, nil, newFunc, newListFunc, getAttrsFunc, + trigger, indexers, LargeObjectSupportDisabled) }, DeleteCollectionWorkers: 0, EnableGarbageCollection: false, diff --git a/pkg/storage/unified/apistore/store.go b/pkg/storage/unified/apistore/store.go index 11a5cc3e5ea..64204714739 100644 --- a/pkg/storage/unified/apistore/store.go +++ b/pkg/storage/unified/apistore/store.go @@ -32,7 +32,11 @@ import ( "github.com/grafana/grafana/pkg/storage/unified/resource" ) -const MaxUpdateAttempts = 30 +const ( + MaxUpdateAttempts = 30 + LargeObjectSupportEnabled = true + LargeObjectSupportDisabled = false +) var _ storage.Interface = (*Storage)(nil) @@ -51,6 +55,10 @@ type Storage struct { getKey func(string) (*resource.ResourceKey, error) versioner storage.Versioner + + // Defines if we want to outsource large objects to another storage type. + // By default, this feature is disabled. + largeObjectSupport bool } // ErrFileNotExists means the file doesn't actually exist. @@ -70,6 +78,7 @@ func NewStorage( getAttrsFunc storage.AttrFunc, trigger storage.IndexerFuncs, indexers *cache.Indexers, + largeObjectSupport bool, ) (storage.Interface, factory.DestroyFunc, error) { s := &Storage{ store: store, @@ -85,6 +94,8 @@ func NewStorage( getKey: keyParser, versioner: &storage.APIObjectVersioner{}, + + largeObjectSupport: largeObjectSupport, } // The key parsing callback allows us to support the hardcoded paths from upstream tests diff --git a/pkg/storage/unified/apistore/watcher_test.go b/pkg/storage/unified/apistore/watcher_test.go index 600e8f11af8..23ae5215784 100644 --- a/pkg/storage/unified/apistore/watcher_test.go +++ b/pkg/storage/unified/apistore/watcher_test.go @@ -176,6 +176,7 @@ func testSetup(t testing.TB, opts ...setupOption) (context.Context, storage.Inte storage.DefaultNamespaceScopedAttr, make(map[string]storage.IndexerFunc, 0), nil, + LargeObjectSupportDisabled, ) if err != nil { return nil, nil, nil, err diff --git a/pkg/storage/unified/client.go b/pkg/storage/unified/client.go index 177af71a2f0..f7041599c3a 100644 --- a/pkg/storage/unified/client.go +++ b/pkg/storage/unified/client.go @@ -12,6 +12,7 @@ import ( "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/storage/unified/resource" "github.com/grafana/grafana/pkg/storage/unified/sql" + "github.com/prometheus/client_golang/prometheus" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "gocloud.dev/blob/fileblob" "google.golang.org/grpc" @@ -24,13 +25,15 @@ func ProvideUnifiedStorageClient( features featuremgmt.FeatureToggles, db infraDB.DB, tracer tracing.Tracer, + reg prometheus.Registerer, ) (resource.ResourceClient, error) { // See: apiserver.ApplyGrafanaConfig(cfg, features, o) apiserverCfg := cfg.SectionWithEnvOverrides("grafana-apiserver") opts := options.StorageOptions{ - StorageType: options.StorageType(apiserverCfg.Key("storage_type").MustString(string(options.StorageTypeLegacy))), - DataPath: apiserverCfg.Key("storage_path").MustString(filepath.Join(cfg.DataPath, "grafana-apiserver")), - Address: apiserverCfg.Key("address").MustString(""), + StorageType: options.StorageType(apiserverCfg.Key("storage_type").MustString(string(options.StorageTypeLegacy))), + DataPath: apiserverCfg.Key("storage_path").MustString(filepath.Join(cfg.DataPath, "grafana-apiserver")), + Address: apiserverCfg.Key("address").MustString(""), // client address + BlobStoreURL: apiserverCfg.Key("blob_url").MustString(""), } ctx := context.Background() @@ -54,6 +57,9 @@ func ProvideUnifiedStorageClient( } server, err := resource.NewResourceServer(resource.ResourceServerOptions{ Backend: backend, + Blob: resource.BlobConfig{ + URL: opts.BlobStoreURL, + }, }) if err != nil { return nil, err @@ -77,7 +83,7 @@ func ProvideUnifiedStorageClient( // Use the local SQL default: - server, err := sql.NewResourceServer(ctx, db, cfg, features, tracer) + server, err := sql.NewResourceServer(ctx, db, cfg, features, tracer, reg) if err != nil { return nil, err } diff --git a/pkg/storage/unified/resource/cdk_backend.go b/pkg/storage/unified/resource/cdk_backend.go index f82a3d5dddb..910a36c7096 100644 --- a/pkg/storage/unified/resource/cdk_backend.go +++ b/pkg/storage/unified/resource/cdk_backend.go @@ -2,7 +2,7 @@ package resource import ( "bytes" - context "context" + "context" "errors" "fmt" "io" @@ -25,7 +25,7 @@ import ( type CDKBackendOptions struct { Tracer trace.Tracer - Bucket *blob.Bucket + Bucket CDKBucket RootFolder string } @@ -60,7 +60,7 @@ func NewCDKBackend(ctx context.Context, opts CDKBackendOptions) (StorageBackend, type cdkBackend struct { tracer trace.Tracer - bucket *blob.Bucket + bucket CDKBucket root string mutex sync.Mutex @@ -247,7 +247,7 @@ type cdkVersion struct { } type cdkListIterator struct { - bucket *blob.Bucket + bucket CDKBucket ctx context.Context err error diff --git a/pkg/storage/unified/resource/cdk_blob.go b/pkg/storage/unified/resource/cdk_blob.go new file mode 100644 index 00000000000..00ddec4ff9d --- /dev/null +++ b/pkg/storage/unified/resource/cdk_blob.go @@ -0,0 +1,185 @@ +package resource + +import ( + "bytes" + context "context" + "crypto/md5" + "encoding/hex" + "fmt" + "mime" + "time" + + "github.com/google/uuid" + "github.com/grafana/grafana/pkg/apimachinery/utils" + "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/noop" + "gocloud.dev/blob" + + // Supported drivers + _ "gocloud.dev/blob/azureblob" + _ "gocloud.dev/blob/fileblob" + _ "gocloud.dev/blob/gcsblob" + _ "gocloud.dev/blob/memblob" + _ "gocloud.dev/blob/s3blob" +) + +type CDKBlobSupportOptions struct { + Tracer trace.Tracer + Bucket CDKBucket + RootFolder string + URLExpiration time.Duration +} + +// Called in a context that loaded the possible drivers +func OpenBlobBucket(ctx context.Context, url string) (*blob.Bucket, error) { + return blob.OpenBucket(ctx, url) +} + +func NewCDKBlobSupport(ctx context.Context, opts CDKBlobSupportOptions) (BlobSupport, error) { + if opts.Tracer == nil { + opts.Tracer = noop.NewTracerProvider().Tracer("cdk-blob-store") + } + + if opts.Bucket == nil { + return nil, fmt.Errorf("missing bucket") + } + if opts.URLExpiration < 1 { + opts.URLExpiration = time.Minute * 10 // 10 min default + } + + found, _, err := opts.Bucket.ListPage(ctx, blob.FirstPageToken, 1, &blob.ListOptions{ + Prefix: opts.RootFolder, + Delimiter: "/", + }) + if err != nil { + return nil, err + } + if found == nil { + return nil, fmt.Errorf("the root folder does not exist") + } + + return &cdkBlobSupport{ + tracer: opts.Tracer, + bucket: opts.Bucket, + root: opts.RootFolder, + cansignurls: false, // TODO depends on the implementation + expiration: opts.URLExpiration, + }, nil +} + +type cdkBlobSupport struct { + tracer trace.Tracer + bucket CDKBucket + root string + cansignurls bool + expiration time.Duration +} + +func (s *cdkBlobSupport) getBlobPath(key *ResourceKey, info *utils.BlobInfo) (string, error) { + var buffer bytes.Buffer + buffer.WriteString(s.root) + + if key.Namespace == "" { + buffer.WriteString("__cluster__/") + } else { + buffer.WriteString(key.Namespace) + buffer.WriteString("/") + } + + if key.Group == "" { + return "", fmt.Errorf("missing group") + } + buffer.WriteString(key.Group) + buffer.WriteString("/") + + if key.Resource == "" { + return "", fmt.Errorf("missing resource") + } + buffer.WriteString(key.Resource) + buffer.WriteString("/") + + if key.Name == "" { + return "", fmt.Errorf("missing name") + } + buffer.WriteString(key.Name) + buffer.WriteString("/") + buffer.WriteString(info.UID) + + ext, err := mime.ExtensionsByType(info.MimeType) + if err != nil { + return "", err + } + if len(ext) > 0 { + buffer.WriteString(ext[0]) + } + return buffer.String(), nil +} + +func (s *cdkBlobSupport) SupportsSignedURLs() bool { + return s.cansignurls +} + +func (s *cdkBlobSupport) PutResourceBlob(ctx context.Context, req *PutBlobRequest) (*PutBlobResponse, error) { + info := &utils.BlobInfo{ + UID: uuid.New().String(), + } + info.SetContentType(req.ContentType) + path, err := s.getBlobPath(req.Resource, info) + if err != nil { + return nil, err + } + + rsp := &PutBlobResponse{Uid: info.UID, MimeType: info.MimeType, Charset: info.Charset} + if req.Method == PutBlobRequest_HTTP { + rsp.Url, err = s.bucket.SignedURL(ctx, path, &blob.SignedURLOptions{ + Method: "PUT", + Expiry: s.expiration, + ContentType: req.ContentType, + }) + return rsp, err + } + if len(req.Value) < 1 { + return nil, fmt.Errorf("missing content value") + } + + // Write the value + err = s.bucket.WriteAll(ctx, path, req.Value, &blob.WriterOptions{ + ContentType: req.ContentType, + }) + if err != nil { + return nil, err + } + + attrs, err := s.bucket.Attributes(ctx, path) + if err != nil { + return nil, err + } + rsp.Size = attrs.Size + + // Set the MD5 hash if missing + if len(attrs.MD5) == 0 { + h := md5.New() + _, _ = h.Write(req.Value) + attrs.MD5 = h.Sum(nil) + } + rsp.Hash = hex.EncodeToString(attrs.MD5[:]) + return rsp, err +} + +func (s *cdkBlobSupport) GetResourceBlob(ctx context.Context, resource *ResourceKey, info *utils.BlobInfo, mustProxy bool) (*GetBlobResponse, error) { + path, err := s.getBlobPath(resource, info) + if err != nil { + return nil, err + } + rsp := &GetBlobResponse{ContentType: info.ContentType()} + if mustProxy || !s.cansignurls { + rsp.Value, err = s.bucket.ReadAll(ctx, path) + return rsp, err + } + rsp.Url, err = s.bucket.SignedURL(ctx, path, &blob.SignedURLOptions{ + Method: "GET", + Expiry: s.expiration, + ContentType: rsp.ContentType, + }) + return rsp, err +} diff --git a/pkg/storage/unified/resource/cdk_blob_test.go b/pkg/storage/unified/resource/cdk_blob_test.go new file mode 100644 index 00000000000..65ea7dcd7ec --- /dev/null +++ b/pkg/storage/unified/resource/cdk_blob_test.go @@ -0,0 +1,66 @@ +package resource + +import ( + "context" + "fmt" + "os" + "testing" + + "github.com/grafana/grafana/pkg/apimachinery/utils" + "github.com/stretchr/testify/require" + "gocloud.dev/blob/fileblob" + "gocloud.dev/blob/memblob" +) + +func TestCDKBlobStore(t *testing.T) { + bucket := memblob.OpenBucket(nil) + if false { + tmp, err := os.MkdirTemp("", "xxx-*") + require.NoError(t, err) + + bucket, err = fileblob.OpenBucket(tmp, &fileblob.Options{ + CreateDir: true, + Metadata: fileblob.MetadataDontWrite, // skip + }) + require.NoError(t, err) + + fmt.Printf("ROOT: %s\n\n", tmp) + } + ctx := context.Background() + + store, err := NewCDKBlobSupport(ctx, CDKBlobSupportOptions{ + Bucket: bucket, + //RootFolder: "xyz", + }) + require.NoError(t, err) + + t.Run("can write then read a blob", func(t *testing.T) { + raw := []byte(`{"hello": "world"}`) + key := &ResourceKey{ + Group: "playlist.grafana.app", + Resource: "rrrr", // can be anything + Namespace: "default", + Name: "fdgsv37qslr0ga", + } + + rsp, err := store.PutResourceBlob(ctx, &PutBlobRequest{ + Resource: key, + Method: PutBlobRequest_GRPC, + ContentType: "application/json", + Value: raw, + }) + require.NoError(t, err) + require.Equal(t, "49dfdd54b01cbcd2d2ab5e9e5ee6b9b9", rsp.Hash) + + found, err := store.GetResourceBlob(ctx, key, &utils.BlobInfo{ + UID: rsp.Uid, + Size: rsp.Size, + Hash: rsp.Hash, + MimeType: rsp.MimeType, + Charset: rsp.Charset, + }, false) + require.NoError(t, err) + require.Equal(t, raw, found.Value) + require.Equal(t, "application/json", found.ContentType) + }) +} diff --git a/pkg/storage/unified/resource/cdk_bucket.go b/pkg/storage/unified/resource/cdk_bucket.go new file mode 100644 index 00000000000..7a789c26eb1 --- /dev/null +++ b/pkg/storage/unified/resource/cdk_bucket.go @@ -0,0 +1,183 @@ +package resource + +import ( + "context" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + "gocloud.dev/blob" +) + +// CDKBucket is an abstraction that provides the same functionality as gocloud.dev/blob.Bucket +// It can be used to wrap gocloud.dev/blob.Bucket with some useful things as o11y. +type CDKBucket interface { + Attributes(context.Context, string) (*blob.Attributes, error) + List(*blob.ListOptions) *blob.ListIterator + ListPage(context.Context, []byte, int, *blob.ListOptions) ([]*blob.ListObject, []byte, error) + WriteAll(context.Context, string, []byte, *blob.WriterOptions) error + ReadAll(context.Context, string) ([]byte, error) + SignedURL(context.Context, string, *blob.SignedURLOptions) (string, error) +} + +var _ CDKBucket = (*blob.Bucket)(nil) + +const ( + cdkBucketOperationLabel = "operation" + cdkBucketStatusLabel = "status" + cdkBucketStatusSuccess = "success" + cdkBucketStatusError = "error" +) + +type InstrumentedBucket struct { + requests *prometheus.CounterVec + latency *prometheus.HistogramVec + tracer trace.Tracer + bucket CDKBucket +} + +func NewInstrumentedBucket(bucket CDKBucket, reg prometheus.Registerer, tracer trace.Tracer) *InstrumentedBucket { + b := &InstrumentedBucket{ + bucket: bucket, + tracer: tracer, + } + b.initMetrics(reg) + return b +} + +func (b *InstrumentedBucket) initMetrics(reg prometheus.Registerer) { + b.requests = promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ + Name: "cdk_blobstorage_requests_total", + }, []string{ + cdkBucketOperationLabel, + cdkBucketStatusLabel, + }) + b.latency = promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ + Name: "cdk_blobstorage_latency_seconds", + Buckets: prometheus.ExponentialBuckets(0.008, 4, 7), + }, []string{ + cdkBucketOperationLabel, + cdkBucketStatusLabel, + }) +} + +func (b *InstrumentedBucket) Attributes(ctx context.Context, key string) (*blob.Attributes, error) { + ctx, span := b.tracer.Start(ctx, "InstrumentedBucket/Attributes") + defer span.End() + start := time.Now() + retVal, err := b.bucket.Attributes(ctx, key) + end := time.Since(start).Seconds() + labels := prometheus.Labels{ + cdkBucketOperationLabel: "Attributes", + } + if err != nil { + labels[cdkBucketStatusLabel] = cdkBucketStatusError + b.requests.With(labels).Inc() + b.latency.With(labels).Observe(end) + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + return retVal, err + } + labels[cdkBucketStatusLabel] = cdkBucketStatusSuccess + b.requests.With(labels).Inc() + b.latency.With(labels).Observe(end) + return retVal, err +} + +func (b *InstrumentedBucket) List(opts *blob.ListOptions) *blob.ListIterator { + // List just returns an iterator struct based on the provided options. No need for extended telemetry. + return b.bucket.List(opts) +} + +func (b *InstrumentedBucket) ListPage(ctx context.Context, pageToken []byte, pageSize int, opts *blob.ListOptions) ([]*blob.ListObject, []byte, error) { + ctx, span := b.tracer.Start(ctx, "InstrumentedBucket/ListPage") + defer span.End() + start := time.Now() + retVal, nextPageToken, err := b.bucket.ListPage(ctx, pageToken, pageSize, opts) + end := time.Since(start).Seconds() + labels := prometheus.Labels{ + cdkBucketOperationLabel: "ListPage", + } + if err != nil { + labels[cdkBucketStatusLabel] = cdkBucketStatusError + b.latency.With(labels).Observe(end) + b.requests.With(labels).Inc() + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + return retVal, nextPageToken, err + } + labels[cdkBucketStatusLabel] = cdkBucketStatusSuccess + b.requests.With(labels).Inc() + return retVal, nextPageToken, err +} + +func (b *InstrumentedBucket) ReadAll(ctx context.Context, key string) ([]byte, error) { + ctx, span := b.tracer.Start(ctx, "InstrumentedBucket/ReadAll") + defer span.End() + start := time.Now() + retVal, err := b.bucket.ReadAll(ctx, key) + end := time.Since(start).Seconds() + labels := prometheus.Labels{ + cdkBucketOperationLabel: "ReadAll", + } + if err != nil { + labels[cdkBucketStatusLabel] = cdkBucketStatusError + b.requests.With(labels).Inc() + b.latency.With(labels).Observe(end) + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + return retVal, err + } + labels[cdkBucketStatusLabel] = cdkBucketStatusSuccess + b.requests.With(labels).Inc() + b.latency.With(labels).Observe(end) + return retVal, err +} + +func (b *InstrumentedBucket) WriteAll(ctx context.Context, key string, p []byte, opts *blob.WriterOptions) error { + ctx, span := b.tracer.Start(ctx, "InstrumentedBucket/WriteAll") + defer span.End() + start := time.Now() + err := b.bucket.WriteAll(ctx, key, p, opts) + end := time.Since(start).Seconds() + labels := prometheus.Labels{ + cdkBucketOperationLabel: "WriteAll", + } + if err != nil { + labels[cdkBucketStatusLabel] = cdkBucketStatusError + b.requests.With(labels).Inc() + b.latency.With(labels).Observe(end) + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + return err + } + labels[cdkBucketStatusLabel] = cdkBucketStatusSuccess + b.requests.With(labels).Inc() + b.latency.With(labels).Observe(end) + return err +} + +func (b *InstrumentedBucket) SignedURL(ctx context.Context, key string, opts *blob.SignedURLOptions) (string, error) { + ctx, span := b.tracer.Start(ctx, "InstrumentedBucket/SignedURL") + defer span.End() + start := time.Now() + retVal, err := b.bucket.SignedURL(ctx, key, opts) + end := time.Since(start).Seconds() + labels := prometheus.Labels{ + cdkBucketOperationLabel: "SignedURL", + } + if err != nil { + labels[cdkBucketStatusLabel] = cdkBucketStatusError + b.requests.With(labels).Inc() + b.latency.With(labels).Observe(end) + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + return retVal, err + } + labels[cdkBucketStatusLabel] = cdkBucketStatusSuccess + b.requests.With(labels).Inc() + b.latency.With(labels).Observe(end) + return retVal, err +} diff --git a/pkg/storage/unified/resource/cdk_bucket_test.go b/pkg/storage/unified/resource/cdk_bucket_test.go new file mode 100644 index 00000000000..71949d94f80 --- /dev/null +++ b/pkg/storage/unified/resource/cdk_bucket_test.go @@ -0,0 +1,188 @@ +package resource + +import ( + "context" + "fmt" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + "gocloud.dev/blob" +) + +type fakeCDKBucket struct { + attributesFunc func(ctx context.Context, key string) (*blob.Attributes, error) + writeAllFunc func(ctx context.Context, key string, p []byte, opts *blob.WriterOptions) error + readAllFunc func(ctx context.Context, key string) ([]byte, error) + signedURLFunc func(ctx context.Context, key string, opts *blob.SignedURLOptions) (string, error) + listFunc func(opts *blob.ListOptions) *blob.ListIterator + listPageFunc func(ctx context.Context, pageToken []byte, pageSize int, opts *blob.ListOptions) ([]*blob.ListObject, []byte, error) +} + +func (f *fakeCDKBucket) Attributes(ctx context.Context, key string) (*blob.Attributes, error) { + if f.attributesFunc != nil { + return f.attributesFunc(ctx, key) + } + return nil, nil +} + +func (f *fakeCDKBucket) WriteAll(ctx context.Context, key string, p []byte, opts *blob.WriterOptions) error { + if f.writeAllFunc != nil { + return f.writeAllFunc(ctx, key, p, opts) + } + return nil +} + +func (f *fakeCDKBucket) ReadAll(ctx context.Context, key string) ([]byte, error) { + if f.readAllFunc != nil { + return f.readAllFunc(ctx, key) + } + return nil, nil +} + +func (f *fakeCDKBucket) SignedURL(ctx context.Context, key string, opts *blob.SignedURLOptions) (string, error) { + if f.signedURLFunc != nil { + return f.signedURLFunc(ctx, key, opts) + } + return "", nil +} + +func (f *fakeCDKBucket) List(opts *blob.ListOptions) *blob.ListIterator { + if f.listFunc != nil { + return f.listFunc(opts) + } + return nil +} + +func (f *fakeCDKBucket) ListPage(ctx context.Context, pageToken []byte, pageSize int, opts *blob.ListOptions) ([]*blob.ListObject, []byte, error) { + if f.listPageFunc != nil { + return f.listPageFunc(ctx, pageToken, pageSize, opts) + } + return nil, nil, nil +} + +func TestInstrumentedBucket(t *testing.T) { + operations := []struct { + name string + operation string + setup func(fakeBucket *fakeCDKBucket, success bool) + call func(instrumentedBucket *InstrumentedBucket) error + }{ + { + name: "Attributes", + operation: "Attributes", + setup: func(fakeBucket *fakeCDKBucket, success bool) { + if success { + fakeBucket.attributesFunc = func(ctx context.Context, key string) (*blob.Attributes, error) { + return &blob.Attributes{}, nil + } + } else { + fakeBucket.attributesFunc = func(ctx context.Context, key string) (*blob.Attributes, error) { + return nil, fmt.Errorf("some error") + } + } + }, + call: func(instrumentedBucket *InstrumentedBucket) error { + _, err := instrumentedBucket.Attributes(context.Background(), "key") + return err + }, + }, + { + name: "WriteAll", + operation: "WriteAll", + setup: func(fakeBucket *fakeCDKBucket, success bool) { + if success { + fakeBucket.writeAllFunc = func(ctx context.Context, key string, p []byte, opts *blob.WriterOptions) error { + return nil + } + } else { + fakeBucket.writeAllFunc = func(ctx context.Context, key string, p []byte, opts *blob.WriterOptions) error { + return fmt.Errorf("some error") + } + } + }, + call: func(instrumentedBucket *InstrumentedBucket) error { + err := instrumentedBucket.WriteAll(context.Background(), "key", []byte("data"), nil) + return err + }, + }, + { + name: "ReadAll", + operation: "ReadAll", + setup: func(fakeBucket *fakeCDKBucket, success bool) { + if success { + fakeBucket.readAllFunc = func(ctx context.Context, key string) ([]byte, error) { + return []byte("data"), nil + } + } else { + fakeBucket.readAllFunc = func(ctx context.Context, key string) ([]byte, error) { + return nil, fmt.Errorf("some error") + } + } + }, + call: func(instrumentedBucket *InstrumentedBucket) error { + _, err := instrumentedBucket.ReadAll(context.Background(), "key") + return err + }, + }, + { + name: "SignedURL", + operation: "SignedURL", + setup: func(fakeBucket *fakeCDKBucket, success bool) { + if success { + fakeBucket.signedURLFunc = func(ctx context.Context, key string, opts *blob.SignedURLOptions) (string, error) { + return "http://signed.url", nil + } + } else { + fakeBucket.signedURLFunc = func(ctx context.Context, key string, opts *blob.SignedURLOptions) (string, error) { + return "", fmt.Errorf("some error") + } + } + }, + call: func(instrumentedBucket *InstrumentedBucket) error { + _, err := instrumentedBucket.SignedURL(context.Background(), "key", nil) + return err + }, + }, + } + + for _, op := range operations { + for _, tc := range []struct { + name string + success bool + expectedCountLabel string + }{ + { + name: "success", + success: true, + expectedCountLabel: cdkBucketStatusSuccess, + }, + { + name: "failure", + success: false, + expectedCountLabel: cdkBucketStatusError, + }, + } { + t.Run(op.name+" "+tc.name, func(t *testing.T) { + fakeBucket := &fakeCDKBucket{} + reg := prometheus.NewPedanticRegistry() + tracer := otel.Tracer("test") + instrumentedBucket := NewInstrumentedBucket(fakeBucket, reg, tracer) + + op.setup(fakeBucket, tc.success) + err := op.call(instrumentedBucket) + + if tc.success { + require.NoError(t, err) + } else { + require.Error(t, err) + } + + count := testutil.ToFloat64(instrumentedBucket.requests.WithLabelValues(op.operation, tc.expectedCountLabel)) + require.Equal(t, 1.0, count) + }) + } + } +} diff --git a/pkg/storage/unified/resource/client.go b/pkg/storage/unified/resource/client.go index bf51b5f1bce..cecbcfcb3b8 100644 --- a/pkg/storage/unified/resource/client.go +++ b/pkg/storage/unified/resource/client.go @@ -23,6 +23,7 @@ import ( type ResourceClient interface { ResourceStoreClient ResourceIndexClient + BlobStoreClient DiagnosticsClient } @@ -30,6 +31,7 @@ type ResourceClient interface { type resourceClient struct { ResourceStoreClient ResourceIndexClient + BlobStoreClient DiagnosticsClient } @@ -38,6 +40,7 @@ func NewResourceClient(channel *grpc.ClientConn) ResourceClient { return &resourceClient{ ResourceStoreClient: NewResourceStoreClient(cc), ResourceIndexClient: NewResourceIndexClient(cc), + BlobStoreClient: NewBlobStoreClient(cc), DiagnosticsClient: NewDiagnosticsClient(cc), } } @@ -49,6 +52,7 @@ func NewLocalResourceClient(server ResourceServer) ResourceClient { for _, desc := range []*grpc.ServiceDesc{ &ResourceStore_ServiceDesc, &ResourceIndex_ServiceDesc, + &BlobStore_ServiceDesc, &Diagnostics_ServiceDesc, } { channel.RegisterService( @@ -71,6 +75,7 @@ func NewLocalResourceClient(server ResourceServer) ResourceClient { return &resourceClient{ ResourceStoreClient: NewResourceStoreClient(cc), ResourceIndexClient: NewResourceIndexClient(cc), + BlobStoreClient: NewBlobStoreClient(cc), DiagnosticsClient: NewDiagnosticsClient(cc), } } diff --git a/pkg/storage/unified/resource/go.mod b/pkg/storage/unified/resource/go.mod index 7f5f354e248..826e43f3092 100644 --- a/pkg/storage/unified/resource/go.mod +++ b/pkg/storage/unified/resource/go.mod @@ -4,6 +4,7 @@ go 1.23.1 require ( github.com/fullstorydev/grpchan v1.1.1 + github.com/google/uuid v1.6.0 github.com/grafana/authlib v0.0.0-20240906122029-0100695765b9 github.com/grafana/authlib/claims v0.0.0-20240903121118-16441568af1e github.com/grafana/grafana/pkg/apimachinery v0.0.0-20240808164224-787abccfbc9e @@ -70,41 +71,89 @@ require ( ) require ( + cloud.google.com/go v0.115.0 // indirect + cloud.google.com/go/auth v0.8.1 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect + cloud.google.com/go/compute/metadata v0.5.0 // indirect + cloud.google.com/go/iam v1.1.13 // indirect + cloud.google.com/go/storage v1.43.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2 // indirect + github.com/Azure/go-autorest v14.2.0+incompatible // indirect + github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect + github.com/aws/aws-sdk-go v1.55.5 // indirect + github.com/aws/aws-sdk-go-v2 v1.30.3 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 // indirect + github.com/aws/aws-sdk-go-v2/config v1.27.27 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.27 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 // indirect + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.58.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 // indirect + github.com/aws/smithy-go v1.20.3 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blevesearch/bleve/v2 v2.4.2 github.com/bufbuild/protocompile v0.4.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-jose/go-jose/v3 v3.0.3 // indirect github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v5 v5.2.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gofuzz v1.2.0 // indirect + github.com/google/s2a-go v0.1.8 // indirect + github.com/google/wire v0.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/google/uuid v1.6.0 github.com/googleapis/gax-go/v2 v2.13.0 // indirect github.com/jhump/protoreflect v1.15.1 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/x448/float16 v0.8.4 // indirect go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect go.opentelemetry.io/otel v1.29.0 // indirect + go.opentelemetry.io/otel/metric v1.29.0 // indirect + go.opentelemetry.io/otel/sdk v1.29.0 // indirect golang.org/x/crypto v0.27.0 // indirect golang.org/x/net v0.29.0 // indirect golang.org/x/oauth2 v0.23.0 // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.25.0 // indirect golang.org/x/text v0.18.0 // indirect + golang.org/x/time v0.6.0 // indirect golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect google.golang.org/api v0.191.0 // indirect + google.golang.org/genproto v0.0.0-20240812133136-8ffd90a71988 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240822170219-fc7c04adadcd // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/pkg/storage/unified/resource/go.sum b/pkg/storage/unified/resource/go.sum index 391c6c08d49..e5bec401022 100644 --- a/pkg/storage/unified/resource/go.sum +++ b/pkg/storage/unified/resource/go.sum @@ -5,13 +5,30 @@ cloud.google.com/go/auth v0.8.1 h1:QZW9FjC5lZzN864p13YxvAtGUlQ+KgRL+8Sg45Z6vxo= cloud.google.com/go/auth v0.8.1/go.mod h1:qGVp/Y3kDRSDZ5gFD/XPUfYQ9xW1iI7q8RIRoCyBbJc= cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY= cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc= -cloud.google.com/go/compute v1.23.4 h1:EBT9Nw4q3zyE7G45Wvv3MzolIrCJEuHys5muLY0wvAw= cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY= cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY= cloud.google.com/go/iam v1.1.13 h1:7zWBXG9ERbMLrzQBRhFliAV+kjcRToDTgQT3CTwYyv4= cloud.google.com/go/iam v1.1.13/go.mod h1:K8mY0uSXwEXS30KrnVb+j54LB/ntfZu1dr+4zFMNbus= +cloud.google.com/go/longrunning v0.5.12 h1:5LqSIdERr71CqfUsFlJdBpOkBH8FBCFD7P1nTWy3TYE= +cloud.google.com/go/longrunning v0.5.12/go.mod h1:S5hMV8CDJ6r50t2ubVJSKQVv5u0rmik5//KgLO3k4lU= cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs= cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 h1:nyQWyZvwGTvunIMxi1Y9uXkcyr+I7TeNrr/foo4Kpk8= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0/go.mod h1:T5RfihdXtBDxt1Ch2wobif3TvzTdumDy29kahv6AV9A= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2 h1:YUUxeiOWgdAQE3pXt2H7QXzZs0q8UBjgRbl56qo8GYM= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2/go.mod h1:dmXQgZuiSubAecswZE+Sm8jkvEa7kQgTPVRvwL/nd0E= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= +github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/RoaringBitmap/roaring v1.9.3 h1:t4EbC5qQwnisr5PrP9nt0IRhRTb9gMUgQF4t4S2OByM= github.com/RoaringBitmap/roaring v1.9.3/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= @@ -120,12 +137,15 @@ github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/go-jose/go-jose/v3 v3.0.3 h1:fFKWeig/irsp7XD2zBxvnmA/XaRWp5V3CBsZXJF7G7k= github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo= github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -158,11 +178,18 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-replayers/grpcreplay v1.3.0 h1:1Keyy0m1sIpqstQmgz307zhiJ1pV4uIlFds5weTmxbo= +github.com/google/go-replayers/grpcreplay v1.3.0/go.mod h1:v6NgKtkijC0d3e3RW8il6Sy5sqRVUwoQa4mHOGEy8DI= +github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQEEioJvFYxYcLRKzk= +github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy+tME4bwyqPcwWbNlUI1Mcg= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= +github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -188,6 +215,8 @@ github.com/jhump/protoreflect v1.15.1 h1:HUMERORf3I3ZdX05WaQ6MIpd/NJ434hTp5YiKgf github.com/jhump/protoreflect v1.15.1/go.mod h1:jD/2GMKKE6OqX8qTjhADU1e6DShO+gavG9e0Q693nKo= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -201,6 +230,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -212,6 +243,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -256,6 +289,8 @@ go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc= go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8= +go.opentelemetry.io/otel/sdk v1.29.0 h1:vkqKjk7gwhS8VaWb0POZKmIEDimRCMsopNYnriHyryo= +go.opentelemetry.io/otel/sdk v1.29.0/go.mod h1:pM8Dx5WKnvxLCb+8lG1PRNIDxu9g9b9g59Qr7hfAAok= go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= gocloud.dev v0.39.0 h1:EYABYGhAalPUaMrbSKOr5lejxoxvXj99nE8XFtsDgds= @@ -264,6 +299,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= @@ -275,6 +312,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -289,6 +328,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -301,6 +342,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -312,8 +355,11 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -321,6 +367,8 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -328,6 +376,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= @@ -343,6 +392,8 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/storage/unified/resource/resource.pb.go b/pkg/storage/unified/resource/resource.pb.go index 1a641f12284..7f43b6899c2 100644 --- a/pkg/storage/unified/resource/resource.pb.go +++ b/pkg/storage/unified/resource/resource.pb.go @@ -176,6 +176,54 @@ func (HealthCheckResponse_ServingStatus) EnumDescriptor() ([]byte, []int) { return file_resource_proto_rawDescGZIP(), []int{28, 0} } +type PutBlobRequest_Method int32 + +const ( + // Use the inline raw []byte + PutBlobRequest_GRPC PutBlobRequest_Method = 0 + // Get a signed URL and PUT the value + PutBlobRequest_HTTP PutBlobRequest_Method = 1 +) + +// Enum value maps for PutBlobRequest_Method. +var ( + PutBlobRequest_Method_name = map[int32]string{ + 0: "GRPC", + 1: "HTTP", + } + PutBlobRequest_Method_value = map[string]int32{ + "GRPC": 0, + "HTTP": 1, + } +) + +func (x PutBlobRequest_Method) Enum() *PutBlobRequest_Method { + p := new(PutBlobRequest_Method) + *p = x + return p +} + +func (x PutBlobRequest_Method) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (PutBlobRequest_Method) Descriptor() protoreflect.EnumDescriptor { + return file_resource_proto_enumTypes[3].Descriptor() +} + +func (PutBlobRequest_Method) Type() protoreflect.EnumType { + return &file_resource_proto_enumTypes[3] +} + +func (x PutBlobRequest_Method) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use PutBlobRequest_Method.Descriptor instead. +func (PutBlobRequest_Method) EnumDescriptor() ([]byte, []int) { + return file_resource_proto_rawDescGZIP(), []int{29, 0} +} + type ResourceKey struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2246,6 +2294,327 @@ func (x *HealthCheckResponse) GetStatus() HealthCheckResponse_ServingStatus { return HealthCheckResponse_UNKNOWN } +type PutBlobRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The resource that will use this blob + // NOTE: the name may not yet exist, but group+resource are required + Resource *ResourceKey `protobuf:"bytes,1,opt,name=resource,proto3" json:"resource,omitempty"` + // How to upload + Method PutBlobRequest_Method `protobuf:"varint,2,opt,name=method,proto3,enum=resource.PutBlobRequest_Method" json:"method,omitempty"` + // Content type header + ContentType string `protobuf:"bytes,3,opt,name=content_type,json=contentType,proto3" json:"content_type,omitempty"` + // Raw value to write + // Not valid when method == HTTP + Value []byte `protobuf:"bytes,4,opt,name=value,proto3" json:"value,omitempty"` +} + +func (x *PutBlobRequest) Reset() { + *x = PutBlobRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_resource_proto_msgTypes[29] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PutBlobRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PutBlobRequest) ProtoMessage() {} + +func (x *PutBlobRequest) ProtoReflect() protoreflect.Message { + mi := &file_resource_proto_msgTypes[29] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PutBlobRequest.ProtoReflect.Descriptor instead. +func (*PutBlobRequest) Descriptor() ([]byte, []int) { + return file_resource_proto_rawDescGZIP(), []int{29} +} + +func (x *PutBlobRequest) GetResource() *ResourceKey { + if x != nil { + return x.Resource + } + return nil +} + +func (x *PutBlobRequest) GetMethod() PutBlobRequest_Method { + if x != nil { + return x.Method + } + return PutBlobRequest_GRPC +} + +func (x *PutBlobRequest) GetContentType() string { + if x != nil { + return x.ContentType + } + return "" +} + +func (x *PutBlobRequest) GetValue() []byte { + if x != nil { + return x.Value + } + return nil +} + +type PutBlobResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Error details + Error *ErrorResult `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"` + // The blob uid. This must be saved into the resource to support access + Uid string `protobuf:"bytes,2,opt,name=uid,proto3" json:"uid,omitempty"` + // The URL where this value can be PUT + Url string `protobuf:"bytes,3,opt,name=url,proto3" json:"url,omitempty"` + // Size of the uploaded blob + Size int64 `protobuf:"varint,4,opt,name=size,proto3" json:"size,omitempty"` + // Content hash used for an etag + Hash string `protobuf:"bytes,5,opt,name=hash,proto3" json:"hash,omitempty"` + // Validated mimetype (from content_type) + MimeType string `protobuf:"bytes,6,opt,name=mime_type,json=mimeType,proto3" json:"mime_type,omitempty"` + // Validated charset (from content_type) + Charset string `protobuf:"bytes,7,opt,name=charset,proto3" json:"charset,omitempty"` +} + +func (x *PutBlobResponse) Reset() { + *x = PutBlobResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_resource_proto_msgTypes[30] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PutBlobResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PutBlobResponse) ProtoMessage() {} + +func (x *PutBlobResponse) ProtoReflect() protoreflect.Message { + mi := &file_resource_proto_msgTypes[30] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PutBlobResponse.ProtoReflect.Descriptor instead. +func (*PutBlobResponse) Descriptor() ([]byte, []int) { + return file_resource_proto_rawDescGZIP(), []int{30} +} + +func (x *PutBlobResponse) GetError() *ErrorResult { + if x != nil { + return x.Error + } + return nil +} + +func (x *PutBlobResponse) GetUid() string { + if x != nil { + return x.Uid + } + return "" +} + +func (x *PutBlobResponse) GetUrl() string { + if x != nil { + return x.Url + } + return "" +} + +func (x *PutBlobResponse) GetSize() int64 { + if x != nil { + return x.Size + } + return 0 +} + +func (x *PutBlobResponse) GetHash() string { + if x != nil { + return x.Hash + } + return "" +} + +func (x *PutBlobResponse) GetMimeType() string { + if x != nil { + return x.MimeType + } + return "" +} + +func (x *PutBlobResponse) GetCharset() string { + if x != nil { + return x.Charset + } + return "" +} + +type GetBlobRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Resource *ResourceKey `protobuf:"bytes,1,opt,name=resource,proto3" json:"resource,omitempty"` + // The new resource version + ResourceVersion int64 `protobuf:"varint,2,opt,name=resource_version,json=resourceVersion,proto3" json:"resource_version,omitempty"` + // Do not return a pre-signed URL (when possible) + MustProxyBytes bool `protobuf:"varint,3,opt,name=must_proxy_bytes,json=mustProxyBytes,proto3" json:"must_proxy_bytes,omitempty"` +} + +func (x *GetBlobRequest) Reset() { + *x = GetBlobRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_resource_proto_msgTypes[31] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetBlobRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetBlobRequest) ProtoMessage() {} + +func (x *GetBlobRequest) ProtoReflect() protoreflect.Message { + mi := &file_resource_proto_msgTypes[31] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetBlobRequest.ProtoReflect.Descriptor instead. +func (*GetBlobRequest) Descriptor() ([]byte, []int) { + return file_resource_proto_rawDescGZIP(), []int{31} +} + +func (x *GetBlobRequest) GetResource() *ResourceKey { + if x != nil { + return x.Resource + } + return nil +} + +func (x *GetBlobRequest) GetResourceVersion() int64 { + if x != nil { + return x.ResourceVersion + } + return 0 +} + +func (x *GetBlobRequest) GetMustProxyBytes() bool { + if x != nil { + return x.MustProxyBytes + } + return false +} + +type GetBlobResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Error details + Error *ErrorResult `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"` + // (optional) When possible, the system will return a presigned URL + // that can be used to actually read the full blob+metadata + // When this is set, neither info nor value will be set + Url string `protobuf:"bytes,2,opt,name=url,proto3" json:"url,omitempty"` + // Content type + ContentType string `protobuf:"bytes,3,opt,name=content_type,json=contentType,proto3" json:"content_type,omitempty"` + // The raw object value + Value []byte `protobuf:"bytes,4,opt,name=value,proto3" json:"value,omitempty"` +} + +func (x *GetBlobResponse) Reset() { + *x = GetBlobResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_resource_proto_msgTypes[32] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetBlobResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetBlobResponse) ProtoMessage() {} + +func (x *GetBlobResponse) ProtoReflect() protoreflect.Message { + mi := &file_resource_proto_msgTypes[32] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetBlobResponse.ProtoReflect.Descriptor instead. +func (*GetBlobResponse) Descriptor() ([]byte, []int) { + return file_resource_proto_rawDescGZIP(), []int{32} +} + +func (x *GetBlobResponse) GetError() *ErrorResult { + if x != nil { + return x.Error + } + return nil +} + +func (x *GetBlobResponse) GetUrl() string { + if x != nil { + return x.Url + } + return "" +} + +func (x *GetBlobResponse) GetContentType() string { + if x != nil { + return x.ContentType + } + return "" +} + +func (x *GetBlobResponse) GetValue() []byte { + if x != nil { + return x.Value + } + return nil +} + type WatchEvent_Resource struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2258,7 +2627,7 @@ type WatchEvent_Resource struct { func (x *WatchEvent_Resource) Reset() { *x = WatchEvent_Resource{} if protoimpl.UnsafeEnabled { - mi := &file_resource_proto_msgTypes[29] + mi := &file_resource_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2271,7 +2640,7 @@ func (x *WatchEvent_Resource) String() string { func (*WatchEvent_Resource) ProtoMessage() {} func (x *WatchEvent_Resource) ProtoReflect() protoreflect.Message { - mi := &file_resource_proto_msgTypes[29] + mi := &file_resource_proto_msgTypes[33] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2571,56 +2940,109 @@ var file_resource_proto_rawDesc = []byte{ 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x0f, 0x0a, 0x0b, 0x4e, 0x4f, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x45, 0x52, - 0x56, 0x49, 0x43, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x03, 0x2a, 0x33, - 0x0a, 0x14, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x10, 0x0a, 0x0c, 0x4e, 0x6f, 0x74, 0x4f, 0x6c, 0x64, - 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x78, 0x61, 0x63, - 0x74, 0x10, 0x01, 0x32, 0xed, 0x02, 0x0a, 0x0d, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x35, 0x0a, 0x04, 0x52, 0x65, 0x61, 0x64, 0x12, 0x15, 0x2e, - 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, - 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3b, 0x0a, 0x06, - 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x12, 0x17, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x18, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3b, 0x0a, 0x06, 0x55, 0x70, 0x64, - 0x61, 0x74, 0x65, 0x12, 0x17, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x55, - 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x72, - 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3b, 0x0a, 0x06, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, - 0x12, 0x17, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x6c, 0x65, - 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x72, 0x65, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x35, 0x0a, 0x04, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x15, 0x2e, 0x72, 0x65, - 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x4c, 0x69, - 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x37, 0x0a, 0x05, 0x57, 0x61, - 0x74, 0x63, 0x68, 0x12, 0x16, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x57, - 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x72, 0x65, - 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x57, 0x61, 0x74, 0x63, 0x68, 0x45, 0x76, 0x65, 0x6e, - 0x74, 0x30, 0x01, 0x32, 0xc9, 0x01, 0x0a, 0x0d, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x49, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x3b, 0x0a, 0x06, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, - 0x17, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, - 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x3e, 0x0a, 0x07, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x18, 0x2e, - 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x2e, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x3b, 0x0a, 0x06, 0x4f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x12, 0x17, 0x2e, 0x72, - 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x4f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x2e, 0x4f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, - 0x57, 0x0a, 0x0b, 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x73, 0x12, 0x48, - 0x0a, 0x09, 0x49, 0x73, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x79, 0x12, 0x1c, 0x2e, 0x72, 0x65, - 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, - 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x72, 0x65, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x39, 0x5a, 0x37, 0x67, 0x69, 0x74, 0x68, - 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x72, 0x61, 0x66, 0x61, 0x6e, 0x61, 0x2f, 0x67, - 0x72, 0x61, 0x66, 0x61, 0x6e, 0x61, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x61, - 0x67, 0x65, 0x2f, 0x75, 0x6e, 0x69, 0x66, 0x69, 0x65, 0x64, 0x2f, 0x72, 0x65, 0x73, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x56, 0x49, 0x43, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x03, 0x22, 0xd3, + 0x01, 0x0a, 0x0e, 0x50, 0x75, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x31, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x52, + 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4b, 0x65, 0x79, 0x52, 0x08, 0x72, 0x65, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x12, 0x37, 0x0a, 0x06, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1f, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, + 0x50, 0x75, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4d, + 0x65, 0x74, 0x68, 0x6f, 0x64, 0x52, 0x06, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x21, 0x0a, + 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, + 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x1c, 0x0a, 0x06, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, + 0x12, 0x08, 0x0a, 0x04, 0x47, 0x52, 0x50, 0x43, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x48, 0x54, + 0x54, 0x50, 0x10, 0x01, 0x22, 0xc1, 0x01, 0x0a, 0x0f, 0x50, 0x75, 0x74, 0x42, 0x6c, 0x6f, 0x62, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2b, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, + 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x05, + 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, + 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x12, 0x0a, + 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x61, 0x73, + 0x68, 0x12, 0x1b, 0x0a, 0x09, 0x6d, 0x69, 0x6d, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x06, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6d, 0x69, 0x6d, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x18, + 0x0a, 0x07, 0x63, 0x68, 0x61, 0x72, 0x73, 0x65, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x07, 0x63, 0x68, 0x61, 0x72, 0x73, 0x65, 0x74, 0x22, 0x98, 0x01, 0x0a, 0x0e, 0x47, 0x65, 0x74, + 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x31, 0x0a, 0x08, 0x72, + 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, + 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x4b, 0x65, 0x79, 0x52, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x29, + 0x0a, 0x10, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x28, 0x0a, 0x10, 0x6d, 0x75, 0x73, + 0x74, 0x5f, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x0e, 0x6d, 0x75, 0x73, 0x74, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x42, 0x79, + 0x74, 0x65, 0x73, 0x22, 0x89, 0x01, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2b, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x05, 0x65, + 0x72, 0x72, 0x6f, 0x72, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, + 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2a, + 0x33, 0x0a, 0x14, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x10, 0x0a, 0x0c, 0x4e, 0x6f, 0x74, 0x4f, 0x6c, + 0x64, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x78, 0x61, + 0x63, 0x74, 0x10, 0x01, 0x32, 0xed, 0x02, 0x0a, 0x0d, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x35, 0x0a, 0x04, 0x52, 0x65, 0x61, 0x64, 0x12, 0x15, + 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x2e, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3b, 0x0a, + 0x06, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x12, 0x17, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x18, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x43, 0x72, 0x65, 0x61, + 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3b, 0x0a, 0x06, 0x55, 0x70, + 0x64, 0x61, 0x74, 0x65, 0x12, 0x17, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, + 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3b, 0x0a, 0x06, 0x44, 0x65, 0x6c, 0x65, 0x74, + 0x65, 0x12, 0x17, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x6c, + 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x72, 0x65, 0x73, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x35, 0x0a, 0x04, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x15, 0x2e, 0x72, + 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x4c, + 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x37, 0x0a, 0x05, 0x57, + 0x61, 0x74, 0x63, 0x68, 0x12, 0x16, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, + 0x57, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x72, + 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x57, 0x61, 0x74, 0x63, 0x68, 0x45, 0x76, 0x65, + 0x6e, 0x74, 0x30, 0x01, 0x32, 0xc9, 0x01, 0x0a, 0x0d, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x3b, 0x0a, 0x06, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, + 0x12, 0x17, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x53, 0x65, 0x61, 0x72, + 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x72, 0x65, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, 0x07, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x18, + 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, + 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x2e, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x3b, 0x0a, 0x06, 0x4f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x12, 0x17, 0x2e, + 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x4f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x2e, 0x4f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x32, 0x8b, 0x01, 0x0a, 0x09, 0x42, 0x6c, 0x6f, 0x62, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x3e, + 0x0a, 0x07, 0x50, 0x75, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x12, 0x18, 0x2e, 0x72, 0x65, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x2e, 0x50, 0x75, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x50, + 0x75, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3e, + 0x0a, 0x07, 0x47, 0x65, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x12, 0x18, 0x2e, 0x72, 0x65, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x47, + 0x65, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0x57, + 0x0a, 0x0b, 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x73, 0x12, 0x48, 0x0a, + 0x09, 0x49, 0x73, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x79, 0x12, 0x1c, 0x2e, 0x72, 0x65, 0x73, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, + 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x39, 0x5a, 0x37, 0x67, 0x69, 0x74, 0x68, 0x75, + 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x72, 0x61, 0x66, 0x61, 0x6e, 0x61, 0x2f, 0x67, 0x72, + 0x61, 0x66, 0x61, 0x6e, 0x61, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, + 0x65, 0x2f, 0x75, 0x6e, 0x69, 0x66, 0x69, 0x65, 0x64, 0x2f, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -2635,99 +3057,113 @@ func file_resource_proto_rawDescGZIP() []byte { return file_resource_proto_rawDescData } -var file_resource_proto_enumTypes = make([]protoimpl.EnumInfo, 3) -var file_resource_proto_msgTypes = make([]protoimpl.MessageInfo, 30) +var file_resource_proto_enumTypes = make([]protoimpl.EnumInfo, 4) +var file_resource_proto_msgTypes = make([]protoimpl.MessageInfo, 34) var file_resource_proto_goTypes = []any{ (ResourceVersionMatch)(0), // 0: resource.ResourceVersionMatch (WatchEvent_Type)(0), // 1: resource.WatchEvent.Type (HealthCheckResponse_ServingStatus)(0), // 2: resource.HealthCheckResponse.ServingStatus - (*ResourceKey)(nil), // 3: resource.ResourceKey - (*ResourceWrapper)(nil), // 4: resource.ResourceWrapper - (*ResourceMeta)(nil), // 5: resource.ResourceMeta - (*ErrorResult)(nil), // 6: resource.ErrorResult - (*ErrorDetails)(nil), // 7: resource.ErrorDetails - (*ErrorCause)(nil), // 8: resource.ErrorCause - (*CreateRequest)(nil), // 9: resource.CreateRequest - (*CreateResponse)(nil), // 10: resource.CreateResponse - (*UpdateRequest)(nil), // 11: resource.UpdateRequest - (*UpdateResponse)(nil), // 12: resource.UpdateResponse - (*DeleteRequest)(nil), // 13: resource.DeleteRequest - (*DeleteResponse)(nil), // 14: resource.DeleteResponse - (*ReadRequest)(nil), // 15: resource.ReadRequest - (*ReadResponse)(nil), // 16: resource.ReadResponse - (*Requirement)(nil), // 17: resource.Requirement - (*ListOptions)(nil), // 18: resource.ListOptions - (*ListRequest)(nil), // 19: resource.ListRequest - (*ListResponse)(nil), // 20: resource.ListResponse - (*WatchRequest)(nil), // 21: resource.WatchRequest - (*WatchEvent)(nil), // 22: resource.WatchEvent - (*SearchRequest)(nil), // 23: resource.SearchRequest - (*SearchResponse)(nil), // 24: resource.SearchResponse - (*HistoryRequest)(nil), // 25: resource.HistoryRequest - (*HistoryResponse)(nil), // 26: resource.HistoryResponse - (*OriginRequest)(nil), // 27: resource.OriginRequest - (*ResourceOriginInfo)(nil), // 28: resource.ResourceOriginInfo - (*OriginResponse)(nil), // 29: resource.OriginResponse - (*HealthCheckRequest)(nil), // 30: resource.HealthCheckRequest - (*HealthCheckResponse)(nil), // 31: resource.HealthCheckResponse - (*WatchEvent_Resource)(nil), // 32: resource.WatchEvent.Resource + (PutBlobRequest_Method)(0), // 3: resource.PutBlobRequest.Method + (*ResourceKey)(nil), // 4: resource.ResourceKey + (*ResourceWrapper)(nil), // 5: resource.ResourceWrapper + (*ResourceMeta)(nil), // 6: resource.ResourceMeta + (*ErrorResult)(nil), // 7: resource.ErrorResult + (*ErrorDetails)(nil), // 8: resource.ErrorDetails + (*ErrorCause)(nil), // 9: resource.ErrorCause + (*CreateRequest)(nil), // 10: resource.CreateRequest + (*CreateResponse)(nil), // 11: resource.CreateResponse + (*UpdateRequest)(nil), // 12: resource.UpdateRequest + (*UpdateResponse)(nil), // 13: resource.UpdateResponse + (*DeleteRequest)(nil), // 14: resource.DeleteRequest + (*DeleteResponse)(nil), // 15: resource.DeleteResponse + (*ReadRequest)(nil), // 16: resource.ReadRequest + (*ReadResponse)(nil), // 17: resource.ReadResponse + (*Requirement)(nil), // 18: resource.Requirement + (*ListOptions)(nil), // 19: resource.ListOptions + (*ListRequest)(nil), // 20: resource.ListRequest + (*ListResponse)(nil), // 21: resource.ListResponse + (*WatchRequest)(nil), // 22: resource.WatchRequest + (*WatchEvent)(nil), // 23: resource.WatchEvent + (*SearchRequest)(nil), // 24: resource.SearchRequest + (*SearchResponse)(nil), // 25: resource.SearchResponse + (*HistoryRequest)(nil), // 26: resource.HistoryRequest + (*HistoryResponse)(nil), // 27: resource.HistoryResponse + (*OriginRequest)(nil), // 28: resource.OriginRequest + (*ResourceOriginInfo)(nil), // 29: resource.ResourceOriginInfo + (*OriginResponse)(nil), // 30: resource.OriginResponse + (*HealthCheckRequest)(nil), // 31: resource.HealthCheckRequest + (*HealthCheckResponse)(nil), // 32: resource.HealthCheckResponse + (*PutBlobRequest)(nil), // 33: resource.PutBlobRequest + (*PutBlobResponse)(nil), // 34: resource.PutBlobResponse + (*GetBlobRequest)(nil), // 35: resource.GetBlobRequest + (*GetBlobResponse)(nil), // 36: resource.GetBlobResponse + (*WatchEvent_Resource)(nil), // 37: resource.WatchEvent.Resource } var file_resource_proto_depIdxs = []int32{ - 7, // 0: resource.ErrorResult.details:type_name -> resource.ErrorDetails - 8, // 1: resource.ErrorDetails.causes:type_name -> resource.ErrorCause - 3, // 2: resource.CreateRequest.key:type_name -> resource.ResourceKey - 6, // 3: resource.CreateResponse.error:type_name -> resource.ErrorResult - 3, // 4: resource.UpdateRequest.key:type_name -> resource.ResourceKey - 6, // 5: resource.UpdateResponse.error:type_name -> resource.ErrorResult - 3, // 6: resource.DeleteRequest.key:type_name -> resource.ResourceKey - 6, // 7: resource.DeleteResponse.error:type_name -> resource.ErrorResult - 3, // 8: resource.ReadRequest.key:type_name -> resource.ResourceKey - 6, // 9: resource.ReadResponse.error:type_name -> resource.ErrorResult - 3, // 10: resource.ListOptions.key:type_name -> resource.ResourceKey - 17, // 11: resource.ListOptions.labels:type_name -> resource.Requirement - 17, // 12: resource.ListOptions.fields:type_name -> resource.Requirement + 8, // 0: resource.ErrorResult.details:type_name -> resource.ErrorDetails + 9, // 1: resource.ErrorDetails.causes:type_name -> resource.ErrorCause + 4, // 2: resource.CreateRequest.key:type_name -> resource.ResourceKey + 7, // 3: resource.CreateResponse.error:type_name -> resource.ErrorResult + 4, // 4: resource.UpdateRequest.key:type_name -> resource.ResourceKey + 7, // 5: resource.UpdateResponse.error:type_name -> resource.ErrorResult + 4, // 6: resource.DeleteRequest.key:type_name -> resource.ResourceKey + 7, // 7: resource.DeleteResponse.error:type_name -> resource.ErrorResult + 4, // 8: resource.ReadRequest.key:type_name -> resource.ResourceKey + 7, // 9: resource.ReadResponse.error:type_name -> resource.ErrorResult + 4, // 10: resource.ListOptions.key:type_name -> resource.ResourceKey + 18, // 11: resource.ListOptions.labels:type_name -> resource.Requirement + 18, // 12: resource.ListOptions.fields:type_name -> resource.Requirement 0, // 13: resource.ListRequest.version_match:type_name -> resource.ResourceVersionMatch - 18, // 14: resource.ListRequest.options:type_name -> resource.ListOptions - 4, // 15: resource.ListResponse.items:type_name -> resource.ResourceWrapper - 6, // 16: resource.ListResponse.error:type_name -> resource.ErrorResult - 18, // 17: resource.WatchRequest.options:type_name -> resource.ListOptions + 19, // 14: resource.ListRequest.options:type_name -> resource.ListOptions + 5, // 15: resource.ListResponse.items:type_name -> resource.ResourceWrapper + 7, // 16: resource.ListResponse.error:type_name -> resource.ErrorResult + 19, // 17: resource.WatchRequest.options:type_name -> resource.ListOptions 1, // 18: resource.WatchEvent.type:type_name -> resource.WatchEvent.Type - 32, // 19: resource.WatchEvent.resource:type_name -> resource.WatchEvent.Resource - 32, // 20: resource.WatchEvent.previous:type_name -> resource.WatchEvent.Resource - 4, // 21: resource.SearchResponse.items:type_name -> resource.ResourceWrapper - 3, // 22: resource.HistoryRequest.key:type_name -> resource.ResourceKey - 5, // 23: resource.HistoryResponse.items:type_name -> resource.ResourceMeta - 6, // 24: resource.HistoryResponse.error:type_name -> resource.ErrorResult - 3, // 25: resource.OriginRequest.key:type_name -> resource.ResourceKey - 3, // 26: resource.ResourceOriginInfo.key:type_name -> resource.ResourceKey - 28, // 27: resource.OriginResponse.items:type_name -> resource.ResourceOriginInfo - 6, // 28: resource.OriginResponse.error:type_name -> resource.ErrorResult + 37, // 19: resource.WatchEvent.resource:type_name -> resource.WatchEvent.Resource + 37, // 20: resource.WatchEvent.previous:type_name -> resource.WatchEvent.Resource + 5, // 21: resource.SearchResponse.items:type_name -> resource.ResourceWrapper + 4, // 22: resource.HistoryRequest.key:type_name -> resource.ResourceKey + 6, // 23: resource.HistoryResponse.items:type_name -> resource.ResourceMeta + 7, // 24: resource.HistoryResponse.error:type_name -> resource.ErrorResult + 4, // 25: resource.OriginRequest.key:type_name -> resource.ResourceKey + 4, // 26: resource.ResourceOriginInfo.key:type_name -> resource.ResourceKey + 29, // 27: resource.OriginResponse.items:type_name -> resource.ResourceOriginInfo + 7, // 28: resource.OriginResponse.error:type_name -> resource.ErrorResult 2, // 29: resource.HealthCheckResponse.status:type_name -> resource.HealthCheckResponse.ServingStatus - 15, // 30: resource.ResourceStore.Read:input_type -> resource.ReadRequest - 9, // 31: resource.ResourceStore.Create:input_type -> resource.CreateRequest - 11, // 32: resource.ResourceStore.Update:input_type -> resource.UpdateRequest - 13, // 33: resource.ResourceStore.Delete:input_type -> resource.DeleteRequest - 19, // 34: resource.ResourceStore.List:input_type -> resource.ListRequest - 21, // 35: resource.ResourceStore.Watch:input_type -> resource.WatchRequest - 23, // 36: resource.ResourceIndex.Search:input_type -> resource.SearchRequest - 25, // 37: resource.ResourceIndex.History:input_type -> resource.HistoryRequest - 27, // 38: resource.ResourceIndex.Origin:input_type -> resource.OriginRequest - 30, // 39: resource.Diagnostics.IsHealthy:input_type -> resource.HealthCheckRequest - 16, // 40: resource.ResourceStore.Read:output_type -> resource.ReadResponse - 10, // 41: resource.ResourceStore.Create:output_type -> resource.CreateResponse - 12, // 42: resource.ResourceStore.Update:output_type -> resource.UpdateResponse - 14, // 43: resource.ResourceStore.Delete:output_type -> resource.DeleteResponse - 20, // 44: resource.ResourceStore.List:output_type -> resource.ListResponse - 22, // 45: resource.ResourceStore.Watch:output_type -> resource.WatchEvent - 24, // 46: resource.ResourceIndex.Search:output_type -> resource.SearchResponse - 26, // 47: resource.ResourceIndex.History:output_type -> resource.HistoryResponse - 29, // 48: resource.ResourceIndex.Origin:output_type -> resource.OriginResponse - 31, // 49: resource.Diagnostics.IsHealthy:output_type -> resource.HealthCheckResponse - 40, // [40:50] is the sub-list for method output_type - 30, // [30:40] is the sub-list for method input_type - 30, // [30:30] is the sub-list for extension type_name - 30, // [30:30] is the sub-list for extension extendee - 0, // [0:30] is the sub-list for field type_name + 4, // 30: resource.PutBlobRequest.resource:type_name -> resource.ResourceKey + 3, // 31: resource.PutBlobRequest.method:type_name -> resource.PutBlobRequest.Method + 7, // 32: resource.PutBlobResponse.error:type_name -> resource.ErrorResult + 4, // 33: resource.GetBlobRequest.resource:type_name -> resource.ResourceKey + 7, // 34: resource.GetBlobResponse.error:type_name -> resource.ErrorResult + 16, // 35: resource.ResourceStore.Read:input_type -> resource.ReadRequest + 10, // 36: resource.ResourceStore.Create:input_type -> resource.CreateRequest + 12, // 37: resource.ResourceStore.Update:input_type -> resource.UpdateRequest + 14, // 38: resource.ResourceStore.Delete:input_type -> resource.DeleteRequest + 20, // 39: resource.ResourceStore.List:input_type -> resource.ListRequest + 22, // 40: resource.ResourceStore.Watch:input_type -> resource.WatchRequest + 24, // 41: resource.ResourceIndex.Search:input_type -> resource.SearchRequest + 26, // 42: resource.ResourceIndex.History:input_type -> resource.HistoryRequest + 28, // 43: resource.ResourceIndex.Origin:input_type -> resource.OriginRequest + 33, // 44: resource.BlobStore.PutBlob:input_type -> resource.PutBlobRequest + 35, // 45: resource.BlobStore.GetBlob:input_type -> resource.GetBlobRequest + 31, // 46: resource.Diagnostics.IsHealthy:input_type -> resource.HealthCheckRequest + 17, // 47: resource.ResourceStore.Read:output_type -> resource.ReadResponse + 11, // 48: resource.ResourceStore.Create:output_type -> resource.CreateResponse + 13, // 49: resource.ResourceStore.Update:output_type -> resource.UpdateResponse + 15, // 50: resource.ResourceStore.Delete:output_type -> resource.DeleteResponse + 21, // 51: resource.ResourceStore.List:output_type -> resource.ListResponse + 23, // 52: resource.ResourceStore.Watch:output_type -> resource.WatchEvent + 25, // 53: resource.ResourceIndex.Search:output_type -> resource.SearchResponse + 27, // 54: resource.ResourceIndex.History:output_type -> resource.HistoryResponse + 30, // 55: resource.ResourceIndex.Origin:output_type -> resource.OriginResponse + 34, // 56: resource.BlobStore.PutBlob:output_type -> resource.PutBlobResponse + 36, // 57: resource.BlobStore.GetBlob:output_type -> resource.GetBlobResponse + 32, // 58: resource.Diagnostics.IsHealthy:output_type -> resource.HealthCheckResponse + 47, // [47:59] is the sub-list for method output_type + 35, // [35:47] is the sub-list for method input_type + 35, // [35:35] is the sub-list for extension type_name + 35, // [35:35] is the sub-list for extension extendee + 0, // [0:35] is the sub-list for field type_name } func init() { file_resource_proto_init() } @@ -3085,6 +3521,54 @@ func file_resource_proto_init() { } } file_resource_proto_msgTypes[29].Exporter = func(v any, i int) any { + switch v := v.(*PutBlobRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_resource_proto_msgTypes[30].Exporter = func(v any, i int) any { + switch v := v.(*PutBlobResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_resource_proto_msgTypes[31].Exporter = func(v any, i int) any { + switch v := v.(*GetBlobRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_resource_proto_msgTypes[32].Exporter = func(v any, i int) any { + switch v := v.(*GetBlobResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_resource_proto_msgTypes[33].Exporter = func(v any, i int) any { switch v := v.(*WatchEvent_Resource); i { case 0: return &v.state @@ -3102,10 +3586,10 @@ func file_resource_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_resource_proto_rawDesc, - NumEnums: 3, - NumMessages: 30, + NumEnums: 4, + NumMessages: 34, NumExtensions: 0, - NumServices: 3, + NumServices: 4, }, GoTypes: file_resource_proto_goTypes, DependencyIndexes: file_resource_proto_depIdxs, diff --git a/pkg/storage/unified/resource/resource.proto b/pkg/storage/unified/resource/resource.proto index bfbb654a3ae..8379b4951f1 100644 --- a/pkg/storage/unified/resource/resource.proto +++ b/pkg/storage/unified/resource/resource.proto @@ -218,7 +218,6 @@ message Requirement { repeated string values = 3; // typically one value, but depends on the operator } - message ListOptions { // Group+Namespace+Resource (not name) ResourceKey key = 1; @@ -432,6 +431,82 @@ message HealthCheckResponse { ServingStatus status = 1; } +//---------------------------- +// Blob Support +//---------------------------- + +message PutBlobRequest { + enum Method { + // Use the inline raw []byte + GRPC = 0; + + // Get a signed URL and PUT the value + HTTP = 1; + } + + // The resource that will use this blob + // NOTE: the name may not yet exist, but group+resource are required + ResourceKey resource = 1; + + // How to upload + Method method = 2; + + // Content type header + string content_type = 3; + + // Raw value to write + // Not valid when method == HTTP + bytes value = 4; +} + +message PutBlobResponse { + // Error details + ErrorResult error = 1; + + // The blob uid. This must be saved into the resource to support access + string uid = 2; + + // The URL where this value can be PUT + string url = 3; + + // Size of the uploaded blob + int64 size = 4; + + // Content hash used for an etag + string hash = 5; + + // Validated mimetype (from content_type) + string mime_type = 6; + + // Validated charset (from content_type) + string charset = 7; +} + +message GetBlobRequest { + ResourceKey resource = 1; + + // The new resource version + int64 resource_version = 2; + + // Do not return a pre-signed URL (when possible) + bool must_proxy_bytes = 3; +} + +message GetBlobResponse { + // Error details + ErrorResult error = 1; + + // (optional) When possible, the system will return a presigned URL + // that can be used to actually read the full blob+metadata + // When this is set, neither info nor value will be set + string url = 2; + + // Content type + string content_type = 3; + + // The raw object value + bytes value = 4; +} // This provides the CRUD+List+Watch support needed for a k8s apiserver // The semantics and behaviors of this service are constrained by kubernetes @@ -466,6 +541,18 @@ service ResourceIndex { rpc Origin(OriginRequest) returns (OriginResponse); } +service BlobStore { + // Upload a blob that will be saved in a resource + rpc PutBlob(PutBlobRequest) returns (PutBlobResponse); + + // Get blob contents. When possible, this will return a signed URL + // For large payloads, signed URLs are required to avoid protobuf message size limits + rpc GetBlob(GetBlobRequest) returns (GetBlobResponse); + + // NOTE: there is no direct access to delete blobs + // >> cleanup will be managed via garbage collection or direct access to the underlying storage +} + // Clients can use this service directly // NOTE: This is read only, and no read afer write guarantees service Diagnostics { diff --git a/pkg/storage/unified/resource/resource_grpc.pb.go b/pkg/storage/unified/resource/resource_grpc.pb.go index f349db5c566..c8c34cfeeb1 100644 --- a/pkg/storage/unified/resource/resource_grpc.pb.go +++ b/pkg/storage/unified/resource/resource_grpc.pb.go @@ -522,6 +522,139 @@ var ResourceIndex_ServiceDesc = grpc.ServiceDesc{ Metadata: "resource.proto", } +const ( + BlobStore_PutBlob_FullMethodName = "/resource.BlobStore/PutBlob" + BlobStore_GetBlob_FullMethodName = "/resource.BlobStore/GetBlob" +) + +// BlobStoreClient is the client API for BlobStore service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type BlobStoreClient interface { + // Upload a blob that will be saved in a resource + PutBlob(ctx context.Context, in *PutBlobRequest, opts ...grpc.CallOption) (*PutBlobResponse, error) + // Get blob contents. When possible, this will return a signed URL + // For large payloads, signed URLs are required to avoid protobuf message size limits + GetBlob(ctx context.Context, in *GetBlobRequest, opts ...grpc.CallOption) (*GetBlobResponse, error) +} + +type blobStoreClient struct { + cc grpc.ClientConnInterface +} + +func NewBlobStoreClient(cc grpc.ClientConnInterface) BlobStoreClient { + return &blobStoreClient{cc} +} + +func (c *blobStoreClient) PutBlob(ctx context.Context, in *PutBlobRequest, opts ...grpc.CallOption) (*PutBlobResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(PutBlobResponse) + err := c.cc.Invoke(ctx, BlobStore_PutBlob_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *blobStoreClient) GetBlob(ctx context.Context, in *GetBlobRequest, opts ...grpc.CallOption) (*GetBlobResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GetBlobResponse) + err := c.cc.Invoke(ctx, BlobStore_GetBlob_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// BlobStoreServer is the server API for BlobStore service. +// All implementations should embed UnimplementedBlobStoreServer +// for forward compatibility +type BlobStoreServer interface { + // Upload a blob that will be saved in a resource + PutBlob(context.Context, *PutBlobRequest) (*PutBlobResponse, error) + // Get blob contents. When possible, this will return a signed URL + // For large payloads, signed URLs are required to avoid protobuf message size limits + GetBlob(context.Context, *GetBlobRequest) (*GetBlobResponse, error) +} + +// UnimplementedBlobStoreServer should be embedded to have forward compatible implementations. +type UnimplementedBlobStoreServer struct { +} + +func (UnimplementedBlobStoreServer) PutBlob(context.Context, *PutBlobRequest) (*PutBlobResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method PutBlob not implemented") +} +func (UnimplementedBlobStoreServer) GetBlob(context.Context, *GetBlobRequest) (*GetBlobResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetBlob not implemented") +} + +// UnsafeBlobStoreServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to BlobStoreServer will +// result in compilation errors. +type UnsafeBlobStoreServer interface { + mustEmbedUnimplementedBlobStoreServer() +} + +func RegisterBlobStoreServer(s grpc.ServiceRegistrar, srv BlobStoreServer) { + s.RegisterService(&BlobStore_ServiceDesc, srv) +} + +func _BlobStore_PutBlob_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(PutBlobRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BlobStoreServer).PutBlob(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: BlobStore_PutBlob_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BlobStoreServer).PutBlob(ctx, req.(*PutBlobRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _BlobStore_GetBlob_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetBlobRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BlobStoreServer).GetBlob(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: BlobStore_GetBlob_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BlobStoreServer).GetBlob(ctx, req.(*GetBlobRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// BlobStore_ServiceDesc is the grpc.ServiceDesc for BlobStore service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var BlobStore_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "resource.BlobStore", + HandlerType: (*BlobStoreServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "PutBlob", + Handler: _BlobStore_PutBlob_Handler, + }, + { + MethodName: "GetBlob", + Handler: _BlobStore_GetBlob_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "resource.proto", +} + const ( Diagnostics_IsHealthy_FullMethodName = "/resource.Diagnostics/IsHealthy" ) diff --git a/pkg/storage/unified/resource/server.go b/pkg/storage/unified/resource/server.go index 0a8da09d322..801c3198a36 100644 --- a/pkg/storage/unified/resource/server.go +++ b/pkg/storage/unified/resource/server.go @@ -10,21 +10,22 @@ import ( "sync/atomic" "time" + "github.com/grafana/authlib/claims" + "github.com/grafana/grafana/pkg/apimachinery/identity" + "github.com/grafana/grafana/pkg/apimachinery/utils" + "github.com/prometheus/client_golang/prometheus" "go.opentelemetry.io/otel/trace" "go.opentelemetry.io/otel/trace/noop" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - - "github.com/grafana/authlib/claims" - "github.com/grafana/grafana/pkg/apimachinery/identity" - "github.com/grafana/grafana/pkg/apimachinery/utils" ) // ResourceServer implements all gRPC services type ResourceServer interface { ResourceStoreServer ResourceIndexServer + BlobStoreServer DiagnosticsServer } @@ -76,6 +77,31 @@ type StorageBackend interface { WatchWriteEvents(ctx context.Context) (<-chan *WrittenEvent, error) } +// This interface is not exposed to end users directly +// Access to this interface is already gated by access control +type BlobSupport interface { + // Indicates if storage layer supports signed urls + SupportsSignedURLs() bool + + // Get the raw blob bytes and metadata -- limited to protobuf message size + // For larger payloads, we should use presigned URLs to upload from the client + PutResourceBlob(context.Context, *PutBlobRequest) (*PutBlobResponse, error) + + // Get blob contents. When possible, this will return a signed URL + // For large payloads, signed URLs are required to avoid protobuf message size limits + GetResourceBlob(ctx context.Context, resource *ResourceKey, info *utils.BlobInfo, mustProxy bool) (*GetBlobResponse, error) + + // TODO? List+Delete? This is for admin access +} + +type BlobConfig struct { + // The CDK configuration URL + URL string + + // Directly implemented blob support + Backend BlobSupport +} + type ResourceServerOptions struct { // OTel tracer Tracer trace.Tracer @@ -83,6 +109,9 @@ type ResourceServerOptions struct { // Real storage backend Backend StorageBackend + // The blob configuration + Blob BlobConfig + // Requests based on a search index Index ResourceIndexServer @@ -98,6 +127,9 @@ type ResourceServerOptions struct { // Get the current time in unix millis Now func() int64 + + // Registerer to register prometheus Metrics for the Resource server + Reg prometheus.Registerer } func NewResourceServer(opts ResourceServerOptions) (ResourceServer, error) { @@ -118,6 +150,24 @@ func NewResourceServer(opts ResourceServerOptions) (ResourceServer, error) { } } + // Initialize the blob storage + blobstore := opts.Blob.Backend + if blobstore == nil && opts.Blob.URL != "" { + ctx := context.Background() + bucket, err := OpenBlobBucket(ctx, opts.Blob.URL) + if err != nil { + return nil, err + } + + blobstore, err = NewCDKBlobSupport(ctx, CDKBlobSupportOptions{ + Tracer: opts.Tracer, + Bucket: NewInstrumentedBucket(bucket, opts.Reg, opts.Tracer), + }) + if err != nil { + return nil, err + } + } + // Make this cancelable ctx, cancel := context.WithCancel(claims.WithClaims(context.Background(), &identity.StaticRequester{ @@ -131,6 +181,7 @@ func NewResourceServer(opts ResourceServerOptions) (ResourceServer, error) { log: slog.Default().With("logger", "resource-server"), backend: opts.Backend, index: opts.Index, + blob: blobstore, diagnostics: opts.Diagnostics, access: opts.WriteAccess, lifecycle: opts.Lifecycle, @@ -146,6 +197,7 @@ type server struct { tracer trace.Tracer log *slog.Logger backend StorageBackend + blob BlobSupport index ResourceIndexServer diagnostics DiagnosticsServer access WriteAccessHooks @@ -717,3 +769,77 @@ func (s *server) IsHealthy(ctx context.Context, req *HealthCheckRequest) (*Healt } return s.diagnostics.IsHealthy(ctx, req) } + +// GetBlob implements BlobStore. +func (s *server) PutBlob(ctx context.Context, req *PutBlobRequest) (*PutBlobResponse, error) { + if s.blob == nil { + return &PutBlobResponse{Error: &ErrorResult{ + Message: "blob store not configured", + Code: http.StatusNotImplemented, + }}, nil + } + if err := s.Init(ctx); err != nil { + return nil, err + } + + rsp, err := s.blob.PutResourceBlob(ctx, req) + if err != nil { + rsp.Error = AsErrorResult(err) + } + return rsp, nil +} + +func (s *server) getPartialObject(ctx context.Context, key *ResourceKey, rv int64) (utils.GrafanaMetaAccessor, *ErrorResult) { + rsp := s.backend.ReadResource(ctx, &ReadRequest{ + Key: key, + ResourceVersion: rv, + }) + if rsp.Error != nil { + return nil, rsp.Error + } + + partial := &metav1.PartialObjectMetadata{} + err := json.Unmarshal(rsp.Value, partial) + if err != nil { + return nil, AsErrorResult(err) + } + obj, err := utils.MetaAccessor(partial) + if err != nil { + return nil, AsErrorResult(err) + } + return obj, nil +} + +// GetBlob implements BlobStore. +func (s *server) GetBlob(ctx context.Context, req *GetBlobRequest) (*GetBlobResponse, error) { + if s.blob == nil { + return &GetBlobResponse{Error: &ErrorResult{ + Message: "blob store not configured", + Code: http.StatusNotImplemented, + }}, nil + } + + if err := s.Init(ctx); err != nil { + return nil, err + } + + // The linked blob is stored in the resource metadata attributes + obj, status := s.getPartialObject(ctx, req.Resource, req.ResourceVersion) + if status != nil { + return &GetBlobResponse{Error: status}, nil + } + + info := obj.GetBlob() + if info == nil || info.UID == "" { + return &GetBlobResponse{Error: &ErrorResult{ + Message: "Resource does not have a linked blob", + Code: 404, + }}, nil + } + + rsp, err := s.blob.GetResourceBlob(ctx, req.Resource, info, req.MustProxyBytes) + if err != nil { + rsp.Error = AsErrorResult(err) + } + return rsp, nil +} diff --git a/pkg/storage/unified/sql/server.go b/pkg/storage/unified/sql/server.go index bdb108fd47e..bffce91e7b2 100644 --- a/pkg/storage/unified/sql/server.go +++ b/pkg/storage/unified/sql/server.go @@ -3,6 +3,8 @@ package sql import ( "context" "errors" + "os" + "strings" "github.com/grafana/authlib/claims" infraDB "github.com/grafana/grafana/pkg/infra/db" @@ -11,12 +13,28 @@ import ( "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/storage/unified/resource" "github.com/grafana/grafana/pkg/storage/unified/sql/db/dbimpl" + "github.com/prometheus/client_golang/prometheus" ) // Creates a new ResourceServer -func NewResourceServer(ctx context.Context, db infraDB.DB, cfg *setting.Cfg, features featuremgmt.FeatureToggles, tracer tracing.Tracer) (resource.ResourceServer, error) { +func NewResourceServer(ctx context.Context, db infraDB.DB, cfg *setting.Cfg, features featuremgmt.FeatureToggles, tracer tracing.Tracer, reg prometheus.Registerer) (resource.ResourceServer, error) { + apiserverCfg := cfg.SectionWithEnvOverrides("grafana-apiserver") opts := resource.ResourceServerOptions{ Tracer: tracer, + Blob: resource.BlobConfig{ + URL: apiserverCfg.Key("blob_url").MustString(""), + }, + Reg: reg, + } + + // Support local file blob + if strings.HasPrefix(opts.Blob.URL, "./data/") { + dir := strings.Replace(opts.Blob.URL, "./data", cfg.DataPath, 1) + err := os.MkdirAll(dir, 0700) + if err != nil { + return nil, err + } + opts.Blob.URL = "file:///" + dir } eDB, err := dbimpl.ProvideResourceDB(db, cfg, tracer) diff --git a/pkg/storage/unified/sql/service.go b/pkg/storage/unified/sql/service.go index 72d1cb0b279..e068a84efd0 100644 --- a/pkg/storage/unified/sql/service.go +++ b/pkg/storage/unified/sql/service.go @@ -46,6 +46,7 @@ type service struct { authenticator interceptors.Authenticator log log.Logger + reg prometheus.Registerer } func ProvideUnifiedStorageGrpcService( @@ -53,6 +54,7 @@ func ProvideUnifiedStorageGrpcService( features featuremgmt.FeatureToggles, db infraDB.DB, log log.Logger, + reg prometheus.Registerer, ) (UnifiedStorageGrpcService, error) { tracingCfg, err := tracing.ProvideTracingConfig(cfg) if err != nil { @@ -75,6 +77,7 @@ func ProvideUnifiedStorageGrpcService( tracing: tracing, db: db, log: log, + reg: reg, } // This will be used when running as a dskit service @@ -84,7 +87,7 @@ func ProvideUnifiedStorageGrpcService( } func (s *service) start(ctx context.Context) error { - server, err := NewResourceServer(ctx, s.db, s.cfg, s.features, s.tracing) + server, err := NewResourceServer(ctx, s.db, s.cfg, s.features, s.tracing, s.reg) if err != nil { return err } @@ -101,6 +104,7 @@ func (s *service) start(ctx context.Context) error { srv := s.handler.GetServer() resource.RegisterResourceStoreServer(srv, server) resource.RegisterResourceIndexServer(srv, server) + resource.RegisterBlobStoreServer(srv, server) resource.RegisterDiagnosticsServer(srv, server) grpc_health_v1.RegisterHealthServer(srv, healthService) diff --git a/pkg/storage/unified/sql/test/integration_test.go b/pkg/storage/unified/sql/test/integration_test.go index d30210f583c..96c0a397c4b 100644 --- a/pkg/storage/unified/sql/test/integration_test.go +++ b/pkg/storage/unified/sql/test/integration_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/require" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" @@ -350,7 +351,7 @@ func TestClientServer(t *testing.T) { features := featuremgmt.WithFeatures() - svc, err := sql.ProvideUnifiedStorageGrpcService(cfg, features, dbstore, nil) + svc, err := sql.ProvideUnifiedStorageGrpcService(cfg, features, dbstore, nil, prometheus.NewPedanticRegistry()) require.NoError(t, err) var client resource.ResourceStoreClient diff --git a/pkg/tests/testinfra/testinfra.go b/pkg/tests/testinfra/testinfra.go index 8468cb154e2..6f5452c1915 100644 --- a/pkg/tests/testinfra/testinfra.go +++ b/pkg/tests/testinfra/testinfra.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gopkg.in/ini.v1" @@ -97,7 +98,8 @@ func StartGrafanaEnv(t *testing.T, grafDir, cfgPath string) (string, *server.Tes // UnifiedStorageOverGRPC var storage sql.UnifiedStorageGrpcService if runstore { - storage, err = sql.ProvideUnifiedStorageGrpcService(env.Cfg, env.FeatureToggles, env.SQLStore, env.Cfg.Logger) + storage, err = sql.ProvideUnifiedStorageGrpcService(env.Cfg, env.FeatureToggles, env.SQLStore, + env.Cfg.Logger, prometheus.NewPedanticRegistry()) require.NoError(t, err) ctx := context.Background() err = storage.StartAsync(ctx)