public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Provisioning: Expose stats (#101927)

Browse Source
This commit is contained in:
Ryan McKinley
2025-03-11 22:24:35 +03:00
committed by GitHub
parent 4e9bfe7b3c
commit fca3d07286
10 changed files with 303 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/apis/provisioning/v0alpha1/types.go
+21 -6
View File
@@ -348,16 +348,31 @@ type ResourceStats struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata,omitempty"`
// Stats across all unified storage
// When legacy storage is still used, this will offer a shim
// +listType=atomic
Items []ResourceCount `json:"items,omitempty"`
Instance []ResourceCount `json:"instance,omitempty"`
// Stats for each manager
// +listType=atomic
Managed []ManagerStats `json:"managed,omitempty"`
}
type ManagerStats struct {
// Manager kind
Kind utils.ManagerKind `json:"kind,omitempty"`
// Manager identity
Identity string `json:"id,omitempty"`
// stats
Stats []ResourceCount `json:"stats"`
}
type ResourceCount struct {
Kind utils.ManagerKind `json:"kind,omitempty"`
Identity string `json:"id,omitempty"`
Group string `json:"group"`
Resource string `json:"resource"`
Count int64 `json:"count"`
Group string `json:"group"`
Resource string `json:"resource"`
Count int64 `json:"count"`
}
// HistoryList is a list of versions of a resource
pkg/apis/provisioning/v0alpha1/zz_generated.deepcopy.go
+30 -2
View File
@@ -352,6 +352,27 @@ func (in *LocalRepositoryConfig) DeepCopy() *LocalRepositoryConfig {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ManagerStats) DeepCopyInto(out *ManagerStats) {
*out = *in
if in.Stats != nil {
in, out := &in.Stats, &out.Stats
*out = make([]ResourceCount, len(*in))
copy(*out, *in)
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ManagerStats.
func (in *ManagerStats) DeepCopy() *ManagerStats {
if in == nil {
return nil
}
out := new(ManagerStats)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *MigrateJobOptions) DeepCopyInto(out *MigrateJobOptions) {
*out = *in
@@ -656,11 +677,18 @@ func (in *ResourceStats) DeepCopyInto(out *ResourceStats) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ListMeta.DeepCopyInto(&out.ListMeta)
if in.Items != nil {
in, out := &in.Items, &out.Items
if in.Instance != nil {
in, out := &in.Instance, &out.Instance
*out = make([]ResourceCount, len(*in))
copy(*out, *in)
}
if in.Managed != nil {
in, out := &in.Managed, &out.Managed
*out = make([]ManagerStats, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
return
}
pkg/apis/provisioning/v0alpha1/zz_generated.openapi.go
+67 -9
View File
@@ -28,6 +28,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA
"github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.JobSpec": schema_pkg_apis_provisioning_v0alpha1_JobSpec(ref),
"github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.JobStatus": schema_pkg_apis_provisioning_v0alpha1_JobStatus(ref),
"github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.LocalRepositoryConfig": schema_pkg_apis_provisioning_v0alpha1_LocalRepositoryConfig(ref),
"github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.ManagerStats": schema_pkg_apis_provisioning_v0alpha1_ManagerStats(ref),
"github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.MigrateJobOptions": schema_pkg_apis_provisioning_v0alpha1_MigrateJobOptions(ref),
"github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.PullRequestJobOptions": schema_pkg_apis_provisioning_v0alpha1_PullRequestJobOptions(ref),
"github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.Repository": schema_pkg_apis_provisioning_v0alpha1_Repository(ref),
@@ -761,6 +762,49 @@ func schema_pkg_apis_provisioning_v0alpha1_LocalRepositoryConfig(ref common.Refe
}
}
func schema_pkg_apis_provisioning_v0alpha1_ManagerStats(ref common.ReferenceCallback) common.OpenAPIDefinition {
return common.OpenAPIDefinition{
Schema: spec.Schema{
SchemaProps: spec.SchemaProps{
Type: []string{"object"},
Properties: map[string]spec.Schema{
"kind": {
SchemaProps: spec.SchemaProps{
Description: "Manager kind",
Type: []string{"string"},
Format: "",
},
},
"id": {
SchemaProps: spec.SchemaProps{
Description: "Manager identity",
Type: []string{"string"},
Format: "",
},
},
"stats": {
SchemaProps: spec.SchemaProps{
Description: "stats",
Type: []string{"array"},
Items: &spec.SchemaOrArray{
Schema: &spec.Schema{
SchemaProps: spec.SchemaProps{
Default: map[string]interface{}{},
Ref: ref("github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.ResourceCount"),
},
},
},
},
},
},
Required: []string{"stats"},
},
},
Dependencies: []string{
"github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.ResourceCount"},
}
}
func schema_pkg_apis_provisioning_v0alpha1_MigrateJobOptions(ref common.ReferenceCallback) common.OpenAPIDefinition {
return common.OpenAPIDefinition{
Schema: spec.Schema{
@@ -1187,12 +1231,6 @@ func schema_pkg_apis_provisioning_v0alpha1_ResourceCount(ref common.ReferenceCal
SchemaProps: spec.SchemaProps{
Type: []string{"object"},
Properties: map[string]spec.Schema{
"repository": {
SchemaProps: spec.SchemaProps{
Type: []string{"string"},
Format: "",
},
},
"group": {
SchemaProps: spec.SchemaProps{
Default: "",
@@ -1468,14 +1506,15 @@ func schema_pkg_apis_provisioning_v0alpha1_ResourceStats(ref common.ReferenceCal
Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta"),
},
},
"items": {
"instance": {
VendorExtensible: spec.VendorExtensible{
Extensions: spec.Extensions{
"x-kubernetes-list-type": "atomic",
},
},
SchemaProps: spec.SchemaProps{
Type: []string{"array"},
Description: "Stats across all unified storage When legacy storage is still used, this will offer a shim",
Type: []string{"array"},
Items: &spec.SchemaOrArray{
Schema: &spec.Schema{
SchemaProps: spec.SchemaProps{
@@ -1486,11 +1525,30 @@ func schema_pkg_apis_provisioning_v0alpha1_ResourceStats(ref common.ReferenceCal
},
},
},
"managed": {
VendorExtensible: spec.VendorExtensible{
Extensions: spec.Extensions{
"x-kubernetes-list-type": "atomic",
},
},
SchemaProps: spec.SchemaProps{
Description: "Stats for each manager",
Type: []string{"array"},
Items: &spec.SchemaOrArray{
Schema: &spec.Schema{
SchemaProps: spec.SchemaProps{
Default: map[string]interface{}{},
Ref: ref("github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.ManagerStats"),
},
},
},
},
},
},
},
},
Dependencies: []string{
"github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.ResourceCount", "k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta"},
"github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.ManagerStats", "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.ResourceCount", "k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta"},
}
}
pkg/apis/provisioning/v0alpha1/zz_generated.openapi_violation_exceptions.list
+2 -1
View File
@@ -3,14 +3,15 @@ API rule violation: list_type_missing,github.com/grafana/grafana/pkg/apis/provis
API rule violation: list_type_missing,github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1,JobResourceSummary,Errors
API rule violation: list_type_missing,github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1,JobStatus,Errors
API rule violation: list_type_missing,github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1,JobStatus,Summary
API rule violation: list_type_missing,github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1,ManagerStats,Stats
API rule violation: list_type_missing,github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1,RepositoryList,Items
API rule violation: list_type_missing,github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1,RepositorySpec,Workflows
API rule violation: list_type_missing,github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1,RepositoryViewList,Items
API rule violation: list_type_missing,github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1,ResourceList,Items
API rule violation: list_type_missing,github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1,ResourceStats,Items
API rule violation: list_type_missing,github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1,TestResults,Errors
API rule violation: list_type_missing,github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1,WebhookStatus,SubscribedEvents
API rule violation: names_match,github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1,JobSpec,PullRequest
API rule violation: names_match,github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1,ManagerStats,Identity
API rule violation: names_match,github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1,RepositorySpec,GitHub
API rule violation: names_match,github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1,ResourceWrapper,URLs
API rule violation: names_match,github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1,SyncStatus,JobID
pkg/generated/applyconfiguration/provisioning/v0alpha1/resourcecount.go
+3 -12
View File
@@ -7,10 +7,9 @@ package v0alpha1
// ResourceCountApplyConfiguration represents a declarative configuration of the ResourceCount type for use
// with apply.
type ResourceCountApplyConfiguration struct {
Repository *string `json:"repository,omitempty"`
Group *string `json:"group,omitempty"`
Resource *string `json:"resource,omitempty"`
Count *int64 `json:"count,omitempty"`
Group *string `json:"group,omitempty"`
Resource *string `json:"resource,omitempty"`
Count *int64 `json:"count,omitempty"`
}
// ResourceCountApplyConfiguration constructs a declarative configuration of the ResourceCount type for use with
@@ -19,14 +18,6 @@ func ResourceCount() *ResourceCountApplyConfiguration {
return &ResourceCountApplyConfiguration{}
}
// WithRepository sets the Repository field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the Repository field is set to the value of the last call.
func (b *ResourceCountApplyConfiguration) WithRepository(value string) *ResourceCountApplyConfiguration {
b.Repository = &value
return b
}
// WithGroup sets the Group field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the Group field is set to the value of the last call.
pkg/registry/apis/provisioning/jobs/sync/worker.go
+2 -2
View File
@@ -117,11 +117,11 @@ func (r *SyncWorker) Process(ctx context.Context, repo repository.Repository, jo
// Only add stats patch if stats are not nil
if stats, err := r.lister.Stats(ctx, cfg.Namespace, cfg.Name); err != nil {
logger.Error("unable to read stats", "error", err)
} else if stats != nil && stats.Items != nil {
} else if stats != nil && len(stats.Managed) == 1 {
patchOperations = append(patchOperations, map[string]interface{}{
"op": "replace",
"path": "/status/stats",
"value": stats.Items,
"value": stats.Managed[0].Stats,
})
}
pkg/registry/apis/provisioning/register.go
+40 -18
View File
@@ -125,7 +125,7 @@ func NewAPIBuilder(
},
render: render,
clonedir: clonedir,
resourceLister: resources.NewResourceLister(unified),
resourceLister: resources.NewResourceLister(unified, unified, legacyMigrator, storageStatus),
legacyMigrator: legacyMigrator,
storageStatus: storageStatus,
unified: unified,
@@ -737,23 +737,7 @@ spec:
}
}
compBase := "com.github.grafana.grafana.pkg.apis.provisioning.v0alpha1."
schema := oas.Components.Schemas[compBase+"ResourceStats"].Properties["items"]
schema.Items = &spec.SchemaOrArray{
Schema: &spec.Schema{
SchemaProps: spec.SchemaProps{
AllOf: []spec.Schema{
{
SchemaProps: spec.SchemaProps{
Ref: spec.MustCreateRef("#/components/schemas/" + compBase + "ResourceCount"),
},
},
},
},
},
}
oas.Components.Schemas[compBase+"ResourceStats"].Properties["items"] = schema
schema = oas.Components.Schemas[compBase+"RepositoryViewList"].Properties["items"]
schema := oas.Components.Schemas[compBase+"RepositoryViewList"].Properties["items"]
schema.Items = &spec.SchemaOrArray{
Schema: &spec.Schema{
SchemaProps: spec.SchemaProps{
@@ -769,6 +753,44 @@ spec:
}
oas.Components.Schemas[compBase+"RepositoryViewList"].Properties["items"] = schema
countSpec := &spec.SchemaOrArray{
Schema: &spec.Schema{
SchemaProps: spec.SchemaProps{
AllOf: []spec.Schema{
{
SchemaProps: spec.SchemaProps{
Ref: spec.MustCreateRef("#/components/schemas/" + compBase + "ResourceCount"),
},
},
},
},
},
}
managerSpec := &spec.SchemaOrArray{
Schema: &spec.Schema{
SchemaProps: spec.SchemaProps{
AllOf: []spec.Schema{
{
SchemaProps: spec.SchemaProps{
Ref: spec.MustCreateRef("#/components/schemas/" + compBase + "ManagerStats"),
},
},
},
},
},
}
schema = oas.Components.Schemas[compBase+"ResourceStats"].Properties["instance"]
schema.Items = countSpec
oas.Components.Schemas[compBase+"ResourceStats"].Properties["instance"] = schema
schema = oas.Components.Schemas[compBase+"ResourceStats"].Properties["managed"]
schema.Items = managerSpec
oas.Components.Schemas[compBase+"ResourceStats"].Properties["managed"] = schema
schema = oas.Components.Schemas[compBase+"ManagerStats"].Properties["stats"]
schema.Items = countSpec
oas.Components.Schemas[compBase+"ManagerStats"].Properties["stats"] = schema
return oas, nil
}
pkg/registry/apis/provisioning/resources/object.go
+85 -12
View File
@@ -3,8 +3,12 @@ package resources
import (
"context"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/grafana/grafana/pkg/apimachinery/utils"
provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
"github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy"
"github.com/grafana/grafana/pkg/storage/legacysql/dualwrite"
"github.com/grafana/grafana/pkg/storage/unified/resource"
)
@@ -15,18 +19,29 @@ type ResourceLister interface {
}
type ResourceListerFromSearch struct {
index resource.ManagedObjectIndexClient
managed resource.ManagedObjectIndexClient
index resource.ResourceIndexClient
legacyMigrator legacy.LegacyMigrator
storageStatus dualwrite.Service
}
func NewResourceLister(index resource.ManagedObjectIndexClient) ResourceLister {
func NewResourceLister(
managed resource.ManagedObjectIndexClient,
index resource.ResourceIndexClient,
legacyMigrator legacy.LegacyMigrator,
storageStatus dualwrite.Service,
) ResourceLister {
return &ResourceListerFromSearch{
index: index,
index: index,
managed: managed,
legacyMigrator: legacyMigrator,
storageStatus: storageStatus,
}
}
// List implements ResourceLister.
func (o *ResourceListerFromSearch) List(ctx context.Context, namespace, repository string) (*provisioning.ResourceList, error) {
objects, err := o.index.ListManagedObjects(ctx, &resource.ListManagedObjectsRequest{
objects, err := o.managed.ListManagedObjects(ctx, &resource.ListManagedObjectsRequest{
Namespace: namespace,
Kind: string(utils.ManagerKindRepo),
Id: repository,
@@ -56,11 +71,15 @@ func (o *ResourceListerFromSearch) List(ctx context.Context, namespace, reposito
// Stats implements ResourceLister.
func (o *ResourceListerFromSearch) Stats(ctx context.Context, namespace, repository string) (*provisioning.ResourceStats, error) {
counts, err := o.index.CountManagedObjects(ctx, &resource.CountManagedObjectsRequest{
req := &resource.CountManagedObjectsRequest{
Namespace: namespace,
Kind: string(utils.ManagerKindRepo),
Id: repository,
})
}
if repository != "" {
req.Kind = string(utils.ManagerKindRepo)
req.Id = repository
}
counts, err := o.managed.CountManagedObjects(ctx, req)
if err != nil {
return nil, err
}
@@ -68,12 +87,66 @@ func (o *ResourceListerFromSearch) Stats(ctx context.Context, namespace, reposit
return nil, resource.GetError(counts.Error)
}
stats := &provisioning.ResourceStats{}
lookup := make(map[string]*provisioning.ManagerStats)
for _, v := range counts.Items {
stats.Items = append(stats.Items, provisioning.ResourceCount{
Kind: utils.ManagerKind(v.Kind),
Identity: v.Id,
key := v.Kind + ":" + v.Id
m := lookup[key]
if m == nil {
m = &provisioning.ManagerStats{
Kind: utils.ManagerKind(v.Kind),
Identity: v.Id,
}
lookup[key] = m
}
m.Stats = append(m.Stats, provisioning.ResourceCount{
Group: v.Group,
Resource: v.Resource,
Count: v.Count,
})
}
stats := &provisioning.ResourceStats{
TypeMeta: metav1.TypeMeta{
APIVersion: provisioning.SchemeGroupVersion.String(),
Kind: "ResourceStats",
},
}
for _, v := range lookup {
stats.Managed = append(stats.Managed, *v)
}
// When selecting an explicit repository, do not fetch global stats
if repository != "" {
return stats, nil
}
// Get the stats based on what a migration could support
if dualwrite.IsReadingLegacyDashboardsAndFolders(ctx, o.storageStatus) {
rsp, err := o.legacyMigrator.Migrate(ctx, legacy.MigrateOptions{
Namespace: namespace,
WithHistory: false,
OnlyCount: true,
})
if err != nil {
return nil, err
}
for _, v := range rsp.Summary {
stats.Instance = append(stats.Instance, provisioning.ResourceCount{
Group: v.Group,
Resource: v.Resource,
Count: v.Count,
})
}
}
// Get full instance stats
info, err := o.index.GetStats(ctx, &resource.ResourceStatsRequest{
Namespace: namespace,
})
if err != nil {
return nil, err
}
for _, v := range info.Stats {
stats.Instance = append(stats.Instance, provisioning.ResourceCount{
Group: v.Group,
Resource: v.Resource,
Count: v.Count,
pkg/tests/apis/openapi_snapshots/provisioning.grafana.app-v0alpha1.json
+41 -4
View File
@@ -2990,6 +2990,33 @@
}
}
},
"com.github.grafana.grafana.pkg.apis.provisioning.v0alpha1.ManagerStats": {
"type": "object",
"required": [
"stats"
],
"properties": {
"id": {
"description": "Manager identity",
"type": "string"
},
"kind": {
"description": "Manager kind",
"type": "string"
},
"stats": {
"description": "stats",
"type": "array",
"items": {
"allOf": [
{
"$ref": "#/components/schemas/com.github.grafana.grafana.pkg.apis.provisioning.v0alpha1.ResourceCount"
}
]
}
}
}
},
"com.github.grafana.grafana.pkg.apis.provisioning.v0alpha1.MigrateJobOptions": {
"type": "object",
"required": [
@@ -3344,9 +3371,6 @@
"type": "string",
"default": ""
},
"repository": {
"type": "string"
},
"resource": {
"type": "string",
"default": ""
@@ -3543,7 +3567,8 @@
"description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources",
"type": "string"
},
"items": {
"instance": {
"description": "Stats across all unified storage When legacy storage is still used, this will offer a shim",
"type": "array",
"items": {
"allOf": [
@@ -3558,6 +3583,18 @@
"description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds",
"type": "string"
},
"managed": {
"description": "Stats for each manager",
"type": "array",
"items": {
"allOf": [
{
"$ref": "#/components/schemas/com.github.grafana.grafana.pkg.apis.provisioning.v0alpha1.ManagerStats"
}
]
},
"x-kubernetes-list-type": "atomic"
},
"metadata": {
"default": {}
}
public/app/features/provisioning/api/endpoints.gen.ts
+12 -2
View File
@@ -899,7 +899,6 @@ export type HealthStatus = {
export type ResourceCount = {
count: number;
group: string;
repository?: string;
resource: string;
};
export type SyncStatus = {
@@ -1162,12 +1161,23 @@ export type RepositoryViewList = {
/** The backend is using legacy storage FIXME: Not sure where this should be exposed... but we need it somewhere The UI should force the onboarding workflow when this is true */
legacyStorage?: boolean;
};
export type ManagerStats = {
/** Manager identity */
id?: string;
/** Manager kind */
kind?: string;
/** stats */
stats: ResourceCount[];
};
export type ResourceStats = {
/** APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources */
apiVersion?: string;
items?: ResourceCount[];
/** Stats across all unified storage When legacy storage is still used, this will offer a shim */
instance?: ResourceCount[];
/** Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds */
kind?: string;
/** Stats for each manager */
managed?: ManagerStats[];
metadata?: any;
};
export const {